Merge of Content Moderation and Jailbreak leaderboard

.gitignore

@@ -0,0 +1,4 @@
+.venv/
+**/__pycache__
+*.egg-info
+**.DS_Store

README.md

@@ -4,7 +4,7 @@ emoji: 🔥
 colorFrom: green
 colorTo: red
 sdk: gradio
-sdk_version: 6.8.0
+sdk_version: 6.0.2
 app_file: app.py
 pinned: false
 license: cc-by-nc-sa-4.0

app.py

@@ -0,0 +1,34 @@
+import gradio as gr
+from content_moderation import build_cm_tab
+from jailbreak import build_jailbreak_tab
+from shared import GLOBAL_CSS, build_footer
+
+
+with gr.Blocks() as demo:
+    gr.HTML(GLOBAL_CSS)
+
+    gr.Markdown("# BELLS-O Leaderboard")
+
+    with gr.Column(elem_classes=["intro-section"]):
+        gr.Markdown(
+            """
+            **BELLS-O** stands for **Benchmark for the Evaluation of LLM Supervision Systems – Operational**. It is the first operational benchmark that systematically evaluates the operation cost of running LLM guardrails.
+
+            This benchmark tests specialized guardrails as well as generalist LLMs across Input and Output Content Moderation, as well as Jailbreak detection.
+            """
+        )
+
+    with gr.Tabs():
+        with gr.Tab("Content Moderation - Input"):
+            build_cm_tab("input")
+
+        with gr.Tab("Content Moderation - Output"):
+            build_cm_tab("output")
+
+        with gr.Tab("Jailbreak"):
+            build_jailbreak_tab()
+
+    build_footer()
+
+if __name__ == "__main__":
+    demo.launch()

content_moderation.py

@@ -0,0 +1,625 @@
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+import gradio as gr
+import pandas as pd
+
+from shared import (
+    CATEGORY_TABLE_CSS,
+    DEFAULT_WEIGHTS,
+    LEADERBOARD_DARK_MODE_CSS,
+    LEADERBOARD_TABLE_CSS,
+    build_pareto_figure,
+    build_weights,
+    create_empty_pareto_figure,
+    create_weight_bar_html,
+    escape_html,
+    get_color_for_accuracy,
+    sort_by_overall_score,
+)
+
+# ── Data loading ────────────────────────────────────────────────────────
+
+_DATA_DIR = Path(__file__).parent / "data"
+
+def _load_metrics() -> Dict[str, Any]:
+    metrics_path = _DATA_DIR / "content_moderation_metrics.json"
+    if not metrics_path.exists():
+        print(f"Warning: {metrics_path} not found")
+        return {}
+    with open(metrics_path, "r") as f:
+        return json.load(f)
+
+METRICS_DATA = _load_metrics()
+
+
+# ── Constants ───────────────────────────────────────────────────────────
+
+CATEGORY_LABELS = {
+    "hate speech": "Hate Speech",
+    "harmful manipulation": "Harmful Manipulation",
+    "privacy": "Privacy",
+    "cyber": "Cyber",
+    "physical harm": "Physical Harm",
+    "sexual content": "Sexual Content",
+    "integrity & quality violations": "Integrity & Quality",
+    "cbrn": "CBRN",
+    "illegal activities": "Illegal Activities",
+    "self-harm": "Self-Harm",
+    "harm to minors": "Harm to Minors",
+}
+
+DATASET_DESCRIPTIONS: Dict[str, Dict[str, Any]] = {
+    "input": {
+        "url": "https://huggingface.co/datasets/bells-o-project/content-moderation-input",
+        "label": "BELLS-O Content Moderation Input Dataset",
+        "summary": "This dataset contains 1,400 samples across 11 harm categories for evaluating input content moderation. It includes 300 benign prompts (for FPR evaluation) and 1,100 harmful prompts (100 per category). Samples were constructed using three methods: AI-generated (27%), extracted from existing datasets (44%), and handcrafted by experts (29%).",
+        "categories": {cat_key: "" for cat_key in CATEGORY_LABELS},
+        "accuracy_categories": list(CATEGORY_LABELS.keys()),
+        "fpr_categories": ["benign"],
+    },
+    "output": {
+        "url": "https://huggingface.co/datasets/bells-o-project/content-moderation-output",
+        "label": "BELLS-O Content Moderation Output Dataset",
+        "summary": "This dataset contains 1,400 samples across 11 harm categories for evaluating output content moderation. It includes 300 benign outputs (for FPR evaluation) and 1,100 harmful outputs (100 per category). Samples were constructed using three methods: AI-generated (27%), extracted from existing datasets (44%), and handcrafted by experts (29%).",
+        "categories": {cat_key: "" for cat_key in CATEGORY_LABELS},
+        "accuracy_categories": list(CATEGORY_LABELS.keys()),
+        "fpr_categories": ["benign"],
+    },
+}
+
+
+def create_dataset_info_html(dataset_type: str) -> str:
+    """Create an HTML info box describing the selected dataset."""
+    info = DATASET_DESCRIPTIONS.get(dataset_type)
+    if not info:
+        return ""
+
+    ds_label = info.get("label", dataset_type)
+    url = info.get("url", "")
+    summary = info.get("summary", "")
+    categories = info.get("categories", {})
+    accuracy_categories = info.get("accuracy_categories", [])
+    fpr_categories = info.get("fpr_categories", [])
+
+    title_html = f'<a href="{url}" target="_blank">{ds_label}</a>' if url else ds_label
+
+    html = """<style>
+        .ds-info-box {
+            background: #f8fafc;
+            border: 1px solid #e2e8f0;
+            border-radius: 8px;
+            padding: 16px;
+            margin: 12px 0;
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+        }
+        .ds-info-box .ds-title { margin: 0 0 8px 0 !important; padding: 0 !important; color: #1e293b; }
+        .ds-info-box .ds-title a { color: #1e293b; text-decoration: underline; }
+        .ds-info-box .ds-text { color: #475569; font-size: 14px; }
+        .ds-info-box .ds-cat-item { color: #475569; font-size: 13px; margin-bottom: 2px; }
+        @media (prefers-color-scheme: dark) {
+            .ds-info-box {
+                background: #2a2a2a !important;
+                border-color: #4a4a4a !important;
+            }
+            .ds-info-box .ds-title,
+            .ds-info-box .ds-title a { color: #f3f4f6 !important; }
+            .ds-info-box .ds-text { color: #d1d5db !important; }
+            .ds-info-box .ds-cat-item { color: #d1d5db !important; }
+        }
+    </style>"""
+
+    html += f'<div class="ds-info-box"><h4 class="ds-title">{title_html}</h4>'
+
+    if summary:
+        html += f'<p class="ds-text" style="margin:0 0 8px 0;">{summary}</p>'
+
+    # Only show categories list if descriptions are non-empty
+    non_empty_cats = {k: v for k, v in categories.items() if v}
+    if non_empty_cats:
+        html += '<div style="margin-top:8px;"><strong class="ds-text">Categories:</strong>'
+        html += '<ul style="margin:4px 0 0 0; padding-left:20px; list-style:none;">'
+        for cat_key, cat_desc in non_empty_cats.items():
+            cat_label = CATEGORY_LABELS.get(cat_key, cat_key)
+            html += f'<li class="ds-cat-item"><strong>{cat_label}:</strong> {cat_desc}</li>'
+        html += "</ul></div>"
+
+    if accuracy_categories:
+        cat_names = ", ".join(CATEGORY_LABELS.get(c, c) for c in accuracy_categories)
+        html += f'<p class="ds-text" style="margin:8px 0 0 0;"><strong>Detection Rate from:</strong> {cat_names}</p>'
+
+    if fpr_categories:
+        cat_names = ", ".join(CATEGORY_LABELS.get(c, c) for c in fpr_categories)
+        html += f'<p class="ds-text" style="margin:4px 0 0 0;"><strong>FPR from:</strong> {cat_names}</p>'
+
+    html += "</div>"
+    return html
+
+
+# ── Data helpers ────────────────────────────────────────────────────────
+
+def filter_metrics_by_dataset(dataset_type: str) -> Dict[str, Any]:
+    dataset_name = f"bells-o-project-content-moderation-{dataset_type}"
+    return {k: v for k, v in METRICS_DATA.items() if v.get("dataset_name") == dataset_name}
+
+
+def prepare_leaderboard_data(selected_categories: List[str] | None = None, dataset_type: str = "input") -> pd.DataFrame:
+    metrics_data = filter_metrics_by_dataset(dataset_type)
+
+    if not metrics_data:
+        return pd.DataFrame(
+            columns=[
+                "Model Snapshot", "Model Developer", "Provider", "Detection Rate (%)",
+                "FPR (%)", "Latency CI 95% (ms)", "Mean Latency (ms)", "Compute Access",
+                "Total Cost", "Cost per 1M units", "Cost per h", "Cost Additional Info",
+                "Model Type", "Execution Info",
+            ]
+        )
+
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    use_overall = (
+        selected_categories is None or len(selected_categories) == 0 or set(selected_categories) == all_categories
+    )
+
+    rows = []
+    for key, data in metrics_data.items():
+        latency_ci = data.get("latency_ci_95", {})
+        cost_info = data.get("cost_info", {})
+
+        latency_ci_str = (
+            f"[{int(round(latency_ci.get('lower', 0), 3) * 1000)}, {int(round(latency_ci.get('upper', 0), 3) * 1000)}]"
+        )
+
+        cost_input = cost_info.get("cost_per_1M_input_tokens", "N/A")
+        cost_output = cost_info.get("cost_per_1M_output_tokens", "N/A")
+        if cost_input == "N/A" or cost_output == "N/A":
+            cost_str = "Unknown"
+        else:
+            cost_str = f"Input: ${round(cost_input, 2)}/1M, Output: ${round(cost_output, 2)}/1M"
+
+        if use_overall:
+            accuracy = data.get("accuracy", 0)
+        else:
+            accuracy_per_category = data.get("accuracy_per_category", {})
+            selected_accuracies = [
+                accuracy_per_category.get(cat, 0) for cat in selected_categories if cat in accuracy_per_category
+            ]
+            accuracy = sum(selected_accuracies) / len(selected_accuracies) if selected_accuracies else 0
+
+        accuracy_pct = accuracy * 100
+        fpr_pct = data.get("fpr", 0) * 100
+
+        cost_per_h_value = data["cost_info"].get("cost_per_h", "Unknown")
+        cost_per_h_str = f"${cost_per_h_value}" if cost_per_h_value != "N/A" else cost_per_h_value
+
+        rows.append(
+            {
+                "Model Snapshot": data.get("model_name", ""),
+                "Model URL": data.get("model_url", ""),
+                "Model Developer": data.get("model_developer", ""),
+                "Provider": data.get("provider", ""),
+                "Detection Rate (%)": round(accuracy_pct, 2),
+                "FPR (%)": round(fpr_pct, 2),
+                "Latency CI 95% (ms)": latency_ci_str,
+                "Mean Latency (ms)": int(round(data.get("mean_latency", 0), 3) * 1000),
+                "Compute Access": data["execution_specifications"].get("type", "Unknown"),
+                "Total Cost": f"{round(data['cost_info'].get('total_cost', 'Unknown') * 100, 1)} ct",
+                "Cost per 1M units": cost_str,
+                "Cost per h": cost_per_h_str,
+                "Cost Additional Info": data["cost_info"].get("cost_additional_info", ""),
+                "Model Type": data.get("model_type", "Unknown"),
+                "Execution Info": data["execution_specifications"].get("details", "Unknown"),
+                "_accuracy": accuracy,
+                "_fpr": data.get("fpr", 0),
+                "_mean_latency": int(round(data.get("mean_latency", 0), 3) * 1000),
+                "_total_cost": round(data["cost_info"].get("total_cost", "Unknown") * 100, 1),
+            }
+        )
+
+    return pd.DataFrame(rows)
+
+
+# ── Leaderboard HTML ────────────────────────────────────────────────────
+
+def create_leaderboard_html(
+    sort_by: str = "Detection Rate (%)",
+    selected_categories: List[str] | None = None,
+    dataset_type: str = "input",
+    weights: Dict[str, float] | None = None,
+) -> str:
+    leaderboard_df = prepare_leaderboard_data(selected_categories=selected_categories, dataset_type=dataset_type)
+
+    sort_mapping = {
+        "Detection Rate (%)": "_accuracy",
+        "FPR (%)": "_fpr",
+        "Mean Latency (ms)": "_mean_latency",
+        "Total Cost": "_total_cost",
+    }
+
+    sort_column = sort_mapping.get(sort_by, "_accuracy")
+
+    if sort_by == "Overall Score":
+        sorted_df = sort_by_overall_score(leaderboard_df, weights).copy()
+    elif sort_by == "Detection Rate (%)":
+        sorted_df = leaderboard_df.sort_values(by=sort_column, ascending=False).copy()
+    elif sort_by in ["FPR (%)", "Mean Latency (ms)", "Total Cost"]:
+        sorted_df = leaderboard_df.sort_values(by=[sort_column, "_accuracy"], ascending=[True, False]).copy()
+    else:
+        sorted_df = leaderboard_df.sort_values(by=sort_column, ascending=False).copy()
+
+    sorted_df = sorted_df.reset_index(drop=True)
+
+    html = LEADERBOARD_TABLE_CSS
+    html += """
+    <div class="leaderboard-container">
+        <div class="footnotes-section" style="margin-top: 1.5rem; padding: 1rem; background: #f9fafb; border-radius: 8px; font-size: 13px; color: #374151; line-height: 1.6;">
+            <h4 style="margin-top: 0; margin-bottom: 0.75rem; font-size: 14px; font-weight: 600; color: #111827;">Column Explanations</h4>
+            <p style="margin: 0.5rem 0;"><sup>1</sup> <strong>Model Type:</strong> Shows if the supervisor is a repurposed generalist model or a specialized model made for content moderation.</p>
+            <p style="margin: 0.5rem 0;"><sup>2</sup> <strong>Detection Rate (%):</strong> Accuracy on harmful content.</p>
+            <p style="margin: 0.5rem 0;"><sup>3</sup> <strong>FPR (%):</strong> False Positive Rate on benign content.</p>
+            <p style="margin: 0.5rem 0;"><sup>4</sup> <strong>Compute Access:</strong> Shows how this supervisor was run. Can be 'Local' or 'API'.</p>
+            <p style="margin: 0.5rem 0;"><sup>5</sup> <strong>Total Cost:</strong> The total cost was measured differently for local inference and API supervisors. For local inference, it is total_latency * cost_per_h. For API supervisors it is calculated using the usage metrics and pricing information.</p>
+            <p style="margin: 0.5rem 0;"><sup>6</sup> <strong>Cost per 1M units:</strong> Different systems calculate usage in different units (e.g. characters, tokens, requests). This column shows the cost per 1M measured units.</p>
+            <p style="margin: 0.5rem 0;"><sup>7</sup> <strong>Cost per h:</strong> Only applicable for local inference. This is the pricing information for the GPU pod (see the "Execution Info" column for more details).</p>
+        </div>
+        <table class="leaderboard-table">
+            <thead>
+                <tr>
+                    <th>Rank</th>
+                    <th>Model Snapshot</th>
+                    <th>Model Developer</th>
+                    <th>Provider</th>
+                    <th title="Shows if the supervisor is a repurposed generalist model or a specialized model made for content moderation.">Model Type <sup>1</sup></th>
+                    <th title="Accuracy on harmful content.">Detection Rate (%) <sup>2</sup></th>
+                    <th title="False Positive Rate on benign content.">FPR (%) <sup>3</sup></th>
+                    <th>Latency CI 95% (ms)</th>
+                    <th>Mean Latency (ms)</th>
+                    <th title="Shows how this supervisor was ran. Can be 'Local' or 'API'.">Compute Access <sup>4</sup></th>
+                    <th title="The total cost was measured differently for local inference and API supervisors. For local inference, it is total_latency * cost_per_h. For API supervisors it is calculated using the usage metrics and pricing information.">Total Cost <sup>5</sup></th>
+                    <th title="Different systems calculate usage in different units (e.g. characters, tokens, requests). This column shows the cost per 1M measured units.">Cost per 1M units <sup>6</sup></th>
+                    <th title="Only applicable for local inference. All models were run on RunPod.">Cost per h <sup>7</sup></th>
+                    <th>Cost Additional Info</th>
+                    <th>Execution Info</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+
+    for idx, row in sorted_df.iterrows():
+        rank = idx + 1
+        detection_rate = row["Detection Rate (%)"]
+        fpr = row["FPR (%)"]
+        latency = row["Mean Latency (ms)"]
+        model_type = row["Model Type"]
+        type_class = "type-specialized" if model_type == "specialized" else "type-generalist"
+
+        model_name = escape_html(row["Model Snapshot"])
+        model_url = str(row["Model URL"]) if row["Model URL"] else ""
+        developer = escape_html(row["Model Developer"])
+        provider = escape_html(row["Provider"])
+        compute_access = escape_html(row["Compute Access"])
+        total_cost = escape_html(row["Total Cost"])
+        cost_per_1m = escape_html(row["Cost per 1M units"])
+        cost_per_h = escape_html(row["Cost per h"])
+        cost_add_info = escape_html(row["Cost Additional Info"])
+        latency_ci = escape_html(row["Latency CI 95% (ms)"])
+        exec_info = escape_html(row["Execution Info"])
+
+        if model_url and model_url != "None":
+            model_name_html = f'<a href="{model_url}" target="_blank" rel="noopener noreferrer" style="color: inherit; text-decoration: none; border-bottom: 1px dotted currentColor;">{model_name}</a>'
+        else:
+            model_name_html = model_name
+
+        html += f"""
+                <tr>
+                    <td><span class="rank-badge">{rank}</span></td>
+                    <td class="model-name">{model_name_html}</td>
+                    <td class="model-developer">{developer}</td>
+                    <td><span class="provider-badge">{provider}</span></td>
+                    <td><span class="model-type-badge {type_class}">{model_type.title()}</span></td>
+                    <td class="detection-rate metric-value">{detection_rate:.2f}%</td>
+                    <td class="metric-value">{fpr:.2f}%</td>
+                    <td>{latency_ci}</td>
+                    <td class="metric-value">{latency:d}</td>
+                    <td>{compute_access}</td>
+                    <td>{total_cost}</td>
+                    <td style="font-size: 12px;">{cost_per_1m}</td>
+                    <td>{cost_per_h}</td>
+                    <td style="font-size: 12px; color: #6b7280;">{cost_add_info}</td>
+                    <td style="font-size: 12px; color: #6b7280;">{exec_info}</td>
+                </tr>
+        """
+
+    html += """
+            </tbody>
+        </table>
+    </div>
+    """
+    html += LEADERBOARD_DARK_MODE_CSS
+    return html
+
+
+# ── Category table ──────────────────────────────────────────────────────
+
+def create_category_accuracy_table_html(selected_models: List[str] | None = None, dataset_type: str = "input") -> str:
+    metrics_data = filter_metrics_by_dataset(dataset_type)
+
+    if selected_models is None or len(selected_models) == 0:
+        selected_models = [data.get("model_name", "") for data in metrics_data.values()]
+
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    category_order = list(CATEGORY_LABELS.keys())
+    sorted_categories = [cat for cat in category_order if cat in all_categories]
+    sorted_categories.extend([cat for cat in all_categories if cat not in category_order])
+
+    html = CATEGORY_TABLE_CSS
+    html += '<div class="category-table-container"><table class="category-table">'
+
+    html += "<thead><tr><th>Model</th>"
+    for category in sorted_categories:
+        category_label = CATEGORY_LABELS.get(category, category.title())
+        html += f"<th>{category_label}</th>"
+    html += "</tr></thead>"
+
+    html += "<tbody>"
+    for key, data in metrics_data.items():
+        model_name = data.get("model_name", "")
+        if model_name not in selected_models:
+            continue
+
+        html += f"<tr><td>{escape_html(model_name)}</td>"
+        accuracy_per_category = data.get("accuracy_per_category", {})
+        for category in sorted_categories:
+            accuracy = accuracy_per_category.get(category, 0)
+            acc_pct = round(accuracy * 100, 1)
+            bg_color = get_color_for_accuracy(accuracy)
+            html += f'<td style="background-color: {bg_color};">{acc_pct}%</td>'
+        html += "</tr>"
+
+    html += "</tbody></table></div>"
+    return html
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────
+
+def get_available_categories(dataset_type: str = "input") -> List[tuple]:
+    metrics_data = filter_metrics_by_dataset(dataset_type)
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    category_order = list(CATEGORY_LABELS.keys())
+    sorted_categories = [cat for cat in category_order if cat in all_categories]
+    sorted_categories.extend([cat for cat in all_categories if cat not in category_order])
+    return [(CATEGORY_LABELS.get(cat, cat.title()), cat) for cat in sorted_categories]
+
+
+def get_available_models(dataset_type: str = "input") -> List[str]:
+    metrics_data = filter_metrics_by_dataset(dataset_type)
+    return sorted([data.get("model_name", "") for data in metrics_data.values()])
+
+
+# ── Pareto ──────────────────────────────────────────────────────────────
+
+_PARETO_METRIC_MAP = {
+    "Detection Rate in %": ("_accuracy", False),
+    "FPR in %": ("_fpr", True),
+    "Mean Latency in ms": ("_mean_latency", True),
+    "Total Cost in ct": ("_total_cost", True),
+}
+
+
+def create_pareto_plot_interactive(x_metric, y_metric, dataset_type="input"):
+    df = prepare_leaderboard_data(None, dataset_type)
+    return build_pareto_figure(df, x_metric, y_metric, _PARETO_METRIC_MAP)
+
+
+# ── Tab builder ─────────────────────────────────────────────────────────
+
+def build_cm_tab(dataset_type: str):
+    """Build the three sub-tabs (Leaderboard, Category Performance, Pareto Frontier)
+    for a given content-moderation dataset type ('input' or 'output')."""
+    label = "Input" if dataset_type == "input" else "Output"
+
+    with gr.Tabs():
+        # ── Leaderboard sub-tab ─────────────────────────────────────
+        with gr.Tab("Leaderboard"):
+            with gr.Column():
+                gr.Markdown(f"### Interactive Leaderboard - {label} Dataset")
+                gr.Markdown(
+                    "Sort the leaderboard by different metrics to compare model performance. "
+                    "Use the dropdown below to change the sorting order. "
+                    "Select specific categories to see detection rates calculated only for those categories."
+                )
+
+                category_choices = get_available_categories(dataset_type)
+
+                with gr.Row():
+                    sort_metric = gr.Dropdown(
+                        choices=[
+                            "Overall Score",
+                            "Detection Rate (%)",
+                            "FPR (%)",
+                            "Mean Latency (ms)",
+                            "Total Cost",
+                        ],
+                        value="Overall Score",
+                        label="Sort by Metric",
+                        interactive=True,
+                        info="Higher detection rate is better. Lower FPR and latency are better.",
+                    )
+
+                with gr.Group(visible=True) as weight_row:
+                    with gr.Row():
+                        weight_detection = gr.Number(
+                            value=DEFAULT_WEIGHTS["Detection Rate (%)"],
+                            label="Detection Rate weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_fpr = gr.Number(
+                            value=DEFAULT_WEIGHTS["FPR (%)"],
+                            label="FPR weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_latency = gr.Number(
+                            value=DEFAULT_WEIGHTS["Mean Latency (ms)"],
+                            label="Latency weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_cost = gr.Number(
+                            value=DEFAULT_WEIGHTS["Total Cost"],
+                            label="Cost weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                    weight_bar = gr.HTML(value=create_weight_bar_html(DEFAULT_WEIGHTS))
+                weight_inputs = [weight_detection, weight_fpr, weight_latency, weight_cost]
+
+                gr.HTML(value=create_dataset_info_html(dataset_type))
+
+                with gr.Row():
+                    category_selector = gr.CheckboxGroup(
+                        choices=category_choices,
+                        value=[cat for _, cat in category_choices],
+                        label="Select Categories for Detection Rate",
+                        interactive=True,
+                        info="Select categories to calculate detection rate. By default, all categories are selected (overall detection rate).",
+                    )
+
+                leaderboard_html = gr.HTML(
+                    value=create_leaderboard_html(sort_by="Overall Score", dataset_type=dataset_type, weights=DEFAULT_WEIGHTS),
+                    label="Model Rankings",
+                )
+
+                def update_leaderboard(sort_by, selected_categories, w_det, w_fpr, w_lat, w_cost):
+                    weights = build_weights(w_det, w_fpr, w_lat, w_cost)
+                    html = create_leaderboard_html(
+                        sort_by=sort_by,
+                        selected_categories=selected_categories,
+                        dataset_type=dataset_type,
+                        weights=weights,
+                    )
+                    bar = create_weight_bar_html(weights)
+                    return html, bar
+
+                def on_sort_change(sort_by, selected_categories, w_det, w_fpr, w_lat, w_cost):
+                    weights = build_weights(w_det, w_fpr, w_lat, w_cost)
+                    new_html = create_leaderboard_html(
+                        sort_by=sort_by,
+                        selected_categories=selected_categories,
+                        dataset_type=dataset_type,
+                        weights=weights,
+                    )
+                    visible = sort_by == "Overall Score"
+                    bar = create_weight_bar_html(weights)
+                    return gr.update(visible=visible), new_html, bar
+
+                all_inputs = [sort_metric, category_selector] + weight_inputs
+
+                sort_metric.change(
+                    fn=on_sort_change,
+                    inputs=all_inputs,
+                    outputs=[weight_row, leaderboard_html, weight_bar],
+                )
+
+                category_selector.change(
+                    fn=update_leaderboard,
+                    inputs=all_inputs,
+                    outputs=[leaderboard_html, weight_bar],
+                )
+
+                for w_input in weight_inputs:
+                    w_input.change(
+                        fn=update_leaderboard,
+                        inputs=all_inputs,
+                        outputs=[leaderboard_html, weight_bar],
+                    )
+
+        # ── Category Performance sub-tab ────────────────────────────
+        with gr.Tab("Category Performance"):
+            with gr.Column():
+                gr.Markdown(f"### Accuracy per Category by Model - {label} Dataset")
+                gr.Markdown(
+                    "Compare how different models perform across the 11 harm categories. "
+                    "Each cell shows the detection rate (accuracy) for that model-category combination. "
+                    "Cell colors indicate performance: darker green = higher accuracy (closer to 100%), lighter colors = lower accuracy."
+                )
+
+                gr.HTML(value=create_dataset_info_html(dataset_type))
+
+                model_choices = get_available_models(dataset_type)
+                model_selector = gr.CheckboxGroup(
+                    choices=model_choices,
+                    value=model_choices,
+                    label="Select Models to Compare",
+                    interactive=True,
+                    info="Select one or more models to compare their performance across categories.",
+                )
+
+                category_table_html = gr.HTML(
+                    value=create_category_accuracy_table_html(model_choices, dataset_type=dataset_type),
+                    label="Category Accuracy Comparison",
+                )
+
+                model_selector.change(
+                    fn=lambda models: create_category_accuracy_table_html(models, dataset_type=dataset_type),
+                    inputs=model_selector,
+                    outputs=category_table_html,
+                )
+
+        # ── Pareto Frontier sub-tab ─────────────────────────────────
+        with gr.Tab("Pareto Frontier"):
+            with gr.Column():
+                gr.Markdown(f"### Pareto Frontier Analysis - {label} Dataset")
+                gr.Markdown(
+                    "Visualize trade-offs between different metrics. Models on the Pareto frontier "
+                    "represent optimal trade-offs where improving one metric would require sacrificing another. "
+                    "Hover over points to see the model name."
+                )
+
+                with gr.Row():
+                    pareto_x_metric = gr.Dropdown(
+                        choices=["Detection Rate in %", "FPR in %", "Mean Latency in ms", "Total Cost in ct"],
+                        value="Detection Rate in %",
+                        label="X-Axis Metric",
+                        interactive=True,
+                    )
+                    pareto_y_metric = gr.Dropdown(
+                        choices=["Detection Rate in %", "FPR in %", "Mean Latency in ms", "Total Cost in ct"],
+                        value="FPR in %",
+                        label="Y-Axis Metric",
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    gr.HTML('<div style="display:flex; justify-content:center;">', visible=False)
+                    pareto_plot = gr.Plot(
+                        value=create_pareto_plot_interactive("Detection Rate in %", "FPR in %", dataset_type)
+                    )
+                    gr.HTML("</div>", visible=False)
+
+                def _update_pareto(x_metric, y_metric):
+                    if x_metric == y_metric:
+                        return create_empty_pareto_figure()
+                    return create_pareto_plot_interactive(x_metric, y_metric, dataset_type)
+
+                pareto_x_metric.change(
+                    fn=_update_pareto,
+                    inputs=[pareto_x_metric, pareto_y_metric],
+                    outputs=pareto_plot,
+                )
+                pareto_y_metric.change(
+                    fn=_update_pareto,
+                    inputs=[pareto_x_metric, pareto_y_metric],
+                    outputs=pareto_plot,
+                )

data/cm_model_info_mapping.json

@@ -0,0 +1,679 @@
+{
+  "_comment": "Consolidated model information mapping. Contains developer, type, cost information, and execution specifications for each model.",
+  "_units": {
+    "api_costs": "cost per 1M input tokens + cost per 1M output tokens (if applicable)",
+    "local_costs": "cost per 1M tokens (calculated using formula)"
+  },
+  "anthropic/claude-haiku-4-5": {
+    "model_developer": "Anthropic",
+    "model_type": "generalist",
+    "url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.0,
+      "cost_per_1M_output_tokens": 5.0,
+      "source": "Claude API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "anthropic/claude-sonnet-4-5": {
+    "model_developer": "Anthropic",
+    "model_type": "generalist",
+    "url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 3.0,
+      "cost_per_1M_output_tokens": 15.0,
+      "source": "Claude API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "azure/analyze-text": {
+    "model_developer": "Azure",
+    "model_type": "specialized",
+    "url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 380.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Azure AI Content Safety",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "google/gemini-2.5-flash": {
+    "model_developer": "Google",
+    "model_type": "generalist",
+    "url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.3,
+      "cost_per_1M_output_tokens": 2.5,
+      "source": "Gemini Developer API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "lakera/lakera-guard_default": {
+    "model_developer": "Lakera",
+    "model_type": "specialized",
+    "url": "https://www.lakera.ai/lakera-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Lakera API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "meta/llama-guard-4-12b": {
+    "model_developer": "Meta",
+    "model_type": "specialized",
+    "url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Together AI pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "mistral/ministral-3b-2512": {
+    "model_developer": "Mistral AI",
+    "model_type": "generalist",
+    "url": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.04,
+      "cost_per_1M_output_tokens": 0.04,
+      "source": "Ministral API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "mistral/mistral-large-3": {
+    "model_developer": "Mistral AI",
+    "model_type": "generalist",
+    "url": "https://docs.mistral.ai/models/mistral-large-3-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 4.0,
+      "cost_per_1M_output_tokens": 12.0,
+      "source": "Mistral API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-5-nano": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://platform.openai.com/docs/models/gpt-5-nano",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.4,
+      "source": "OpenAI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-5.2": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://platform.openai.com/docs/models/gpt-5.2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.75,
+      "cost_per_1M_output_tokens": 14.0,
+      "source": "OpenAI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-20b": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://www.together.ai/models/gpt-oss-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-120b": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://www.together.ai/models/gpt-oss-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-safeguard-20b": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.07,
+      "cost_per_1M_output_tokens": 0.3,
+      "source": "OpenRouter API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "openai/omni-moderation": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://platform.openai.com/docs/models/omni-moderation-latest",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "OpenAI Moderation API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "virtue-ai/virtueguard-text-lite": {
+    "model_developer": "Virtue AI",
+    "model_type": "specialized",
+    "url": "https://www.together.ai/models/virtueguard-text-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "x-ai/grok-4-1-fast-non-reasoning": {
+    "model_developer": "X-AI",
+    "model_type": "generalist",
+    "url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.5,
+      "source": "X-AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "aws/bedrock-guardrail": {
+    "model_developer": "Amazon Web Services",
+    "model_type": "specialized",
+    "url": "https://aws.amazon.com/bedrock/guardrails/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 150.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Amazon Bedrock API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    }
+  },
+  "nvidia/aegis-ai-content-safety-llamaguard-defensive-1.0": {
+    "model_developer": "NVIDIA",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-2b": {
+    "model_developer": "Google",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/google/shieldgemma-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-9b": {
+    "model_developer": "Google",
+    "url": "https://huggingface.co/google/shieldgemma-9b",
+    "model_type": "specialized",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-27b": {
+    "model_developer": "Google",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/google/shieldgemma-27b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "saillab/xguard": {
+    "model_developer": "SAIL Lab",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/saillab/x-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "openai/gpt-oss-safeguard-120b": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/openai/gpt-oss-safeguard-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 3.07,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-8b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-4b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-4B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-0.6b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": {
+    "model_developer": "NVIDIA",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "rakancorle1/thinkguard": {
+    "model_developer": "RakanCorle1",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Rakancorle1/ThinkGuard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "allenai/wildguard": {
+    "model_developer": "AllenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/allenai/wildguard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-ministral": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-qwen": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-qwen-smol": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen-Smol",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.0-2b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.0-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.1-2b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.1-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.1-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.1-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.2-5b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-5b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.2-3b-a800m": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-3b-a800m",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.3-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "govtech/lionguard-2": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "govtech/lionguard-2.1": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2.1",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "govtech/lionguard-2-lite": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  }
+}

data/content_moderation_metrics.json

@@ -0,0 +1,2702 @@
+{
+  "allenai/bells-o-project-content-moderation-input/wildguard": {
+    "model_provider_use_case": "allenai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "wildguard",
+    "accuracy": 0.9209090909090909,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.92,
+      "privacy": 0.93,
+      "hate speech": 0.83,
+      "sexual content": 0.98,
+      "cbrn": 0.93,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.91,
+      "illegal activities": 0.94,
+      "self-harm": 0.81,
+      "physical harm": 0.94,
+      "cyber": 0.96
+    },
+    "fpr": 0.01,
+    "mean_latency": 0.16794882569994246,
+    "latency_ci_95": {
+      "lower": 0.1672896153698199,
+      "upper": 0.16860803603006502,
+      "mean": 0.16794882569994246,
+      "std_dev": 0.012584383679289867,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "AllenAI",
+    "model_url": "https://huggingface.co/allenai/wildguard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.7238035981627982,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.15609910299777988
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "allenai/bells-o-project-content-moderation-output/wildguard": {
+    "model_provider_use_case": "allenai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "wildguard",
+    "accuracy": 0.9,
+    "accuracy_per_category": {
+      "hate speech": 0.82,
+      "physical harm": 0.97,
+      "harmful manipulation": 0.83,
+      "self-harm": 0.85,
+      "privacy": 0.99,
+      "cbrn": 0.92,
+      "harm to minors": 0.98,
+      "cyber": 0.9,
+      "sexual content": 0.94,
+      "illegal activities": 0.95,
+      "integrity & quality violations": 0.75
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.17618908030646188,
+    "latency_ci_95": {
+      "lower": 0.17568036971954096,
+      "upper": 0.1766977908933828,
+      "mean": 0.17618908030646188,
+      "std_dev": 0.009711330231034699,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "AllenAI",
+    "model_url": "https://huggingface.co/allenai/wildguard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.25754181310468993,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.1637579618626171
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "anthropic/bells-o-project-content-moderation-input/claude-haiku-4-5": {
+    "model_provider_use_case": "anthropic",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "claude-haiku-4-5",
+    "accuracy": 0.64,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.48,
+      "privacy": 0.49,
+      "hate speech": 0.6,
+      "sexual content": 0.74,
+      "cbrn": 0.74,
+      "harm to minors": 0.94,
+      "integrity & quality violations": 0.42,
+      "illegal activities": 0.77,
+      "self-harm": 0.38,
+      "physical harm": 0.67,
+      "cyber": 0.81
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.6755487293856485,
+    "latency_ci_95": {
+      "lower": 0.6616969457280298,
+      "upper": 0.6894005130432671,
+      "mean": 0.6755487293856485,
+      "std_dev": 0.26443177878838336,
+      "n": 1400
+    },
+    "provider": "Anthropic",
+    "model_type": "generalist",
+    "model_developer": "Anthropic",
+    "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.0,
+      "cost_per_1M_output_tokens": 5.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Claude API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.145574
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "anthropic/bells-o-project-content-moderation-input/claude-sonnet-4-5": {
+    "model_provider_use_case": "anthropic",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "claude-sonnet-4-5",
+    "accuracy": 0.7909090909090909,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.73,
+      "privacy": 0.68,
+      "hate speech": 0.65,
+      "sexual content": 0.81,
+      "cbrn": 0.94,
+      "harm to minors": 0.95,
+      "integrity & quality violations": 0.7,
+      "illegal activities": 0.87,
+      "self-harm": 0.59,
+      "physical harm": 0.83,
+      "cyber": 0.95
+    },
+    "fpr": 0.01,
+    "mean_latency": 2.0321146711281366,
+    "latency_ci_95": {
+      "lower": 1.9980452484611515,
+      "upper": 2.0661840937951217,
+      "mean": 2.0321146711281366,
+      "std_dev": 0.6503883009441236,
+      "n": 1400
+    },
+    "provider": "Anthropic",
+    "model_type": "generalist",
+    "model_developer": "Anthropic",
+    "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 3.0,
+      "cost_per_1M_output_tokens": 15.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Claude API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.443607
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "anthropic/bells-o-project-content-moderation-output/claude-haiku-4-5": {
+    "model_provider_use_case": "anthropic",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "claude-haiku-4-5",
+    "accuracy": 0.9118181818181819,
+    "accuracy_per_category": {
+      "hate speech": 0.78,
+      "physical harm": 0.97,
+      "harmful manipulation": 0.88,
+      "self-harm": 0.88,
+      "privacy": 0.91,
+      "cbrn": 1.0,
+      "harm to minors": 1.0,
+      "cyber": 0.97,
+      "sexual content": 0.89,
+      "illegal activities": 0.96,
+      "integrity & quality violations": 0.79
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.660408535855157,
+    "latency_ci_95": {
+      "lower": 0.6482668563627765,
+      "upper": 0.6725502153475376,
+      "mean": 0.660408535855157,
+      "std_dev": 0.23178573857401796,
+      "n": 1400
+    },
+    "provider": "Anthropic",
+    "model_type": "generalist",
+    "model_developer": "Anthropic",
+    "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.0,
+      "cost_per_1M_output_tokens": 5.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Claude API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.545895
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "anthropic/bells-o-project-content-moderation-output/claude-sonnet-4-5": {
+    "model_provider_use_case": "anthropic",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "claude-sonnet-4-5",
+    "accuracy": 0.8690909090909091,
+    "accuracy_per_category": {
+      "hate speech": 0.65,
+      "physical harm": 0.94,
+      "harmful manipulation": 0.87,
+      "self-harm": 0.76,
+      "privacy": 0.86,
+      "cbrn": 0.96,
+      "harm to minors": 0.99,
+      "cyber": 0.97,
+      "sexual content": 0.86,
+      "illegal activities": 0.95,
+      "integrity & quality violations": 0.75
+    },
+    "fpr": 0.0,
+    "mean_latency": 1.980507916041783,
+    "latency_ci_95": {
+      "lower": 1.9466451605985857,
+      "upper": 2.01437067148498,
+      "mean": 1.980507916041783,
+      "std_dev": 0.6464430053089691,
+      "n": 1400
+    },
+    "provider": "Anthropic",
+    "model_type": "generalist",
+    "model_developer": "Anthropic",
+    "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 3.0,
+      "cost_per_1M_output_tokens": 15.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Claude API pricing",
+      "cost_additional_info": "",
+      "total_cost": 1.6417950000000001
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "aws/bells-o-project-content-moderation-input/bedrock-guardrail": {
+    "model_provider_use_case": "aws",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "bedrock-guardrail",
+    "accuracy": 0.7318181818181818,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.44,
+      "privacy": 0.53,
+      "hate speech": 0.8,
+      "sexual content": 0.93,
+      "cbrn": 0.67,
+      "harm to minors": 0.93,
+      "integrity & quality violations": 0.49,
+      "illegal activities": 0.82,
+      "self-harm": 0.61,
+      "physical harm": 0.91,
+      "cyber": 0.92
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.3027071602003915,
+    "latency_ci_95": {
+      "lower": 0.2893886246223259,
+      "upper": 0.3160256957784571,
+      "mean": 0.3027071602003915,
+      "std_dev": 0.2542520256463307,
+      "n": 1400
+    },
+    "provider": "AWS",
+    "model_type": "specialized",
+    "model_developer": "Amazon Web Services",
+    "model_url": "https://aws.amazon.com/bedrock/guardrails/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 150.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Amazon Bedrock API pricing",
+      "cost_additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$.",
+      "total_cost": 0.2157
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "aws/bells-o-project-content-moderation-output/bedrock-guardrail": {
+    "model_provider_use_case": "aws",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "bedrock-guardrail",
+    "accuracy": 0.78,
+    "accuracy_per_category": {
+      "hate speech": 0.71,
+      "physical harm": 0.96,
+      "harmful manipulation": 0.49,
+      "self-harm": 0.7,
+      "privacy": 0.73,
+      "cbrn": 0.83,
+      "harm to minors": 0.93,
+      "cyber": 0.89,
+      "sexual content": 0.94,
+      "illegal activities": 0.91,
+      "integrity & quality violations": 0.49
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.3212733970369612,
+    "latency_ci_95": {
+      "lower": 0.3112962016769463,
+      "upper": 0.331250592396976,
+      "mean": 0.3212733970369612,
+      "std_dev": 0.1904655444800315,
+      "n": 1400
+    },
+    "provider": "AWS",
+    "model_type": "specialized",
+    "model_developer": "Amazon Web Services",
+    "model_url": "https://aws.amazon.com/bedrock/guardrails/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 150.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Amazon Bedrock API pricing",
+      "cost_additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$.",
+      "total_cost": 0.3981
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "azure/bells-o-project-content-moderation-input/analyze-text": {
+    "model_provider_use_case": "azure",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "analyze-text",
+    "accuracy": 0.5009090909090909,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.24,
+      "privacy": 0.1,
+      "hate speech": 0.87,
+      "sexual content": 0.9,
+      "cbrn": 0.51,
+      "harm to minors": 0.8,
+      "integrity & quality violations": 0.13,
+      "illegal activities": 0.28,
+      "self-harm": 0.89,
+      "physical harm": 0.72,
+      "cyber": 0.07
+    },
+    "fpr": 0.01,
+    "mean_latency": 0.6510780201639448,
+    "latency_ci_95": {
+      "lower": 0.6438075360683082,
+      "upper": 0.6583485042595814,
+      "mean": 0.6510780201639448,
+      "std_dev": 0.13879418633602486,
+      "n": 1400
+    },
+    "provider": "Azure",
+    "model_type": "specialized",
+    "model_developer": "Azure",
+    "model_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 380.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Azure AI Content Safety",
+      "cost_additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan.",
+      "total_cost": 0.54644
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "azure/bells-o-project-content-moderation-output/analyze-text": {
+    "model_provider_use_case": "azure",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "analyze-text",
+    "accuracy": 0.6090909090909091,
+    "accuracy_per_category": {
+      "hate speech": 0.92,
+      "physical harm": 0.91,
+      "harmful manipulation": 0.38,
+      "self-harm": 0.86,
+      "privacy": 0.26,
+      "cbrn": 0.8,
+      "harm to minors": 0.71,
+      "cyber": 0.16,
+      "sexual content": 0.89,
+      "illegal activities": 0.47,
+      "integrity & quality violations": 0.34
+    },
+    "fpr": 0.013333333333333334,
+    "mean_latency": 0.597562814269747,
+    "latency_ci_95": {
+      "lower": 0.5849396956294183,
+      "upper": 0.6101859329100757,
+      "mean": 0.597562814269747,
+      "std_dev": 0.24097645461586636,
+      "n": 1400
+    },
+    "provider": "Azure",
+    "model_type": "specialized",
+    "model_developer": "Azure",
+    "model_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 380.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Azure AI Content Safety",
+      "cost_additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan.",
+      "total_cost": 1.00852
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-input/gemini-2.5-flash": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gemini-2.5-flash",
+    "accuracy": 0.8818181818181818,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.95,
+      "privacy": 0.98,
+      "hate speech": 0.86,
+      "sexual content": 0.78,
+      "cbrn": 0.99,
+      "harm to minors": 0.56,
+      "integrity & quality violations": 0.88,
+      "illegal activities": 0.98,
+      "self-harm": 0.78,
+      "physical harm": 0.95,
+      "cyber": 0.99
+    },
+    "fpr": 0.016666666666666666,
+    "mean_latency": 1.4193411101613727,
+    "latency_ci_95": {
+      "lower": 1.3612213357967249,
+      "upper": 1.4774608845260204,
+      "mean": 1.4193411101613727,
+      "std_dev": 1.1095116483118301,
+      "n": 1400
+    },
+    "provider": "GoogleAIStudio",
+    "model_type": "generalist",
+    "model_developer": "Google",
+    "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.3,
+      "cost_per_1M_output_tokens": 2.5,
+      "cost_per_h": "N/A",
+      "cost_source": "Gemini Developer API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.5480927
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-input/shieldgemma-27b": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "shieldgemma-27b",
+    "accuracy": 0.5809090909090909,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.32,
+      "privacy": 0.09,
+      "hate speech": 0.73,
+      "sexual content": 0.84,
+      "cbrn": 0.91,
+      "harm to minors": 0.8,
+      "integrity & quality violations": 0.18,
+      "illegal activities": 0.64,
+      "self-harm": 0.72,
+      "physical harm": 0.87,
+      "cyber": 0.29
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.1099088318007333,
+    "latency_ci_95": {
+      "lower": 0.1090000827959994,
+      "upper": 0.11081758080546719,
+      "mean": 0.1099088318007333,
+      "std_dev": 0.0173480991137042,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Google",
+    "model_url": "https://huggingface.co/google/shieldgemma-27b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.19256129687083262,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.10215415311257045
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-input/shieldgemma-2b": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "shieldgemma-2b",
+    "accuracy": 0.24818181818181817,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.08,
+      "privacy": 0.02,
+      "hate speech": 0.1,
+      "sexual content": 0.21,
+      "cbrn": 0.48,
+      "harm to minors": 0.34,
+      "integrity & quality violations": 0.05,
+      "illegal activities": 0.38,
+      "self-harm": 0.39,
+      "physical harm": 0.53,
+      "cyber": 0.15
+    },
+    "fpr": 0.01,
+    "mean_latency": 0.031726044927324566,
+    "latency_ci_95": {
+      "lower": 0.031453915259710516,
+      "upper": 0.031998174594938616,
+      "mean": 0.031726044927324566,
+      "std_dev": 0.005194979494839089,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Google",
+    "model_url": "https://huggingface.co/google/shieldgemma-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05558432617011184,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.02948759620189667
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-output/gemini-2.5-flash": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gemini-2.5-flash",
+    "accuracy": 0.8909090909090909,
+    "accuracy_per_category": {
+      "hate speech": 0.88,
+      "physical harm": 0.98,
+      "harmful manipulation": 0.91,
+      "self-harm": 0.92,
+      "privacy": 0.97,
+      "cbrn": 0.97,
+      "harm to minors": 0.6,
+      "cyber": 0.95,
+      "sexual content": 0.83,
+      "illegal activities": 0.91,
+      "integrity & quality violations": 0.88
+    },
+    "fpr": 0.0,
+    "mean_latency": 2.131153161866324,
+    "latency_ci_95": {
+      "lower": 2.0531708906747133,
+      "upper": 2.209135433057935,
+      "mean": 2.131153161866324,
+      "std_dev": 1.4886884747015303,
+      "n": 1400
+    },
+    "provider": "GoogleAIStudio",
+    "model_type": "generalist",
+    "model_developer": "Google",
+    "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.3,
+      "cost_per_1M_output_tokens": 2.5,
+      "cost_per_h": "N/A",
+      "cost_source": "Gemini Developer API pricing",
+      "cost_additional_info": "",
+      "total_cost": 1.1270327
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-output/shieldgemma-27b": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "shieldgemma-27b",
+    "accuracy": 0.6336363636363637,
+    "accuracy_per_category": {
+      "hate speech": 0.73,
+      "physical harm": 0.91,
+      "harmful manipulation": 0.42,
+      "self-harm": 0.74,
+      "privacy": 0.32,
+      "cbrn": 0.96,
+      "harm to minors": 0.75,
+      "cyber": 0.33,
+      "sexual content": 0.82,
+      "illegal activities": 0.7,
+      "integrity & quality violations": 0.29
+    },
+    "fpr": 0.006666666666666667,
+    "mean_latency": 0.13334267480032785,
+    "latency_ci_95": {
+      "lower": 0.13205096274443018,
+      "upper": 0.13463438685622553,
+      "mean": 0.13334267480032785,
+      "std_dev": 0.024658897732318774,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Google",
+    "model_url": "https://huggingface.co/google/shieldgemma-27b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.1388315066243309,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.12393460830052694
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "google/bells-o-project-content-moderation-output/shieldgemma-2b": {
+    "model_provider_use_case": "google",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "shieldgemma-2b",
+    "accuracy": 0.26636363636363636,
+    "accuracy_per_category": {
+      "hate speech": 0.17,
+      "physical harm": 0.58,
+      "harmful manipulation": 0.14,
+      "self-harm": 0.44,
+      "privacy": 0.08,
+      "cbrn": 0.38,
+      "harm to minors": 0.34,
+      "cyber": 0.14,
+      "sexual content": 0.22,
+      "illegal activities": 0.39,
+      "integrity & quality violations": 0.05
+    },
+    "fpr": 0.006666666666666667,
+    "mean_latency": 0.037216948441096714,
+    "latency_ci_95": {
+      "lower": 0.036785634085487344,
+      "upper": 0.037648262796706085,
+      "mean": 0.037216948441096714,
+      "std_dev": 0.008233829309629823,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Google",
+    "model_url": "https://huggingface.co/google/shieldgemma-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.03874892289189925,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.03459108596775267
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "govtech/bells-o-project-content-moderation-input/lionguard-2": {
+    "model_provider_use_case": "govtech",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "lionguard-2",
+    "accuracy": 0.8845454545454545,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.83,
+      "privacy": 0.67,
+      "hate speech": 0.83,
+      "sexual content": 0.98,
+      "cbrn": 0.91,
+      "harm to minors": 0.97,
+      "integrity & quality violations": 0.79,
+      "illegal activities": 0.96,
+      "self-harm": 0.93,
+      "physical harm": 0.92,
+      "cyber": 0.94
+    },
+    "fpr": 0.01,
+    "mean_latency": 0.009503273112433297,
+    "latency_ci_95": {
+      "lower": 0.009311582881572434,
+      "upper": 0.00969496334329416,
+      "mean": 0.009503273112433297,
+      "std_dev": 0.0036593835115966812,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "GovTech",
+    "model_url": "https://huggingface.co/govtech/lionguard-2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.1812443960764434,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.008832764398389393
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    },
+    "num_samples": 1400
+  },
+  "govtech/bells-o-project-content-moderation-output/lionguard-2": {
+    "model_provider_use_case": "govtech",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "lionguard-2",
+    "accuracy": 0.9036363636363637,
+    "accuracy_per_category": {
+      "hate speech": 0.79,
+      "physical harm": 0.98,
+      "harmful manipulation": 0.77,
+      "self-harm": 0.94,
+      "privacy": 0.85,
+      "cbrn": 0.95,
+      "harm to minors": 0.96,
+      "cyber": 0.93,
+      "sexual content": 0.99,
+      "illegal activities": 0.97,
+      "integrity & quality violations": 0.81
+    },
+    "fpr": 0.01,
+    "mean_latency": 0.009445810999189104,
+    "latency_ci_95": {
+      "lower": 0.009313593155526146,
+      "upper": 0.009578028842852061,
+      "mean": 0.009445810999189104,
+      "std_dev": 0.0025240503643103483,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "GovTech",
+    "model_url": "https://huggingface.co/govtech/lionguard-2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.021758170584833442,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.00877935655646854
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    },
+    "num_samples": 1400
+  },
+  "ibm-granite/bells-o-project-content-moderation-input/granite-guardian-3.3-8b": {
+    "model_provider_use_case": "ibm-granite",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "granite-guardian-3.3-8b",
+    "accuracy": 0.9272727272727272,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.94,
+      "privacy": 0.92,
+      "hate speech": 0.84,
+      "sexual content": 0.97,
+      "cbrn": 1.0,
+      "harm to minors": 0.99,
+      "integrity & quality violations": 0.8,
+      "illegal activities": 0.98,
+      "self-harm": 0.8,
+      "physical harm": 0.98,
+      "cyber": 0.98
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.19252273099763054,
+    "latency_ci_95": {
+      "lower": 0.19230705557305205,
+      "upper": 0.19273840642220902,
+      "mean": 0.19252273099763054,
+      "std_dev": 0.004117262987345297,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "IBM",
+    "model_url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.8130975128709368,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.17893918275501994
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "ibm-granite/bells-o-project-content-moderation-output/granite-guardian-3.3-8b": {
+    "model_provider_use_case": "ibm-granite",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "granite-guardian-3.3-8b",
+    "accuracy": 0.9054545454545454,
+    "accuracy_per_category": {
+      "hate speech": 0.86,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.78,
+      "self-harm": 0.85,
+      "privacy": 0.91,
+      "cbrn": 0.99,
+      "harm to minors": 0.99,
+      "cyber": 0.94,
+      "sexual content": 0.98,
+      "illegal activities": 0.93,
+      "integrity & quality violations": 0.74
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.20689634186880929,
+    "latency_ci_95": {
+      "lower": 0.2061988732441346,
+      "upper": 0.20759381049348397,
+      "mean": 0.20689634186880929,
+      "std_dev": 0.013314737916107277,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "IBM",
+    "model_url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2968973859968493,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.1922986555258433
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "lakera/bells-o-project-content-moderation-input/lakera-guard_default": {
+    "model_provider_use_case": "lakera",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "lakera-guard_default",
+    "accuracy": 0.9081818181818182,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.9,
+      "privacy": 0.84,
+      "hate speech": 0.87,
+      "sexual content": 1.0,
+      "cbrn": 0.97,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.78,
+      "illegal activities": 0.95,
+      "self-harm": 0.79,
+      "physical harm": 0.96,
+      "cyber": 0.95
+    },
+    "fpr": 0.16,
+    "mean_latency": 0.20956397243908473,
+    "latency_ci_95": {
+      "lower": 0.20730703245330986,
+      "upper": 0.2118209124248596,
+      "mean": 0.20956397243908473,
+      "std_dev": 0.04308518453714262,
+      "n": 1400
+    },
+    "provider": "Lakera",
+    "model_type": "specialized",
+    "model_developer": "Lakera",
+    "model_url": "https://www.lakera.ai/lakera-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Lakera API pricing",
+      "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.",
+      "total_cost": 0.0
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "lakera/bells-o-project-content-moderation-output/lakera-guard_default": {
+    "model_provider_use_case": "lakera",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "lakera-guard_default",
+    "accuracy": 0.860909090909091,
+    "accuracy_per_category": {
+      "hate speech": 0.77,
+      "physical harm": 0.96,
+      "harmful manipulation": 0.76,
+      "self-harm": 0.84,
+      "privacy": 0.88,
+      "cbrn": 0.89,
+      "harm to minors": 0.91,
+      "cyber": 0.87,
+      "sexual content": 0.96,
+      "illegal activities": 0.94,
+      "integrity & quality violations": 0.69
+    },
+    "fpr": 0.14333333333333334,
+    "mean_latency": 0.20940651042120798,
+    "latency_ci_95": {
+      "lower": 0.20735007341754588,
+      "upper": 0.21146294742487007,
+      "mean": 0.20940651042120798,
+      "std_dev": 0.03925756482238554,
+      "n": 1400
+    },
+    "provider": "Lakera",
+    "model_type": "specialized",
+    "model_developer": "Lakera",
+    "model_url": "https://www.lakera.ai/lakera-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Lakera API pricing",
+      "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.",
+      "total_cost": 0.0
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "meta/bells-o-project-content-moderation-input/llama-guard-4-12b": {
+    "model_provider_use_case": "meta",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "llama-guard-4-12b",
+    "accuracy": 0.7054545454545454,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.49,
+      "privacy": 0.69,
+      "hate speech": 0.42,
+      "sexual content": 0.79,
+      "cbrn": 0.78,
+      "harm to minors": 0.87,
+      "integrity & quality violations": 0.47,
+      "illegal activities": 0.84,
+      "self-harm": 0.69,
+      "physical harm": 0.86,
+      "cyber": 0.86
+    },
+    "fpr": 0.03,
+    "mean_latency": 0.2919416810785021,
+    "latency_ci_95": {
+      "lower": 0.2889681782255528,
+      "upper": 0.2949151839314514,
+      "mean": 0.2919416810785021,
+      "std_dev": 0.05676443323638302,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "specialized",
+    "model_developer": "Meta",
+    "model_url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.0671434
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "meta/bells-o-project-content-moderation-output/llama-guard-4-12b": {
+    "model_provider_use_case": "meta",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "llama-guard-4-12b",
+    "accuracy": 0.8036363636363636,
+    "accuracy_per_category": {
+      "hate speech": 0.62,
+      "physical harm": 0.94,
+      "harmful manipulation": 0.54,
+      "self-harm": 0.88,
+      "privacy": 0.85,
+      "cbrn": 0.94,
+      "harm to minors": 0.95,
+      "cyber": 0.89,
+      "sexual content": 0.89,
+      "illegal activities": 0.9,
+      "integrity & quality violations": 0.44
+    },
+    "fpr": 0.04,
+    "mean_latency": 0.3172705331870488,
+    "latency_ci_95": {
+      "lower": 0.29924274832923003,
+      "upper": 0.33529831804486754,
+      "mean": 0.3172705331870488,
+      "std_dev": 0.3441520121450457,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "specialized",
+    "model_developer": "Meta",
+    "model_url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.1373716
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "mistral/bells-o-project-content-moderation-input/ministral-3b-2512": {
+    "model_provider_use_case": "mistral",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "ministral-3b-2512",
+    "accuracy": 0.9345454545454546,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.94,
+      "privacy": 0.86,
+      "hate speech": 0.9,
+      "sexual content": 0.97,
+      "cbrn": 0.95,
+      "harm to minors": 0.96,
+      "integrity & quality violations": 0.86,
+      "illegal activities": 0.95,
+      "self-harm": 0.96,
+      "physical harm": 0.97,
+      "cyber": 0.96
+    },
+    "fpr": 0.023333333333333334,
+    "mean_latency": 0.4279132558618273,
+    "latency_ci_95": {
+      "lower": 0.4038809537163048,
+      "upper": 0.45194555800734987,
+      "mean": 0.4279132558618273,
+      "std_dev": 0.4587787798162123,
+      "n": 1400
+    },
+    "provider": "Mistral",
+    "model_type": "generalist",
+    "model_developer": "Mistral AI",
+    "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.04,
+      "cost_per_1M_output_tokens": 0.04,
+      "cost_per_h": "N/A",
+      "cost_source": "Ministral API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.00413752
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "mistral/bells-o-project-content-moderation-input/mistral-large-3": {
+    "model_provider_use_case": "mistral",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "mistral-large-3",
+    "accuracy": 0.9072727272727272,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.88,
+      "privacy": 0.79,
+      "hate speech": 0.85,
+      "sexual content": 0.96,
+      "cbrn": 0.96,
+      "harm to minors": 0.97,
+      "integrity & quality violations": 0.75,
+      "illegal activities": 0.95,
+      "self-harm": 0.93,
+      "physical harm": 0.97,
+      "cyber": 0.97
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.7675250434875488,
+    "latency_ci_95": {
+      "lower": 0.7033643424555687,
+      "upper": 0.8316857445195289,
+      "mean": 0.7675250434875488,
+      "std_dev": 1.2248334742699123,
+      "n": 1400
+    },
+    "provider": "Mistral",
+    "model_type": "generalist",
+    "model_developer": "Mistral AI",
+    "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 4.0,
+      "cost_per_1M_output_tokens": 12.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Mistral API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.43615200000000004
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "mistral/bells-o-project-content-moderation-output/ministral-3b-2512": {
+    "model_provider_use_case": "mistral",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "ministral-3b-2512",
+    "accuracy": 0.9518181818181818,
+    "accuracy_per_category": {
+      "hate speech": 0.97,
+      "physical harm": 0.98,
+      "harmful manipulation": 0.92,
+      "self-harm": 0.9,
+      "privacy": 0.91,
+      "cbrn": 1.0,
+      "harm to minors": 0.98,
+      "cyber": 0.97,
+      "sexual content": 0.93,
+      "illegal activities": 0.99,
+      "integrity & quality violations": 0.92
+    },
+    "fpr": 0.07,
+    "mean_latency": 0.4416024809224265,
+    "latency_ci_95": {
+      "lower": 0.41586757287002707,
+      "upper": 0.4673373889748259,
+      "mean": 0.4416024809224265,
+      "std_dev": 0.491281677613309,
+      "n": 1400
+    },
+    "provider": "Mistral",
+    "model_type": "generalist",
+    "model_developer": "Mistral AI",
+    "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.04,
+      "cost_per_1M_output_tokens": 0.04,
+      "cost_per_h": "N/A",
+      "cost_source": "Ministral API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.01877732
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "mistral/bells-o-project-content-moderation-output/mistral-large-3": {
+    "model_provider_use_case": "mistral",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "mistral-large-3",
+    "accuracy": 0.9190909090909091,
+    "accuracy_per_category": {
+      "hate speech": 0.94,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.78,
+      "self-harm": 0.9,
+      "privacy": 0.96,
+      "cbrn": 0.99,
+      "harm to minors": 1.0,
+      "cyber": 0.94,
+      "sexual content": 0.92,
+      "illegal activities": 0.92,
+      "integrity & quality violations": 0.77
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.8493295400483267,
+    "latency_ci_95": {
+      "lower": 0.7685226981889265,
+      "upper": 0.930136381907727,
+      "mean": 0.8493295400483267,
+      "std_dev": 1.5426097793117277,
+      "n": 1400
+    },
+    "provider": "Mistral",
+    "model_type": "generalist",
+    "model_developer": "Mistral AI",
+    "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 4.0,
+      "cost_per_1M_output_tokens": 12.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Mistral API pricing",
+      "cost_additional_info": "",
+      "total_cost": 1.898772
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "nvidia/bells-o-project-content-moderation-input/llama-3.1-nemotron-safety-guard-8b-v3": {
+    "model_provider_use_case": "nvidia",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "llama-3.1-nemotron-safety-guard-8b-v3",
+    "accuracy": 0.8636363636363636,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.62,
+      "privacy": 0.9,
+      "hate speech": 0.78,
+      "sexual content": 0.97,
+      "cbrn": 0.94,
+      "harm to minors": 0.93,
+      "integrity & quality violations": 0.66,
+      "illegal activities": 0.92,
+      "self-harm": 0.85,
+      "physical harm": 0.97,
+      "cyber": 0.96
+    },
+    "fpr": 0.043333333333333335,
+    "mean_latency": 0.15576645374298095,
+    "latency_ci_95": {
+      "lower": 0.15397129295656875,
+      "upper": 0.15756161452939316,
+      "mean": 0.15576645374298095,
+      "std_dev": 0.03426977865778628,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "NVIDIA",
+    "model_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.22782226487771615,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.14477626506222618
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "nvidia/bells-o-project-content-moderation-output/llama-3.1-nemotron-safety-guard-8b-v3": {
+    "model_provider_use_case": "nvidia",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "llama-3.1-nemotron-safety-guard-8b-v3",
+    "accuracy": 0.9054545454545454,
+    "accuracy_per_category": {
+      "hate speech": 0.87,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.73,
+      "self-harm": 0.86,
+      "privacy": 0.96,
+      "cbrn": 1.0,
+      "harm to minors": 0.95,
+      "cyber": 0.95,
+      "sexual content": 0.97,
+      "illegal activities": 0.96,
+      "integrity & quality violations": 0.72
+    },
+    "fpr": 0.056666666666666664,
+    "mean_latency": 0.18621215888432094,
+    "latency_ci_95": {
+      "lower": 0.18557276212673715,
+      "upper": 0.18685155564190473,
+      "mean": 0.18621215888432094,
+      "std_dev": 0.01220614082190163,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "NVIDIA",
+    "model_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.17354468328880082,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.1730738565630383
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-5-nano": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-5-nano",
+    "accuracy": 0.9254545454545454,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.98,
+      "privacy": 0.96,
+      "hate speech": 0.8,
+      "sexual content": 0.75,
+      "cbrn": 1.0,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.79,
+      "illegal activities": 1.0,
+      "self-harm": 0.92,
+      "physical harm": 1.0,
+      "cyber": 1.0
+    },
+    "fpr": 0.006666666666666667,
+    "mean_latency": 1.5597400277001516,
+    "latency_ci_95": {
+      "lower": 1.5193856746455585,
+      "upper": 1.6000943807547447,
+      "mean": 1.5597400277001516,
+      "std_dev": 0.7703681795673553,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/gpt-5-nano",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.4,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.048563300000000004
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-5.2": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-5.2",
+    "accuracy": 0.9381818181818182,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.94,
+      "privacy": 0.93,
+      "hate speech": 0.78,
+      "sexual content": 0.97,
+      "cbrn": 0.99,
+      "harm to minors": 0.97,
+      "integrity & quality violations": 0.83,
+      "illegal activities": 0.98,
+      "self-harm": 0.96,
+      "physical harm": 0.98,
+      "cyber": 0.99
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 1.0442856402056557,
+    "latency_ci_95": {
+      "lower": 1.0024455032847635,
+      "upper": 1.086125777126548,
+      "mean": 1.0442856402056557,
+      "std_dev": 0.7987319253759664,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/gpt-5.2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.75,
+      "cost_per_1M_output_tokens": 14.0,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.6885235000000001
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-oss-120b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-oss-120b",
+    "accuracy": 0.9318181818181818,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.97,
+      "privacy": 0.96,
+      "hate speech": 0.77,
+      "sexual content": 0.87,
+      "cbrn": 0.99,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.84,
+      "illegal activities": 0.97,
+      "self-harm": 0.92,
+      "physical harm": 0.99,
+      "cyber": 0.99
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.7237181276934488,
+    "latency_ci_95": {
+      "lower": 0.708643406560615,
+      "upper": 0.7387928488262826,
+      "mean": 0.7237181276934488,
+      "std_dev": 0.28777776367461705,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://www.together.ai/models/gpt-oss-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.032422900000000004
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-oss-20b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-oss-20b",
+    "accuracy": 0.6218181818181818,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.54,
+      "privacy": 0.69,
+      "hate speech": 0.69,
+      "sexual content": 0.62,
+      "cbrn": 0.58,
+      "harm to minors": 0.61,
+      "integrity & quality violations": 0.47,
+      "illegal activities": 0.69,
+      "self-harm": 0.72,
+      "physical harm": 0.62,
+      "cyber": 0.61
+    },
+    "fpr": 0.02,
+    "mean_latency": 1.7985665757315499,
+    "latency_ci_95": {
+      "lower": 1.7140022692418668,
+      "upper": 1.883130882221233,
+      "mean": 1.7985665757315499,
+      "std_dev": 1.6143401124211116,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://www.together.ai/models/gpt-oss-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.06419470000000001
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-oss-safeguard-120b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-oss-safeguard-120b",
+    "accuracy": 0.8781818181818182,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.82,
+      "privacy": 0.94,
+      "hate speech": 0.7,
+      "sexual content": 0.73,
+      "cbrn": 1.0,
+      "harm to minors": 0.96,
+      "integrity & quality violations": 0.69,
+      "illegal activities": 0.95,
+      "self-harm": 0.9,
+      "physical harm": 0.98,
+      "cyber": 0.99
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.9912308720179966,
+    "latency_ci_95": {
+      "lower": 0.9414346608201876,
+      "upper": 1.0410270832158057,
+      "mean": 0.9912308720179966,
+      "std_dev": 0.950614089089987,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 5.898870113479002,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 3.07,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 1.1834195244259302
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/gpt-oss-safeguard-20b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "gpt-oss-safeguard-20b",
+    "accuracy": 0.8881818181818182,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.9,
+      "privacy": 0.93,
+      "hate speech": 0.71,
+      "sexual content": 0.79,
+      "cbrn": 0.99,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.71,
+      "illegal activities": 0.94,
+      "self-harm": 0.88,
+      "physical harm": 0.95,
+      "cyber": 0.99
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.5077587234973907,
+    "latency_ci_95": {
+      "lower": 0.4910731057974469,
+      "upper": 0.5244443411973345,
+      "mean": 0.5077587234973907,
+      "std_dev": 0.3185299220402077,
+      "n": 1400
+    },
+    "provider": "OpenRouter",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.07,
+      "cost_per_1M_output_tokens": 0.3,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenRouter API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.10993276
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-input/omni-moderation": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "omni-moderation",
+    "accuracy": 0.74,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.38,
+      "privacy": 0.46,
+      "hate speech": 0.76,
+      "sexual content": 0.68,
+      "cbrn": 0.88,
+      "harm to minors": 0.95,
+      "integrity & quality violations": 0.47,
+      "illegal activities": 0.84,
+      "self-harm": 0.87,
+      "physical harm": 0.95,
+      "cyber": 0.9
+    },
+    "fpr": 0.023333333333333334,
+    "mean_latency": 0.3429920823233468,
+    "latency_ci_95": {
+      "lower": 0.33798123785984197,
+      "upper": 0.3480029267868517,
+      "mean": 0.3429920823233468,
+      "std_dev": 0.09565746531044984,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/omni-moderation-latest",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI Moderation API pricing",
+      "cost_additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits.",
+      "total_cost": 0.0
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-5-nano": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-5-nano",
+    "accuracy": 0.9027272727272727,
+    "accuracy_per_category": {
+      "hate speech": 0.92,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.83,
+      "self-harm": 0.93,
+      "privacy": 0.96,
+      "cbrn": 0.99,
+      "harm to minors": 0.99,
+      "cyber": 0.96,
+      "sexual content": 0.67,
+      "illegal activities": 0.94,
+      "integrity & quality violations": 0.75
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 1.4641724479198457,
+    "latency_ci_95": {
+      "lower": 1.4271079360623544,
+      "upper": 1.501236959777337,
+      "mean": 1.4641724479198457,
+      "std_dev": 0.7075648192793513,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/gpt-5-nano",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.4,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.05068805
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-5.2": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-5.2",
+    "accuracy": 0.9518181818181818,
+    "accuracy_per_category": {
+      "hate speech": 0.88,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.92,
+      "self-harm": 0.96,
+      "privacy": 0.97,
+      "cbrn": 0.99,
+      "harm to minors": 0.98,
+      "cyber": 0.98,
+      "sexual content": 0.97,
+      "illegal activities": 0.98,
+      "integrity & quality violations": 0.85
+    },
+    "fpr": 0.0,
+    "mean_latency": 1.1199401940618243,
+    "latency_ci_95": {
+      "lower": 1.0988383402519575,
+      "upper": 1.1410420478716912,
+      "mean": 1.1199401940618243,
+      "std_dev": 0.40283626113424703,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/gpt-5.2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.75,
+      "cost_per_1M_output_tokens": 14.0,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 1.3295257500000002
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-oss-120b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-oss-120b",
+    "accuracy": 0.9281818181818182,
+    "accuracy_per_category": {
+      "hate speech": 0.81,
+      "physical harm": 0.97,
+      "harmful manipulation": 0.9,
+      "self-harm": 0.93,
+      "privacy": 0.98,
+      "cbrn": 0.98,
+      "harm to minors": 0.99,
+      "cyber": 0.99,
+      "sexual content": 0.85,
+      "illegal activities": 0.96,
+      "integrity & quality violations": 0.85
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.7561430590493339,
+    "latency_ci_95": {
+      "lower": 0.7382452744998518,
+      "upper": 0.774040843598816,
+      "mean": 0.7561430590493339,
+      "std_dev": 0.34167029472682847,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://www.together.ai/models/gpt-oss-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.051996850000000004
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-oss-20b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-oss-20b",
+    "accuracy": 0.7181818181818181,
+    "accuracy_per_category": {
+      "hate speech": 0.76,
+      "physical harm": 0.76,
+      "harmful manipulation": 0.72,
+      "self-harm": 0.79,
+      "privacy": 0.72,
+      "cbrn": 0.7,
+      "harm to minors": 0.7,
+      "cyber": 0.76,
+      "sexual content": 0.68,
+      "illegal activities": 0.65,
+      "integrity & quality violations": 0.66
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 1.757279393843242,
+    "latency_ci_95": {
+      "lower": 1.6735166402449926,
+      "upper": 1.8410421474414915,
+      "mean": 1.757279393843242,
+      "std_dev": 1.5990383966194697,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "generalist",
+    "model_developer": "OpenAI",
+    "model_url": "https://www.together.ai/models/gpt-oss-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.08121905
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-oss-safeguard-120b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-oss-safeguard-120b",
+    "accuracy": 0.8418181818181818,
+    "accuracy_per_category": {
+      "hate speech": 0.68,
+      "physical harm": 0.93,
+      "harmful manipulation": 0.72,
+      "self-harm": 0.9,
+      "privacy": 0.92,
+      "cbrn": 0.99,
+      "harm to minors": 0.96,
+      "cyber": 0.97,
+      "sexual content": 0.56,
+      "illegal activities": 0.95,
+      "integrity & quality violations": 0.68
+    },
+    "fpr": 0.0,
+    "mean_latency": 1.1339499698366438,
+    "latency_ci_95": {
+      "lower": 1.0827253500645733,
+      "upper": 1.1851745896087142,
+      "mean": 1.1339499698366438,
+      "std_dev": 0.9778825354839475,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 2.437501498074139,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 3.07,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 1.3538102695438596
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/gpt-oss-safeguard-20b": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "gpt-oss-safeguard-20b",
+    "accuracy": 0.8927272727272727,
+    "accuracy_per_category": {
+      "hate speech": 0.75,
+      "physical harm": 0.96,
+      "harmful manipulation": 0.93,
+      "self-harm": 0.85,
+      "privacy": 0.94,
+      "cbrn": 0.98,
+      "harm to minors": 0.97,
+      "cyber": 0.97,
+      "sexual content": 0.78,
+      "illegal activities": 0.92,
+      "integrity & quality violations": 0.77
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.5182382569994245,
+    "latency_ci_95": {
+      "lower": 0.5020243010997294,
+      "upper": 0.5344522128991196,
+      "mean": 0.5182382569994245,
+      "std_dev": 0.309525856433271,
+      "n": 1400
+    },
+    "provider": "OpenRouter",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.07,
+      "cost_per_1M_output_tokens": 0.3,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenRouter API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.13713153
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "openai/bells-o-project-content-moderation-output/omni-moderation": {
+    "model_provider_use_case": "openai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "omni-moderation",
+    "accuracy": 0.7090909090909091,
+    "accuracy_per_category": {
+      "hate speech": 0.81,
+      "physical harm": 0.93,
+      "harmful manipulation": 0.32,
+      "self-harm": 0.84,
+      "privacy": 0.72,
+      "cbrn": 0.84,
+      "harm to minors": 0.79,
+      "cyber": 0.75,
+      "sexual content": 0.64,
+      "illegal activities": 0.76,
+      "integrity & quality violations": 0.4
+    },
+    "fpr": 0.006666666666666667,
+    "mean_latency": 0.3155126539298466,
+    "latency_ci_95": {
+      "lower": 0.3097630727618276,
+      "upper": 0.32126223509786567,
+      "mean": 0.3155126539298466,
+      "std_dev": 0.1097600145315037,
+      "n": 1400
+    },
+    "provider": "OpenAI",
+    "model_type": "specialized",
+    "model_developer": "OpenAI",
+    "model_url": "https://platform.openai.com/docs/models/omni-moderation-latest",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "OpenAI Moderation API pricing",
+      "cost_additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits.",
+      "total_cost": 0.0
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "qwen/bells-o-project-content-moderation-input/qwen3guard-gen-0.6b": {
+    "model_provider_use_case": "qwen",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "qwen3guard-gen-0.6b",
+    "accuracy": 0.9245454545454546,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.92,
+      "privacy": 0.93,
+      "hate speech": 0.84,
+      "sexual content": 1.0,
+      "cbrn": 0.95,
+      "harm to minors": 0.97,
+      "integrity & quality violations": 0.85,
+      "illegal activities": 0.96,
+      "self-harm": 0.8,
+      "physical harm": 0.97,
+      "cyber": 0.98
+    },
+    "fpr": 0.02666666666666667,
+    "mean_latency": 0.19419640813555036,
+    "latency_ci_95": {
+      "lower": 0.19107999562025785,
+      "upper": 0.19731282065084288,
+      "mean": 0.19419640813555036,
+      "std_dev": 0.05949259136775017,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Qwen",
+    "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.38936574188219714,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.1804947726726532
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "qwen/bells-o-project-content-moderation-input/qwen3guard-gen-8b": {
+    "model_provider_use_case": "qwen",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "qwen3guard-gen-8b",
+    "accuracy": 0.9454545454545454,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.95,
+      "privacy": 0.94,
+      "hate speech": 0.85,
+      "sexual content": 0.98,
+      "cbrn": 1.0,
+      "harm to minors": 0.99,
+      "integrity & quality violations": 0.84,
+      "illegal activities": 0.97,
+      "self-harm": 0.9,
+      "physical harm": 0.98,
+      "cyber": 1.0
+    },
+    "fpr": 0.006666666666666667,
+    "mean_latency": 0.19008652653012956,
+    "latency_ci_95": {
+      "lower": 0.18743975410113897,
+      "upper": 0.19273329895912014,
+      "mean": 0.19008652653012956,
+      "std_dev": 0.05052712045939928,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Qwen",
+    "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.38112538813051666,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.17667486604717045
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "qwen/bells-o-project-content-moderation-output/qwen3guard-gen-0.6b": {
+    "model_provider_use_case": "qwen",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "qwen3guard-gen-0.6b",
+    "accuracy": 0.9227272727272727,
+    "accuracy_per_category": {
+      "hate speech": 0.92,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.83,
+      "self-harm": 0.86,
+      "privacy": 0.96,
+      "cbrn": 0.94,
+      "harm to minors": 0.99,
+      "cyber": 0.96,
+      "sexual content": 0.96,
+      "illegal activities": 0.94,
+      "integrity & quality violations": 0.8
+    },
+    "fpr": 0.016666666666666666,
+    "mean_latency": 0.048780372994286676,
+    "latency_ci_95": {
+      "lower": 0.0480823539105477,
+      "upper": 0.04947839207802565,
+      "mean": 0.048780372994286676,
+      "std_dev": 0.013325246228475014,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Qwen",
+    "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05528456019474252,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.04533864667746756
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "qwen/bells-o-project-content-moderation-output/qwen3guard-gen-8b": {
+    "model_provider_use_case": "qwen",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "qwen3guard-gen-8b",
+    "accuracy": 0.9545454545454546,
+    "accuracy_per_category": {
+      "hate speech": 0.92,
+      "physical harm": 1.0,
+      "harmful manipulation": 0.93,
+      "self-harm": 0.91,
+      "privacy": 0.98,
+      "cbrn": 0.98,
+      "harm to minors": 1.0,
+      "cyber": 0.97,
+      "sexual content": 0.99,
+      "illegal activities": 0.99,
+      "integrity & quality violations": 0.83
+    },
+    "fpr": 0.016666666666666666,
+    "mean_latency": 0.13238392829895018,
+    "latency_ci_95": {
+      "lower": 0.13105481108714015,
+      "upper": 0.13371304551076021,
+      "mean": 0.13238392829895018,
+      "std_dev": 0.025372965476823234,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "Qwen",
+    "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.1500354918097283,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.12304350669119093
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "rakancorle1/bells-o-project-content-moderation-input/thinkguard": {
+    "model_provider_use_case": "rakancorle1",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "thinkguard",
+    "accuracy": 0.7781818181818182,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.69,
+      "privacy": 0.73,
+      "hate speech": 0.72,
+      "sexual content": 0.86,
+      "cbrn": 0.84,
+      "harm to minors": 0.86,
+      "integrity & quality violations": 0.63,
+      "illegal activities": 0.83,
+      "self-harm": 0.64,
+      "physical harm": 0.9,
+      "cyber": 0.86
+    },
+    "fpr": 0.08,
+    "mean_latency": 0.1785528426510947,
+    "latency_ci_95": {
+      "lower": 0.17834486711263356,
+      "upper": 0.17876081818955583,
+      "mean": 0.1785528426510947,
+      "std_dev": 0.003970271478323348,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "RakanCorle1",
+    "model_url": "https://huggingface.co/Rakancorle1/ThinkGuard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2906798634516623,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.16595494764182303
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "rakancorle1/bells-o-project-content-moderation-output/thinkguard": {
+    "model_provider_use_case": "rakancorle1",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "thinkguard",
+    "accuracy": 0.8145454545454546,
+    "accuracy_per_category": {
+      "hate speech": 0.71,
+      "physical harm": 0.94,
+      "harmful manipulation": 0.74,
+      "self-harm": 0.69,
+      "privacy": 0.87,
+      "cbrn": 0.86,
+      "harm to minors": 0.92,
+      "cyber": 0.84,
+      "sexual content": 0.86,
+      "illegal activities": 0.91,
+      "integrity & quality violations": 0.62
+    },
+    "fpr": 0.08333333333333333,
+    "mean_latency": 0.18875198875154767,
+    "latency_ci_95": {
+      "lower": 0.1882683485403946,
+      "upper": 0.18923562896270074,
+      "mean": 0.18875198875154767,
+      "std_dev": 0.00923273453368272,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "RakanCorle1",
+    "model_url": "https://huggingface.co/Rakancorle1/ThinkGuard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.19000934406913317,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.17543448732296626
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "saillab/bells-o-project-content-moderation-input/xguard": {
+    "model_provider_use_case": "saillab",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "xguard",
+    "accuracy": 0.8281818181818181,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.77,
+      "privacy": 0.82,
+      "hate speech": 0.76,
+      "sexual content": 0.87,
+      "cbrn": 0.92,
+      "harm to minors": 0.9,
+      "integrity & quality violations": 0.65,
+      "illegal activities": 0.91,
+      "self-harm": 0.72,
+      "physical harm": 0.9,
+      "cyber": 0.89
+    },
+    "fpr": 0.0,
+    "mean_latency": 1.438086905990328,
+    "latency_ci_95": {
+      "lower": 1.4220830667384288,
+      "upper": 1.4540907452422274,
+      "mean": 1.438086905990328,
+      "std_dev": 0.30551471098781485,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "SAIL Lab",
+    "model_url": "https://huggingface.co/saillab/x-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 11.345572408123338,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 1.3366218854010106
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "saillab/bells-o-project-content-moderation-output/xguard": {
+    "model_provider_use_case": "saillab",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "xguard",
+    "accuracy": 0.9009090909090909,
+    "accuracy_per_category": {
+      "hate speech": 0.86,
+      "physical harm": 0.98,
+      "harmful manipulation": 0.77,
+      "self-harm": 0.9,
+      "privacy": 0.94,
+      "cbrn": 0.94,
+      "harm to minors": 0.98,
+      "cyber": 0.91,
+      "sexual content": 0.95,
+      "illegal activities": 0.96,
+      "integrity & quality violations": 0.72
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.4900521091052464,
+    "latency_ci_95": {
+      "lower": 0.4689487805620818,
+      "upper": 0.511155437648411,
+      "mean": 0.4900521091052464,
+      "std_dev": 0.40286441392372,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "SAIL Lab",
+    "model_url": "https://huggingface.co/saillab/x-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.9604441053405044,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.4554762102961541
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "toxicityprompts/bells-o-project-content-moderation-input/polyguard-ministral": {
+    "model_provider_use_case": "toxicityprompts",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "polyguard-ministral",
+    "accuracy": 0.92,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.89,
+      "privacy": 0.84,
+      "hate speech": 0.82,
+      "sexual content": 0.99,
+      "cbrn": 1.0,
+      "harm to minors": 0.99,
+      "integrity & quality violations": 0.8,
+      "illegal activities": 0.95,
+      "self-harm": 0.89,
+      "physical harm": 0.97,
+      "cyber": 0.98
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.17871589626584733,
+    "latency_ci_95": {
+      "lower": 0.17849986671699192,
+      "upper": 0.17893192581470274,
+      "mean": 0.17871589626584733,
+      "std_dev": 0.0041240232512055235,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "ToxicityPrompts",
+    "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.4409996758824333,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.16610649691820145
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "toxicityprompts/bells-o-project-content-moderation-input/polyguard-qwen": {
+    "model_provider_use_case": "toxicityprompts",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "polyguard-qwen",
+    "accuracy": 0.9354545454545454,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.88,
+      "privacy": 0.93,
+      "hate speech": 0.83,
+      "sexual content": 1.0,
+      "cbrn": 0.99,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.84,
+      "illegal activities": 0.98,
+      "self-harm": 0.91,
+      "physical harm": 0.97,
+      "cyber": 0.98
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.16735810961042133,
+    "latency_ci_95": {
+      "lower": 0.167181707969697,
+      "upper": 0.16753451125114566,
+      "mean": 0.16735810961042133,
+      "std_dev": 0.0033675229696693413,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "ToxicityPrompts",
+    "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.42708542860944143,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.15555006521013048
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "toxicityprompts/bells-o-project-content-moderation-output/polyguard-ministral": {
+    "model_provider_use_case": "toxicityprompts",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "polyguard-ministral",
+    "accuracy": 0.9318181818181818,
+    "accuracy_per_category": {
+      "hate speech": 0.83,
+      "physical harm": 0.99,
+      "harmful manipulation": 0.88,
+      "self-harm": 0.88,
+      "privacy": 0.97,
+      "cbrn": 0.99,
+      "harm to minors": 0.98,
+      "cyber": 0.96,
+      "sexual content": 0.97,
+      "illegal activities": 0.97,
+      "integrity & quality violations": 0.83
+    },
+    "fpr": 0.013333333333333334,
+    "mean_latency": 0.3748241812842233,
+    "latency_ci_95": {
+      "lower": 0.37309410657377223,
+      "upper": 0.37655425599467435,
+      "mean": 0.3748241812842233,
+      "std_dev": 0.03302727969403025,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "ToxicityPrompts",
+    "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.4686604178361155,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.34837825293805863
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "toxicityprompts/bells-o-project-content-moderation-output/polyguard-qwen": {
+    "model_provider_use_case": "toxicityprompts",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "polyguard-qwen",
+    "accuracy": 0.9572727272727273,
+    "accuracy_per_category": {
+      "hate speech": 0.92,
+      "physical harm": 1.0,
+      "harmful manipulation": 0.88,
+      "self-harm": 0.93,
+      "privacy": 0.98,
+      "cbrn": 1.0,
+      "harm to minors": 1.0,
+      "cyber": 0.98,
+      "sexual content": 0.98,
+      "illegal activities": 1.0,
+      "integrity & quality violations": 0.86
+    },
+    "fpr": 0.013333333333333334,
+    "mean_latency": 0.3705882680416107,
+    "latency_ci_95": {
+      "lower": 0.3683706245632213,
+      "upper": 0.37280591152000014,
+      "mean": 0.3705882680416107,
+      "std_dev": 0.04233501072523893,
+      "n": 1400
+    },
+    "provider": "RunPod",
+    "model_type": "specialized",
+    "model_developer": "ToxicityPrompts",
+    "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.47809043060129436,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": 2.39,
+      "cost_source": "RunPod",
+      "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).",
+      "total_cost": 0.34444120690756375
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    },
+    "num_samples": 1400
+  },
+  "virtue-ai/bells-o-project-content-moderation-input/virtueguard-text-lite": {
+    "model_provider_use_case": "virtue-ai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "virtueguard-text-lite",
+    "accuracy": 0.7290909090909091,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.68,
+      "privacy": 0.61,
+      "hate speech": 0.57,
+      "sexual content": 0.64,
+      "cbrn": 0.86,
+      "harm to minors": 0.9,
+      "integrity & quality violations": 0.61,
+      "illegal activities": 0.86,
+      "self-harm": 0.62,
+      "physical harm": 0.84,
+      "cyber": 0.83
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.24810820971216474,
+    "latency_ci_95": {
+      "lower": 0.22206087989181125,
+      "upper": 0.27415553953251826,
+      "mean": 0.24810820971216474,
+      "std_dev": 0.4972458368778715,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "specialized",
+    "model_developer": "Virtue AI",
+    "model_url": "https://www.together.ai/models/virtueguard-text-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.0098006
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "virtue-ai/bells-o-project-content-moderation-output/virtueguard-text-lite": {
+    "model_provider_use_case": "virtue-ai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "virtueguard-text-lite",
+    "accuracy": 0.11181818181818182,
+    "accuracy_per_category": {
+      "hate speech": 0.02,
+      "physical harm": 0.01,
+      "harmful manipulation": 0.12,
+      "self-harm": 0.0,
+      "privacy": 0.08,
+      "cbrn": 0.25,
+      "harm to minors": 0.1,
+      "cyber": 0.27,
+      "sexual content": 0.11,
+      "illegal activities": 0.14,
+      "integrity & quality violations": 0.13
+    },
+    "fpr": 0.0033333333333333335,
+    "mean_latency": 0.31551449128559655,
+    "latency_ci_95": {
+      "lower": 0.2464019144914514,
+      "upper": 0.3846270680797417,
+      "mean": 0.31551449128559655,
+      "std_dev": 1.3193652218407883,
+      "n": 1400
+    },
+    "provider": "Together AI",
+    "model_type": "specialized",
+    "model_developer": "Virtue AI",
+    "model_url": "https://www.together.ai/models/virtueguard-text-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "cost_per_h": "N/A",
+      "cost_source": "Together AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.0810904
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "x-ai/bells-o-project-content-moderation-input/grok-4-1-fast-non-reasoning": {
+    "model_provider_use_case": "x-ai",
+    "dataset_name": "bells-o-project-content-moderation-input",
+    "model_name": "grok-4-1-fast-non-reasoning",
+    "accuracy": 0.8263636363636364,
+    "accuracy_per_category": {
+      "harmful manipulation": 0.9,
+      "privacy": 0.76,
+      "hate speech": 0.48,
+      "sexual content": 0.72,
+      "cbrn": 0.93,
+      "harm to minors": 0.98,
+      "integrity & quality violations": 0.73,
+      "illegal activities": 0.97,
+      "self-harm": 0.66,
+      "physical harm": 0.97,
+      "cyber": 0.99
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.9228259367602212,
+    "latency_ci_95": {
+      "lower": 0.6722643846420433,
+      "upper": 1.173387488878399,
+      "mean": 0.9228259367602212,
+      "std_dev": 4.783242256757777,
+      "n": 1400
+    },
+    "provider": "xAI",
+    "model_type": "generalist",
+    "model_developer": "X-AI",
+    "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.5,
+      "cost_per_h": "N/A",
+      "cost_source": "X-AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.06539510000000001
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  },
+  "x-ai/bells-o-project-content-moderation-output/grok-4-1-fast-non-reasoning": {
+    "model_provider_use_case": "x-ai",
+    "dataset_name": "bells-o-project-content-moderation-output",
+    "model_name": "grok-4-1-fast-non-reasoning",
+    "accuracy": 0.8363636363636363,
+    "accuracy_per_category": {
+      "hate speech": 0.64,
+      "physical harm": 0.95,
+      "harmful manipulation": 0.83,
+      "self-harm": 0.9,
+      "privacy": 0.91,
+      "cbrn": 0.75,
+      "harm to minors": 0.96,
+      "cyber": 0.96,
+      "sexual content": 0.63,
+      "illegal activities": 0.95,
+      "integrity & quality violations": 0.72
+    },
+    "fpr": 0.0,
+    "mean_latency": 0.8401098469325474,
+    "latency_ci_95": {
+      "lower": 0.6645747595715246,
+      "upper": 1.0156449342935703,
+      "mean": 0.8401098469325474,
+      "std_dev": 3.350980389093776,
+      "n": 1400
+    },
+    "provider": "xAI",
+    "model_type": "generalist",
+    "model_developer": "X-AI",
+    "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.5,
+      "cost_per_h": "N/A",
+      "cost_source": "X-AI API pricing",
+      "cost_additional_info": "",
+      "total_cost": 0.1310404
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2."
+    },
+    "num_samples": 1400
+  }
+}

data/jb_model_info_mapping.json

@@ -0,0 +1,775 @@
+{
+  "_comment": "Consolidated model information mapping. Contains developer, type, cost information, and execution specifications for each model.",
+  "_units": {
+    "api_costs": "cost per 1M input tokens + cost per 1M output tokens (if applicable)",
+    "local_costs": "cost per 1M tokens (calculated using formula)"
+  },
+  "anthropic/claude-haiku-4-5": {
+    "model_developer": "Anthropic",
+    "model_type": "generalist",
+    "url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.0,
+      "cost_per_1M_output_tokens": 5.0,
+      "source": "Claude API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "anthropic/claude-sonnet-4-5": {
+    "model_developer": "Anthropic",
+    "model_type": "generalist",
+    "url": "https://platform.claude.com/docs/en/about-claude/models/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 3.0,
+      "cost_per_1M_output_tokens": 15.0,
+      "source": "Claude API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "azure/analyze-text": {
+    "model_developer": "Azure",
+    "model_type": "specialized",
+    "url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 380.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Azure AI Content Safety",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "azure/promptshield": {
+    "model_developer": "Azure",
+    "model_type": "specialized",
+    "url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/content-filter-prompt-shields?view=foundry-classic#prompt-shields-for-user-prompts",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 380.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Azure AI Content Safety",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "google/gemini-2.5-flash": {
+    "model_developer": "Google",
+    "model_type": "generalist",
+    "url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.3,
+      "cost_per_1M_output_tokens": 2.5,
+      "source": "Gemini Developer API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "lakera/lakera-guard_default": {
+    "model_developer": "Lakera",
+    "model_type": "specialized",
+    "url": "https://www.lakera.ai/lakera-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Lakera API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "meta/llama-guard-4-12b": {
+    "model_developer": "Meta",
+    "model_type": "specialized",
+    "url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Together AI pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "mistral/ministral-3b-2512": {
+    "model_developer": "Mistral AI",
+    "model_type": "generalist",
+    "url": "https://docs.mistral.ai/models/ministral-3-3b-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.04,
+      "cost_per_1M_output_tokens": 0.04,
+      "source": "Ministral API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "mistral/mistral-large-3": {
+    "model_developer": "Mistral AI",
+    "model_type": "generalist",
+    "url": "https://docs.mistral.ai/models/mistral-large-3-25-12",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 4.0,
+      "cost_per_1M_output_tokens": 12.0,
+      "source": "Mistral API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-5-nano": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://platform.openai.com/docs/models/gpt-5-nano",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.4,
+      "source": "OpenAI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-5.2": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://platform.openai.com/docs/models/gpt-5.2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 1.75,
+      "cost_per_1M_output_tokens": 14.0,
+      "source": "OpenAI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-20b": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://www.together.ai/models/gpt-oss-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-120b": {
+    "model_developer": "OpenAI",
+    "model_type": "generalist",
+    "url": "https://www.together.ai/models/gpt-oss-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.05,
+      "cost_per_1M_output_tokens": 0.2,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/gpt-oss-safeguard-20b": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/openai/gpt-oss-safeguard-20b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.07,
+      "cost_per_1M_output_tokens": 0.3,
+      "source": "OpenRouter API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "openai/omni-moderation": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://platform.openai.com/docs/models/omni-moderation-latest",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "OpenAI Moderation API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "virtue-ai/virtueguard-text-lite": {
+    "model_developer": "Virtue AI",
+    "model_type": "specialized",
+    "url": "https://www.together.ai/models/virtueguard-text-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Together AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "x-ai/grok-4-1-fast-non-reasoning": {
+    "model_developer": "X-AI",
+    "model_type": "generalist",
+    "url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.2,
+      "cost_per_1M_output_tokens": 0.5,
+      "source": "X-AI API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": ""
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "aws/bedrock-guardrail": {
+    "model_developer": "Amazon Web Services",
+    "model_type": "specialized",
+    "url": "https://aws.amazon.com/bedrock/guardrails/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 150.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "Amazon Bedrock API pricing",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  },
+  "nvidia/aegis-ai-content-safety-llamaguard-defensive-1.0": {
+    "model_developer": "NVIDIA",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/nvidia/Aegis-AI-Content-Safety-LlamaGuard-Defensive-1.0",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-2b": {
+    "model_developer": "Google",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/google/shieldgemma-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-9b": {
+    "model_developer": "Google",
+    "url": "https://huggingface.co/google/shieldgemma-9b",
+    "model_type": "specialized",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "google/shieldgemma-27b": {
+    "model_developer": "Google",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/google/shieldgemma-27b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "saillab/xguard": {
+    "model_developer": "SAIL Lab",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/saillab/x-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "openai/gpt-oss-safeguard-120b": {
+    "model_developer": "OpenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/openai/gpt-oss-safeguard-120b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 3.07,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-8b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-4b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-4B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "qwen/qwen3guard-gen-0.6b": {
+    "model_developer": "Qwen",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "nvidia/llama-3.1-nemotron-safety-guard-8b-v3": {
+    "model_developer": "NVIDIA",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "rakancorle1/thinkguard": {
+    "model_developer": "RakanCorle1",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/Rakancorle1/ThinkGuard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "allenai/wildguard": {
+    "model_developer": "AllenAI",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/allenai/wildguard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-ministral": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-qwen": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "toxicityprompts/polyguard-qwen-smol": {
+    "model_developer": "ToxicityPrompts",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen-Smol",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.0-2b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.0-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.0-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.1-2b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.1-2b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.1-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.1-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.2-5b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-5b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.2-3b-a800m": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.2-3b-a800m",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "ibm-granite/granite-guardian-3.3-8b": {
+    "model_developer": "IBM",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM."
+    }
+  },
+  "govtech/lionguard-2": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "govtech/lionguard-2.1": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2.1",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "govtech/lionguard-2-lite": {
+    "model_developer": "GovTech",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/govtech/lionguard-2-lite",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "leolee99/piguard": {
+    "model_developer": "Leolee99",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/leolee99/PIGuard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "meta-llama/llama-prompt-guard-2-22m": {
+    "model_developer": "Meta",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-22M",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "meta-llama/llama-prompt-guard-2-86m": {
+    "model_developer": "Meta",
+    "model_type": "specialized",
+    "url": "https://huggingface.co/meta-llama/Llama-Prompt-Guard-2-86M",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library."
+    }
+  },
+  "protectai/llm-guard": {
+    "model_developer": "ProtectAI",
+    "model_type": "specialized",
+    "url": "https://protectai.github.io/llm-guard/input_scanners/prompt_injection/",
+    "cost_info": {
+      "cost_per_1M_input_tokens": "N/A",
+      "cost_per_1M_output_tokens": "N/A",
+      "source": "RunPod",
+      "cost_per_h": 2.39,
+      "additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens)."
+    },
+    "execution_specifications": {
+      "type": "Local",
+      "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the llm-guard Python SDK."
+    }
+  },
+  "neuraltrust/promptguard": {
+    "model_developer": "NeuralTrust",
+    "model_type": "specialized",
+    "url": "https://neuraltrust.ai/prompt-guard",
+    "cost_info": {
+      "cost_per_1M_input_tokens": 0.0,
+      "cost_per_1M_output_tokens": 0.0,
+      "source": "NeuralTrust",
+      "cost_per_h": "N/A",
+      "additional_info": "Usage is free on the API. We were banned after about 1800 requests."
+    },
+    "execution_specifications": {
+      "type": "API",
+      "details": "REST API accessed through a 'Memory-Optimized 16GB RAM' CPU3 RunPod instance from US-KS-2."
+    }
+  }
+}

jailbreak.py

@@ -0,0 +1,981 @@
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+import gradio as gr
+import pandas as pd
+
+from shared import (
+    CATEGORY_TABLE_CSS,
+    DEFAULT_WEIGHTS,
+    LEADERBOARD_DARK_MODE_CSS,
+    LEADERBOARD_TABLE_CSS,
+    build_pareto_figure,
+    build_weights,
+    create_empty_pareto_figure,
+    create_weight_bar_html,
+    escape_html,
+    get_color_for_accuracy,
+    sort_by_overall_score,
+)
+
+# ── Data loading ────────────────────────────────────────────────────────
+
+_DATA_DIR = Path(__file__).parent / "data"
+
+
+def _load_metrics() -> Dict[str, Any]:
+    metrics_path = _DATA_DIR / "jailbreak_metrics.json"
+    if not metrics_path.exists():
+        print(f"Warning: {metrics_path} not found")
+        return {}
+    with open(metrics_path, "r") as f:
+        return json.load(f)
+
+
+METRICS_DATA = _load_metrics()
+
+
+# ── Constants ───────────────────────────────────────────────────────────
+
+AGGREGATE_OPTION = "Aggregate (All Datasets)"
+OVERVIEW_OPTION = "Overview (All Datasets)"
+
+DATASET_DESCRIPTIONS: Dict[str, Dict[str, Any]] = {
+    "allenai-wildjailbreak": {
+        "url": "https://huggingface.co/datasets/allenai/wildjailbreak",
+        "summary": "This dataset generated adversarial prompt using WildTeaming (https://github.com/allenai/wildteaming).\nWe are using the eval split of this dataset. It only contains prompts that look like jailbreaks, but do not necessarily have harmful content.",
+        "categories": {
+            "adversarial_harmful": "Jailbreak prompts with harmful content.",
+            "adversarial_benign": "Jailbreak prompts with benign content.",
+        },
+        "accuracy_categories": ["adversarial_harmful"],
+        "fpr_categories": ["adversarial_benign"],
+    },
+    "bells-o-project-jailbreak-dataset": {
+        "url": "https://huggingface.co/datasets/centrepourlasecuriteia/jailbreak-dataset",
+        "summary": "This dataset generated adversarial prompts from the companion dataset centrepourlasecuriteia/content-moderation-input utilizing uncensored models.",
+        "categories": {
+            "Benign": "Adversarial prompts with benign content.",
+            "CBRN": "Adversarial prompts with harmful CBRN content.",
+            "Cyber": "Adversarial prompts with harmful Cyber content.",
+            "Harm to Minors": "Adversarial prompts with harmful content regarding Harm to Minors.",
+            "Harmful Manipulation": "Adversarial prompts with harmful content regarding manipulation.",
+            "Illegal Activities": "Adversarial prompts with harmful content regarding Illegal Activities.",
+            "Integrity & Quality": "Adversarial prompts with harmful content regarding Integrity & Quality.",
+            "Physical Harm": "Adversarial prompts with Physical Harm content.",
+            "Privacy": "Adversarial prompts with harmful Privacy content.",
+            "Self-Harm": "Adversarial prompts with harmful Self-Harm\t content.",
+            "Sexual Content": "Adversarial prompts with harmful sexual content.",
+        },
+        "fpr_categories": ["Benign"],
+        "accuracy_categories": [
+            "CBRN", "Cyber", "Harm to Minors", "Harmful Manipulation",
+            "Illegal Activities", "Integrity & Quality", "Physical Harm",
+            "Privacy", "Self-Harm", "Sexual Content",
+        ],
+    },
+    "bells-o-project-BET-jailbreak-dataset": {
+        "url": "https://huggingface.co/datasets/centrepourlasecuriteia/BET-jailbreak-dataset",
+        "summary": "This dataset generated adversarial prompts from the companion dataset centrepourlasecuriteia/content-moderation-input using BET (https://github.com/qfeuilla/BehaviorEliciationTool).",
+        "categories": {
+            "Benign": "Adversarial prompts with benign content.",
+            "CBRN": "Adversarial prompts with harmful CBRN content.",
+            "Cyber": "Adversarial prompts with harmful Cyber content.",
+            "Harm to Minors": "Adversarial prompts with harmful content regarding Harm to Minors.",
+            "Harmful Manipulation": "Adversarial prompts with harmful content regarding manipulation.",
+            "Illegal Activities": "Adversarial prompts with harmful content regarding Illegal Activities.",
+            "Integrity & Quality": "Adversarial prompts with harmful content regarding Integrity & Quality.",
+            "Physical Harm": "Adversarial prompts with Physical Harm content.",
+            "Privacy": "Adversarial prompts with harmful Privacy content.",
+            "Self-Harm": "Adversarial prompts with harmful Self-Harm content.",
+            "Sexual Content": "Adversarial prompts with harmful sexual content.",
+        },
+        "fpr_categories": ["Benign"],
+        "accuracy_categories": [
+            "CBRN", "Cyber", "Harm to Minors", "Harmful Manipulation",
+            "Illegal Activities", "Integrity & Quality", "Physical Harm",
+            "Privacy", "Self-Harm", "Sexual Content",
+        ],
+    },
+    "deepset-prompt-injections": {
+        "url": "https://huggingface.co/datasets/deepset/prompt-injections",
+        "summary": "The prompt injections dataset by Deepset.",
+        "categories": {"1": "Adversarial prompts with harmful content.", "0": "Normal prompts with benign content."},
+        "accuracy_categories": ["1"],
+        "fpr_categories": ["0"],
+    },
+    "local_datasets-malignant": {
+        "url": "https://www.kaggle.com/datasets/marycamilainfo/prompt-injection-malignant",
+        "summary": "This dataset is specifically curated for jailbreak prompt injection instances.",
+        "categories": {
+            "conversation": "Normal prompts with benign content, extracted from the Persona-Chat dataset (https://www.kaggle.com/datasets/atharvjairath/personachat).",
+            "act_as": "Role-playing prompts with benign content.",
+            "jailbreak": "Adversarial prompts with harmful content.",
+        },
+        "accuracy_categories": ["jailbreak"],
+        "fpr_categories": ["act_as", "conversation"],
+    },
+    "TrustAIRLab-in-the-wild-jailbreak-prompts": {
+        "url": "https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts",
+        "summary": "This dataset consists of jailbreak prompts that were collected from Reddit, Discord, and websites. We are using the 'jailbreak_2023_12_25' version. It does not contain any benign samples.",
+        "categories": {
+            "true": "Adversarial prompts with harmful content.",
+        },
+        "accuracy_categories": ["true"],
+        "fpr_categories": ["N/A"],
+    },
+}
+
+DATASET_LABELS = {
+    "allenai-wildjailbreak": "WildJailbreak (AllenAI)",
+    "bells-o-project-jailbreak-dataset": "BELLS-O Jailbreak (CeSIA)",
+    "bells-o-project-BET-jailbreak-dataset": "BELLS-O BET Jailbreak (CeSIA)",
+    "deepset-prompt-injections": "Prompt Injections (Deepset)",
+    "local_datasets-malignant": "Prompt Injection Malignant (Mary Camila)",
+    "TrustAIRLab-in-the-wild-jailbreak-prompts": "In-the-Wild Jailbreak Prompts (TrustAIRLab)",
+}
+
+CATEGORY_LABELS = {
+    "hate speech": "Hate Speech",
+    "harmful manipulation": "Harmful Manipulation",
+    "privacy": "Privacy",
+    "cyber": "Cyber",
+    "physical harm": "Physical Harm",
+    "sexual content": "Sexual Content",
+    "integrity & quality violations": "Integrity & Quality",
+    "cbrn": "CBRN",
+    "illegal activities": "Illegal Activities",
+    "self-harm": "Self-Harm",
+    "harm to minors": "Harm to Minors",
+}
+
+
+# ── Dataset info HTML ───────────────────────────────────────────────────
+
+def create_dataset_info_html(dataset_name: str) -> str:
+    info = DATASET_DESCRIPTIONS.get(dataset_name)
+    if not info:
+        return ""
+
+    ds_label = DATASET_LABELS.get(dataset_name, dataset_name)
+    url = info.get("url", "")
+    summary = info.get("summary", "")
+    categories = info.get("categories", {})
+    accuracy_categories = info.get("accuracy_categories", [])
+    fpr_categories = info.get("fpr_categories", [])
+
+    title_html = f'<a href="{url}" target="_blank">{ds_label}</a>' if url else ds_label
+
+    html = """<style>
+        .ds-info-box {
+            background: #f8fafc;
+            border: 1px solid #e2e8f0;
+            border-radius: 8px;
+            padding: 16px;
+            margin: 12px 0;
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+        }
+        .ds-info-box .ds-title { margin: 0 0 8px 0 !important; padding: 0 !important; color: #1e293b; }
+        .ds-info-box .ds-title a { color: #1e293b; text-decoration: underline; }
+        .ds-info-box .ds-text { color: #475569; font-size: 14px; }
+        .ds-info-box .ds-cat-item { color: #475569; font-size: 13px; margin-bottom: 2px; }
+        @media (prefers-color-scheme: dark) {
+            .ds-info-box {
+                background: #2a2a2a !important;
+                border-color: #4a4a4a !important;
+            }
+            .ds-info-box .ds-title,
+            .ds-info-box .ds-title a { color: #f3f4f6 !important; }
+            .ds-info-box .ds-text { color: #d1d5db !important; }
+            .ds-info-box .ds-cat-item { color: #d1d5db !important; }
+        }
+    </style>"""
+
+    html += f'<div class="ds-info-box"><h4 class="ds-title">{title_html}</h4>'
+
+    if summary:
+        html += f'<p class="ds-text" style="margin:0 0 8px 0;">{summary}</p>'
+
+    if categories:
+        html += '<div style="margin-top:8px;"><strong class="ds-text">Categories:</strong>'
+        html += '<ul style="margin:4px 0 0 0; padding-left:20px; list-style:none;">'
+        for cat_key, cat_desc in categories.items():
+            cat_label = CATEGORY_LABELS.get(cat_key, cat_key)
+            html += f'<li class="ds-cat-item"><strong>{cat_label}:</strong> {cat_desc}</li>'
+        html += "</ul></div>"
+
+    if accuracy_categories:
+        cat_names = ", ".join(CATEGORY_LABELS.get(c, c) for c in accuracy_categories)
+        html += f'<p class="ds-text" style="margin:8px 0 0 0;"><strong>Detection Rate from:</strong> {cat_names}</p>'
+
+    if fpr_categories:
+        cat_names = ", ".join(CATEGORY_LABELS.get(c, c) for c in fpr_categories)
+        html += f'<p class="ds-text" style="margin:4px 0 0 0;"><strong>FPR from:</strong> {cat_names}</p>'
+
+    html += "</div>"
+    return html
+
+
+# ── Data helpers ────────────────────────────────────────────────────────
+
+def get_available_datasets() -> List[str]:
+    datasets = set()
+    for data in METRICS_DATA.values():
+        ds = data.get("dataset_name", "")
+        if ds:
+            datasets.add(ds)
+    return sorted(datasets)
+
+
+def get_dataset_choices() -> List[tuple]:
+    available = get_available_datasets()
+    choices = [(AGGREGATE_OPTION, AGGREGATE_OPTION)]
+    for ds in available:
+        label = DATASET_LABELS.get(ds, ds)
+        choices.append((label, ds))
+    return choices
+
+
+def filter_metrics_by_dataset(dataset_name: str) -> Dict[str, Any]:
+    return {k: v for k, v in METRICS_DATA.items() if v.get("dataset_name") == dataset_name}
+
+
+def _get_model_identity_key(data: dict) -> str:
+    return f"{data.get('model_provider_use_case', '')}/{data.get('model_name', '')}"
+
+
+def compute_aggregate_metrics() -> Dict[str, Any]:
+    model_groups: Dict[str, List[Dict[str, Any]]] = {}
+    for key, data in METRICS_DATA.items():
+        identity = _get_model_identity_key(data)
+        if identity not in model_groups:
+            model_groups[identity] = []
+        model_groups[identity].append(data)
+
+    aggregated = {}
+    for identity, entries in model_groups.items():
+        template = entries[0].copy()
+
+        template["accuracy"] = sum(e.get("accuracy", 0) for e in entries) / len(entries)
+        fpr_values = [e.get("fpr") for e in entries if isinstance(e.get("fpr"), (int, float))]
+        if fpr_values:
+            template["fpr"] = sum(fpr_values) / len(fpr_values)
+        else:
+            template["fpr"] = "N/A"
+        template["mean_latency"] = sum(e.get("mean_latency", 0) for e in entries) / len(entries)
+
+        all_categories = set()
+        for e in entries:
+            all_categories.update(e.get("accuracy_per_category", {}).keys())
+
+        agg_categories = {}
+        for cat in all_categories:
+            values = [
+                e.get("accuracy_per_category", {}).get(cat, 0)
+                for e in entries
+                if cat in e.get("accuracy_per_category", {})
+            ]
+            if values:
+                agg_categories[cat] = sum(values) / len(values)
+        template["accuracy_per_category"] = agg_categories
+
+        ci_entries = [e.get("latency_ci_95", {}) for e in entries]
+        if ci_entries:
+            template["latency_ci_95"] = {
+                "lower": sum(c.get("lower", 0) for c in ci_entries) / len(ci_entries),
+                "upper": sum(c.get("upper", 0) for c in ci_entries) / len(ci_entries),
+                "mean": sum(c.get("mean", 0) for c in ci_entries) / len(ci_entries),
+                "std_dev": sum(c.get("std_dev", 0) for c in ci_entries) / len(ci_entries),
+                "n": sum(c.get("n", 0) for c in ci_entries),
+            }
+
+        total_cost = 0
+        for e in entries:
+            tc = e.get("cost_info", {}).get("total_cost", 0)
+            if isinstance(tc, (int, float)):
+                total_cost += tc
+        template_cost = template.get("cost_info", {}).copy()
+        template_cost["total_cost"] = total_cost
+        template["cost_info"] = template_cost
+
+        template["dataset_name"] = AGGREGATE_OPTION
+        template["num_datasets"] = len(entries)
+
+        agg_key = f"aggregate/{identity}"
+        aggregated[agg_key] = template
+
+    return aggregated
+
+
+def get_metrics_for_display(dataset_name: str) -> Dict[str, Any]:
+    if dataset_name == AGGREGATE_OPTION:
+        return compute_aggregate_metrics()
+    return filter_metrics_by_dataset(dataset_name)
+
+
+def prepare_leaderboard_data(
+    selected_categories: List[str] | None = None, dataset_name: str = AGGREGATE_OPTION
+) -> pd.DataFrame:
+    metrics_data = get_metrics_for_display(dataset_name)
+
+    if not metrics_data:
+        return pd.DataFrame(
+            columns=[
+                "Model Snapshot", "Model Developer", "Provider", "Detection Rate (%)",
+                "FPR (%)", "Latency CI 95% (ms)", "Mean Latency (ms)", "Compute Access",
+                "Total Cost", "Cost per 1M units", "Cost per h", "Cost Additional Info",
+                "Model Type", "Execution Info",
+            ]
+        )
+
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    use_overall = (
+        selected_categories is None or len(selected_categories) == 0 or set(selected_categories) == all_categories
+    )
+
+    rows = []
+    for key, data in metrics_data.items():
+        latency_ci = data.get("latency_ci_95", {})
+        cost_info = data.get("cost_info", {})
+
+        latency_ci_str = (
+            f"[{int(round(latency_ci.get('lower', 0), 3) * 1000)}, {int(round(latency_ci.get('upper', 0), 3) * 1000)}]"
+        )
+
+        cost_input = cost_info.get("cost_per_1M_input_tokens", "N/A")
+        cost_output = cost_info.get("cost_per_1M_output_tokens", "N/A")
+        if cost_input == "N/A" or cost_output == "N/A":
+            cost_str = "Unknown"
+        else:
+            cost_str = f"Input: ${round(cost_input, 2)}/1M, Output: ${round(cost_output, 2)}/1M"
+
+        if use_overall:
+            accuracy = data.get("accuracy", 0)
+        else:
+            accuracy_per_category = data.get("accuracy_per_category", {})
+            selected_accuracies = [
+                accuracy_per_category.get(cat, 0) for cat in selected_categories if cat in accuracy_per_category
+            ]
+            accuracy = sum(selected_accuracies) / len(selected_accuracies) if selected_accuracies else 0
+
+        accuracy_pct = accuracy * 100
+
+        fpr = data.get("fpr", 0)
+        if isinstance(fpr, float):
+            fpr = f"{round(fpr * 100, 2):.2f}%"
+
+        cost_per_h_value = data["cost_info"].get("cost_per_h", "Unknown")
+        cost_per_h_str = f"${cost_per_h_value}" if cost_per_h_value != "N/A" else cost_per_h_value
+
+        total_cost_raw = data["cost_info"].get("total_cost", 0)
+        if isinstance(total_cost_raw, (int, float)):
+            total_cost_display = f"${round(total_cost_raw, 2)}"
+            total_cost_sort = total_cost_raw
+        else:
+            total_cost_display = "Unknown"
+            total_cost_sort = float("inf")
+
+        rows.append(
+            {
+                "Model Snapshot": data.get("model_name", ""),
+                "Model URL": data.get("model_url", ""),
+                "Model Developer": data.get("model_developer", ""),
+                "Provider": data.get("provider", ""),
+                "Detection Rate (%)": round(accuracy_pct, 2),
+                "FPR (%)": fpr,
+                "Latency CI 95% (ms)": latency_ci_str,
+                "Mean Latency (ms)": int(round(data.get("mean_latency", 0), 3) * 1000),
+                "Compute Access": data["execution_specifications"].get("type", "Unknown"),
+                "Total Cost": total_cost_display,
+                "Cost per 1M units": cost_str,
+                "Cost per h": cost_per_h_str,
+                "Cost Additional Info": data["cost_info"].get("cost_additional_info", ""),
+                "Model Type": data.get("model_type", "Unknown"),
+                "Execution Info": data["execution_specifications"].get("details", "Unknown"),
+                "_accuracy": accuracy,
+                "_fpr": data.get("fpr", 0),
+                "_mean_latency": int(round(data.get("mean_latency", 0), 3) * 1000),
+                "_total_cost": total_cost_sort,
+            }
+        )
+
+    return pd.DataFrame(rows)
+
+
+# ── Leaderboard HTML ────────────────────────────────────────────────────
+
+def create_leaderboard_html(
+    sort_by: str = "Detection Rate (%)",
+    selected_categories: List[str] | None = None,
+    dataset_name: str = AGGREGATE_OPTION,
+    weights: Dict[str, float] | None = None,
+) -> str:
+    leaderboard_df = prepare_leaderboard_data(selected_categories=selected_categories, dataset_name=dataset_name)
+
+    sort_mapping = {
+        "Detection Rate (%)": "_accuracy",
+        "FPR (%)": "_fpr",
+        "Mean Latency (ms)": "_mean_latency",
+        "Total Cost": "_total_cost",
+    }
+
+    sort_column = sort_mapping.get(sort_by, "_accuracy")
+
+    if sort_by == "Overall Score":
+        sorted_df = sort_by_overall_score(leaderboard_df, weights).copy()
+    elif sort_by == "Detection Rate (%)":
+        sorted_df = leaderboard_df.sort_values(by=sort_column, ascending=False).copy()
+    elif sort_by in ["FPR (%)", "Mean Latency (ms)", "Total Cost"]:
+        sorted_df = leaderboard_df.sort_values(by=[sort_column, "_accuracy"], ascending=[True, False]).copy()
+    else:
+        sorted_df = leaderboard_df.sort_values(by=sort_column, ascending=False).copy()
+
+    sorted_df = sorted_df.reset_index(drop=True)
+
+    html = LEADERBOARD_TABLE_CSS
+    html += """
+    <div class="leaderboard-container">
+        <div class="footnotes-section" style="margin-top: 1.5rem; padding: 1rem; background: #f9fafb; border-radius: 8px; font-size: 13px; color: #374151; line-height: 1.6;">
+            <h4 style="margin-top: 0; margin-bottom: 0.75rem; font-size: 14px; font-weight: 600; color: #111827;">Column Explanations</h4>
+            <p style="margin: 0.5rem 0;"><sup>1</sup> <strong>Model Type:</strong> Shows if the supervisor is a repurposed generalist model or a specialized model made for jailbreak detection.</p>
+            <p style="margin: 0.5rem 0;"><sup>2</sup> <strong>Detection Rate (%):</strong> Accuracy on harmful/jailbreak content.</p>
+            <p style="margin: 0.5rem 0;"><sup>3</sup> <strong>FPR (%):</strong> False Positive Rate on benign content.</p>
+            <p style="margin: 0.5rem 0;"><sup>4</sup> <strong>Compute Access:</strong> Shows how this supervisor was run. Can be 'Local' or 'API'.</p>
+            <p style="margin: 0.5rem 0;"><sup>5</sup> <strong>Total Cost:</strong> The total cost was measured differently for local inference and API supervisors. For local inference, it is total_latency * cost_per_h. For API supervisors it is calculated using the usage metrics and pricing information.</p>
+            <p style="margin: 0.5rem 0;"><sup>6</sup> <strong>Cost per 1M units:</strong> Different systems calculate usage in different units (e.g. characters, tokens, requests). This column shows the cost per 1M measured units.</p>
+            <p style="margin: 0.5rem 0;"><sup>7</sup> <strong>Cost per h:</strong> Only applicable for local inference. This is the pricing information for the GPU pod (see the "Execution Info" column for more details).</p>
+        </div>
+        <table class="leaderboard-table">
+            <thead>
+                <tr>
+                    <th>Rank</th>
+                    <th>Model Snapshot</th>
+                    <th>Model Developer</th>
+                    <th>Provider</th>
+                    <th title="Shows if the supervisor is a repurposed generalist model or a specialized model made for jailbreak detection.">Model Type <sup>1</sup></th>
+                    <th title="Accuracy on harmful/jailbreak content.">Detection Rate (%) <sup>2</sup></th>
+                    <th title="False Positive Rate on benign content.">FPR (%) <sup>3</sup></th>
+                    <th>Latency CI 95% (ms)</th>
+                    <th>Mean Latency (ms)</th>
+                    <th title="Shows how this supervisor was ran. Can be 'Local' or 'API'.">Compute Access <sup>4</sup></th>
+                    <th title="The total cost was measured differently for local inference and API supervisors. For local inference, it is total_latency * cost_per_h. For API supervisors it is calculated using the usage metrics and pricing information.">Total Cost <sup>5</sup></th>
+                    <th title="Different systems calculate usage in different units (e.g. characters, tokens, requests). This column shows the cost per 1M measured units.">Cost per 1M units <sup>6</sup></th>
+                    <th title="Only applicable for local inference. All models were run on RunPod.">Cost per h <sup>7</sup></th>
+                    <th>Cost Additional Info</th>
+                    <th>Execution Info</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+
+    for idx, row in sorted_df.iterrows():
+        rank = idx + 1
+        detection_rate = row["Detection Rate (%)"]
+        fpr = row["FPR (%)"]
+        latency = row["Mean Latency (ms)"]
+        model_type = row["Model Type"]
+        type_class = "type-specialized" if model_type == "specialized" else "type-generalist"
+
+        model_name = escape_html(row["Model Snapshot"])
+        model_url = str(row["Model URL"]) if row["Model URL"] else ""
+        developer = escape_html(row["Model Developer"])
+        provider = escape_html(row["Provider"])
+        compute_access = escape_html(row["Compute Access"])
+        total_cost = escape_html(row["Total Cost"])
+        cost_per_1m = escape_html(row["Cost per 1M units"])
+        cost_per_h = escape_html(row["Cost per h"])
+        cost_add_info = escape_html(row["Cost Additional Info"])
+        latency_ci = escape_html(row["Latency CI 95% (ms)"])
+        exec_info = escape_html(row["Execution Info"])
+
+        if model_url and model_url != "None":
+            model_name_html = f'<a href="{model_url}" target="_blank" rel="noopener noreferrer" style="color: inherit; text-decoration: none; border-bottom: 1px dotted currentColor;">{model_name}</a>'
+        else:
+            model_name_html = model_name
+
+        html += f"""
+                <tr>
+                    <td><span class="rank-badge">{rank}</span></td>
+                    <td class="model-name">{model_name_html}</td>
+                    <td class="model-developer">{developer}</td>
+                    <td><span class="provider-badge">{provider}</span></td>
+                    <td><span class="model-type-badge {type_class}">{model_type.title()}</span></td>
+                    <td class="detection-rate metric-value">{detection_rate:.2f}%</td>
+                    <td class="metric-value">{fpr}</td>
+                    <td>{latency_ci}</td>
+                    <td class="metric-value">{latency:d}</td>
+                    <td>{compute_access}</td>
+                    <td>{total_cost}</td>
+                    <td style="font-size: 12px;">{cost_per_1m}</td>
+                    <td>{cost_per_h}</td>
+                    <td style="font-size: 12px; color: #6b7280;">{cost_add_info}</td>
+                    <td style="font-size: 12px; color: #6b7280;">{exec_info}</td>
+                </tr>
+        """
+
+    html += """
+            </tbody>
+        </table>
+    </div>
+    """
+    html += LEADERBOARD_DARK_MODE_CSS
+    return html
+
+
+# ── Category tables ─────────────────────────────────────────────────────
+
+def create_category_accuracy_table_html(
+    selected_models: List[str] | None = None, dataset_name: str = AGGREGATE_OPTION
+) -> str:
+    metrics_data = get_metrics_for_display(dataset_name)
+
+    if selected_models is None or len(selected_models) == 0:
+        selected_models = [data.get("model_name", "") for data in metrics_data.values()]
+
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    category_order = list(CATEGORY_LABELS.keys())
+    sorted_categories = [cat for cat in category_order if cat in all_categories]
+    sorted_categories.extend([cat for cat in all_categories if cat not in category_order])
+
+    html = CATEGORY_TABLE_CSS
+    html += '<div class="category-table-container"><table class="category-table">'
+
+    html += "<thead><tr><th>Model</th>"
+    for category in sorted_categories:
+        category_label = CATEGORY_LABELS.get(category, category.title())
+        html += f"<th>{category_label}</th>"
+    html += "</tr></thead>"
+
+    html += "<tbody>"
+    for key, data in metrics_data.items():
+        model_name = data.get("model_name", "")
+        if model_name not in selected_models:
+            continue
+
+        html += f"<tr><td>{escape_html(model_name)}</td>"
+        accuracy_per_category = data.get("accuracy_per_category", {})
+        for category in sorted_categories:
+            accuracy = accuracy_per_category.get(category, 0)
+            acc_pct = round(accuracy * 100, 1)
+            bg_color = get_color_for_accuracy(accuracy)
+            html += f'<td style="background-color: {bg_color};">{acc_pct}%</td>'
+        html += "</tr>"
+
+    html += "</tbody></table></div>"
+    return html
+
+
+def create_category_overview_html(selected_models: List[str] | None = None, metric: str = "Detection Rate") -> str:
+    datasets = get_available_datasets()
+    metric_key = "accuracy" if metric == "Detection Rate" else "fpr"
+
+    dataset_metrics: Dict[str, Dict[str, Any]] = {}
+    all_models: List[str] = []
+    for ds in datasets:
+        ds_data = filter_metrics_by_dataset(ds)
+        values_by_model: Dict[str, Any] = {}
+        for data in ds_data.values():
+            model = data.get("model_name", "")
+            values_by_model[model] = data.get(metric_key, 0)
+            if model not in all_models:
+                all_models.append(model)
+        dataset_metrics[ds] = values_by_model
+
+    all_models.sort()
+    if selected_models is not None and len(selected_models) > 0:
+        all_models = [m for m in all_models if m in selected_models]
+
+    html = CATEGORY_TABLE_CSS
+    html += '<div class="category-table-container"><table class="category-table">'
+
+    html += "<thead><tr><th>Model</th>"
+    for ds in datasets:
+        ds_label = DATASET_LABELS.get(ds, ds)
+        html += f"<th>{ds_label}</th>"
+    html += "</tr></thead>"
+
+    html += "<tbody>"
+    for model in all_models:
+        html += f"<tr><td>{escape_html(model)}</td>"
+        for ds in datasets:
+            val = dataset_metrics[ds].get(model)
+            if val is None or val == "N/A" or not isinstance(val, (int, float)):
+                html += '<td style="text-align: center; color: #9ca3af;">N/A</td>'
+            else:
+                val_pct = round(val * 100, 1)
+                if metric == "Detection Rate":
+                    bg_color = get_color_for_accuracy(val)
+                else:
+                    bg_color = get_color_for_accuracy(1.0 - val)
+                html += f'<td style="background-color: {bg_color};">{val_pct}%</td>'
+        html += "</tr>"
+
+    html += "</tbody></table></div>"
+    return html
+
+
+# ── Helpers ─────────────────────────────────────────────────────────────
+
+def get_available_categories(dataset_name: str = AGGREGATE_OPTION) -> List[tuple]:
+    metrics_data = get_metrics_for_display(dataset_name)
+    all_categories = set()
+    for data in metrics_data.values():
+        all_categories.update(data.get("accuracy_per_category", {}).keys())
+
+    category_order = list(CATEGORY_LABELS.keys())
+    sorted_categories = [cat for cat in category_order if cat in all_categories]
+    sorted_categories.extend([cat for cat in all_categories if cat not in category_order])
+    return [(CATEGORY_LABELS.get(cat, cat.title()), cat) for cat in sorted_categories]
+
+
+def get_available_models(dataset_name: str = AGGREGATE_OPTION) -> List[str]:
+    metrics_data = get_metrics_for_display(dataset_name)
+    return sorted(set(data.get("model_name", "") for data in metrics_data.values()))
+
+
+# ── Pareto ──────────────────────────────────────────────────────────────
+
+_PARETO_METRIC_MAP = {
+    "Detection Rate in %": ("_accuracy", False),
+    "FPR in %": ("_fpr", True),
+    "Mean Latency in ms": ("_mean_latency", True),
+    "Total Cost in USD": ("_total_cost", True),
+}
+
+
+def create_pareto_plot_interactive(x_metric, y_metric, dataset_name=AGGREGATE_OPTION):
+    df = prepare_leaderboard_data(None, dataset_name)
+    return build_pareto_figure(df, x_metric, y_metric, _PARETO_METRIC_MAP)
+
+
+# ── Tab builder ─────────────────────────────────────────────────────────
+
+def build_jailbreak_tab():
+    """Build the three sub-tabs (Leaderboard, Category Performance, Pareto Frontier)
+    for the jailbreak detection leaderboard."""
+
+    dataset_choices = get_dataset_choices()
+    default_dataset = AGGREGATE_OPTION
+
+    with gr.Tabs():
+        # ── Leaderboard sub-tab ─────────────────────────────────────
+        with gr.Tab("Leaderboard"):
+            with gr.Column():
+                gr.Markdown("### Interactive Leaderboard")
+                gr.Markdown(
+                    "Sort the leaderboard by different metrics to compare model performance. "
+                    "Use the dropdowns below to change the sorting order and dataset. "
+                    "Select specific categories to see detection rates calculated only for those categories."
+                )
+
+                with gr.Row():
+                    dataset_selector_lb = gr.Dropdown(
+                        choices=dataset_choices,
+                        value=default_dataset,
+                        label="Dataset",
+                        interactive=True,
+                        info="Select a specific dataset or aggregate across all datasets.",
+                    )
+                    sort_metric = gr.Dropdown(
+                        choices=[
+                            "Overall Score",
+                            "Detection Rate (%)",
+                            "FPR (%)",
+                            "Mean Latency (ms)",
+                            "Total Cost",
+                        ],
+                        value="Overall Score",
+                        label="Sort by Metric",
+                        interactive=True,
+                        info="Higher detection rate is better. Lower FPR and latency are better.",
+                    )
+
+                with gr.Group(visible=True) as weight_row:
+                    with gr.Row():
+                        weight_detection = gr.Number(
+                            value=DEFAULT_WEIGHTS["Detection Rate (%)"],
+                            label="Detection Rate weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_fpr = gr.Number(
+                            value=DEFAULT_WEIGHTS["FPR (%)"],
+                            label="FPR weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_latency = gr.Number(
+                            value=DEFAULT_WEIGHTS["Mean Latency (ms)"],
+                            label="Latency weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                        weight_cost = gr.Number(
+                            value=DEFAULT_WEIGHTS["Total Cost"],
+                            label="Cost weight",
+                            minimum=0.0,
+                            interactive=True,
+                        )
+                    weight_bar = gr.HTML(value=create_weight_bar_html(DEFAULT_WEIGHTS))
+                weight_inputs = [weight_detection, weight_fpr, weight_latency, weight_cost]
+
+                dataset_info_html = gr.HTML(value="")
+
+                with gr.Row():
+                    category_selector = gr.CheckboxGroup(
+                        choices=[],
+                        value=[],
+                        label="Select Categories for Detection Rate",
+                        interactive=False,
+                        visible=False,
+                        info="Select categories to calculate detection rate. Only available for individual datasets.",
+                    )
+
+                leaderboard_html = gr.HTML(
+                    value=create_leaderboard_html(
+                        sort_by="Overall Score", dataset_name=default_dataset, weights=DEFAULT_WEIGHTS
+                    ),
+                    label="Model Rankings",
+                )
+
+                def update_leaderboard(sort_by, selected_categories, dataset_name, w_det, w_fpr, w_lat, w_cost):
+                    weights = build_weights(w_det, w_fpr, w_lat, w_cost)
+                    html = create_leaderboard_html(
+                        sort_by=sort_by,
+                        selected_categories=selected_categories,
+                        dataset_name=dataset_name,
+                        weights=weights,
+                    )
+                    bar = create_weight_bar_html(weights)
+                    is_aggregate = dataset_name == AGGREGATE_OPTION
+                    info = "" if is_aggregate else create_dataset_info_html(dataset_name)
+                    return html, bar, info
+
+                def on_sort_change(sort_by, selected_categories, dataset_name, w_det, w_fpr, w_lat, w_cost):
+                    weights = build_weights(w_det, w_fpr, w_lat, w_cost)
+                    new_html = create_leaderboard_html(
+                        sort_by=sort_by,
+                        selected_categories=selected_categories,
+                        dataset_name=dataset_name,
+                        weights=weights,
+                    )
+                    visible = sort_by == "Overall Score"
+                    bar = create_weight_bar_html(weights)
+                    is_aggregate = dataset_name == AGGREGATE_OPTION
+                    info = "" if is_aggregate else create_dataset_info_html(dataset_name)
+                    return (
+                        gr.update(visible=visible),
+                        new_html,
+                        bar,
+                        info,
+                    )
+
+                def update_categories_on_dataset_change(dataset_name, sort_by, w_det, w_fpr, w_lat, w_cost):
+                    weights = build_weights(w_det, w_fpr, w_lat, w_cost)
+                    new_html = create_leaderboard_html(sort_by=sort_by, dataset_name=dataset_name, weights=weights)
+                    is_aggregate = dataset_name == AGGREGATE_OPTION
+                    info = "" if is_aggregate else create_dataset_info_html(dataset_name)
+                    if is_aggregate:
+                        return (
+                            gr.update(choices=[], value=[], interactive=False, visible=False),
+                            new_html,
+                            info,
+                        )
+                    else:
+                        new_choices = get_available_categories(dataset_name)
+                        new_values = [cat for _, cat in new_choices]
+                        return (
+                            gr.update(choices=new_choices, value=new_values, interactive=True, visible=True),
+                            new_html,
+                            info,
+                        )
+
+                all_inputs = [sort_metric, category_selector, dataset_selector_lb] + weight_inputs
+
+                dataset_selector_lb.change(
+                    fn=update_categories_on_dataset_change,
+                    inputs=[dataset_selector_lb, sort_metric] + weight_inputs,
+                    outputs=[category_selector, leaderboard_html, dataset_info_html],
+                )
+
+                sort_metric.change(
+                    fn=on_sort_change,
+                    inputs=all_inputs,
+                    outputs=[weight_row, leaderboard_html, weight_bar, dataset_info_html],
+                )
+
+                category_selector.change(
+                    fn=update_leaderboard,
+                    inputs=all_inputs,
+                    outputs=[leaderboard_html, weight_bar, dataset_info_html],
+                )
+
+                for w_input in weight_inputs:
+                    w_input.change(
+                        fn=update_leaderboard,
+                        inputs=all_inputs,
+                        outputs=[leaderboard_html, weight_bar, dataset_info_html],
+                    )
+
+        # ── Category Performance sub-tab ────────────────────────────
+        with gr.Tab("Category Performance"):
+            with gr.Column():
+                gr.Markdown("### Performance per Category by Model")
+                gr.Markdown(
+                    "Compare how different models perform across harm categories. "
+                    "Each cell shows the detection rate (accuracy) for that model-category combination. "
+                    "Cell colors indicate performance: darker green = higher accuracy (closer to 100%), lighter colors = lower accuracy."
+                )
+
+                dataset_choices_cat = [(OVERVIEW_OPTION, OVERVIEW_OPTION)] + [
+                    c
+                    for c in dataset_choices
+                    if c[1] != AGGREGATE_OPTION and len(get_available_categories(c[1])) > 1
+                ]
+                cat_default_dataset = OVERVIEW_OPTION
+
+                with gr.Row():
+                    dataset_selector_cat = gr.Dropdown(
+                        choices=dataset_choices_cat,
+                        value=cat_default_dataset,
+                        label="Dataset",
+                        interactive=True,
+                        info="Select a dataset or Overview. Datasets with only one category are excluded.",
+                    )
+                    metric_selector_cat = gr.Dropdown(
+                        choices=["Detection Rate", "FPR"],
+                        value="Detection Rate",
+                        label="Metric",
+                        interactive=True,
+                        visible=True,
+                        info="Choose which metric to display. Only changeable in Overview mode.",
+                    )
+
+                dataset_info_cat = gr.HTML(value="")
+
+                initial_model_choices = get_available_models(AGGREGATE_OPTION)
+                model_selector = gr.CheckboxGroup(
+                    choices=initial_model_choices,
+                    value=initial_model_choices,
+                    label="Select Models to Compare",
+                    interactive=True,
+                    info="Select one or more models to compare their performance across categories.",
+                )
+
+                category_table_html = gr.HTML(
+                    value=create_category_overview_html(initial_model_choices),
+                    label="Category Accuracy Comparison",
+                )
+
+                def update_cat_table(models, dataset_name, metric):
+                    if dataset_name == OVERVIEW_OPTION:
+                        return create_category_overview_html(models, metric=metric), ""
+                    table = create_category_accuracy_table_html(models, dataset_name=dataset_name)
+                    info = create_dataset_info_html(dataset_name)
+                    return table, info
+
+                def update_models_on_dataset_change(dataset_name, metric):
+                    is_overview = dataset_name == OVERVIEW_OPTION
+                    if is_overview:
+                        new_models = get_available_models(AGGREGATE_OPTION)
+                        new_html = create_category_overview_html(new_models, metric=metric)
+                    else:
+                        new_models = get_available_models(dataset_name)
+                        new_html = create_category_accuracy_table_html(new_models, dataset_name=dataset_name)
+                    info = "" if is_overview else create_dataset_info_html(dataset_name)
+                    if is_overview:
+                        metric_update = gr.update(value=metric, interactive=True)
+                    else:
+                        metric_update = gr.update(value="Detection Rate", interactive=False)
+                    return (
+                        gr.update(choices=new_models, value=new_models),
+                        new_html,
+                        metric_update,
+                        info,
+                    )
+
+                dataset_selector_cat.change(
+                    fn=update_models_on_dataset_change,
+                    inputs=[dataset_selector_cat, metric_selector_cat],
+                    outputs=[model_selector, category_table_html, metric_selector_cat, dataset_info_cat],
+                )
+
+                model_selector.change(
+                    fn=update_cat_table,
+                    inputs=[model_selector, dataset_selector_cat, metric_selector_cat],
+                    outputs=[category_table_html, dataset_info_cat],
+                )
+
+                metric_selector_cat.change(
+                    fn=update_cat_table,
+                    inputs=[model_selector, dataset_selector_cat, metric_selector_cat],
+                    outputs=[category_table_html, dataset_info_cat],
+                )
+
+        # ── Pareto Frontier sub-tab ─────────────────────────────────
+        with gr.Tab("Pareto Frontier"):
+            with gr.Column():
+                gr.Markdown("### Pareto Frontier Analysis")
+                gr.Markdown(
+                    "Visualize trade-offs between different metrics. Models on the Pareto frontier "
+                    "represent optimal trade-offs where improving one metric would require sacrificing another. "
+                    "Hover over points to see the model name."
+                )
+
+                with gr.Row():
+                    dataset_selector_pareto = gr.Dropdown(
+                        choices=dataset_choices,
+                        value=default_dataset,
+                        label="Dataset",
+                        interactive=True,
+                        info="Select a specific dataset or aggregate across all datasets.",
+                    )
+
+                with gr.Row():
+                    pareto_x_metric = gr.Dropdown(
+                        choices=["Detection Rate in %", "FPR in %", "Mean Latency in ms", "Total Cost in USD"],
+                        value="Detection Rate in %",
+                        label="X-Axis Metric",
+                        interactive=True,
+                    )
+                    pareto_y_metric = gr.Dropdown(
+                        choices=["Detection Rate in %", "FPR in %", "Mean Latency in ms", "Total Cost in USD"],
+                        value="FPR in %",
+                        label="Y-Axis Metric",
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    gr.HTML('<div style="display:flex; justify-content:center;">', visible=False)
+                    pareto_plot = gr.Plot(
+                        value=create_pareto_plot_interactive("Detection Rate in %", "FPR in %", default_dataset)
+                    )
+                    gr.HTML("</div>", visible=False)
+
+                def _update_pareto(x_metric, y_metric, dataset_name):
+                    if x_metric == y_metric:
+                        return create_empty_pareto_figure()
+                    return create_pareto_plot_interactive(x_metric, y_metric, dataset_name)
+
+                pareto_x_metric.change(
+                    fn=_update_pareto,
+                    inputs=[pareto_x_metric, pareto_y_metric, dataset_selector_pareto],
+                    outputs=pareto_plot,
+                )
+                pareto_y_metric.change(
+                    fn=_update_pareto,
+                    inputs=[pareto_x_metric, pareto_y_metric, dataset_selector_pareto],
+                    outputs=pareto_plot,
+                )
+                dataset_selector_pareto.change(
+                    fn=_update_pareto,
+                    inputs=[pareto_x_metric, pareto_y_metric, dataset_selector_pareto],
+                    outputs=pareto_plot,
+                )

requirements.txt

@@ -0,0 +1,4 @@
+gradio>=4.0.0
+pandas>=2.0.0
+plotly>=5.0.0
+scipy>=1.16.0

shared.py

@@ -0,0 +1,869 @@
+import base64
+from pathlib import Path
+from typing import Dict
+
+import gradio as gr
+import numpy as np
+import plotly.graph_objects as go
+from scipy.interpolate import UnivariateSpline
+from scipy.optimize import curve_fit
+
+
+def get_color_for_accuracy(accuracy: float) -> str:
+    """Get background color based on accuracy (0-1). Closer to 1.0 = greener."""
+    acc_pct = accuracy * 100
+
+    if acc_pct >= 90:
+        green = 200 + int((acc_pct - 90) * 5.5)
+        return f"rgb({255 - green}, {min(255, green)}, {100})"
+    elif acc_pct >= 70:
+        green = 150 + int((acc_pct - 70) * 2.5)
+        return f"rgb({255 - green}, {min(255, green)}, {50})"
+    elif acc_pct >= 50:
+        yellow = 200 + int((acc_pct - 50) * 2.75)
+        return f"rgb({255}, {min(255, yellow)}, {100})"
+    elif acc_pct >= 30:
+        red = 255 - int((acc_pct - 30) * 2.75)
+        return f"rgb({red}, {150 + int((acc_pct - 30) * 2.75)}, {50})"
+    else:
+        red = 255
+        green = 100 + int(acc_pct * 1.67)
+        return f"rgb({red}, {green}, {100})"
+
+
+def _short_label(name: str, max_len: int = 35) -> str:
+    return name if len(name) <= max_len else name[: max_len - 1] + "\u2026"
+
+
+def escape_html(s: str) -> str:
+    return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+# ── Weight / Overall-Score helpers ──────────────────────────────────────
+
+DEFAULT_WEIGHTS = {
+    "Detection Rate (%)": 1.0,
+    "FPR (%)": 1.0,
+    "Mean Latency (ms)": 1.0,
+    "Total Cost": 1.0,
+}
+
+_RANK_CONFIG = {
+    "Detection Rate (%)": ("_accuracy", False),
+    "FPR (%)": ("_fpr", True),
+    "Mean Latency (ms)": ("_mean_latency", True),
+    "Total Cost": ("_total_cost", True),
+}
+
+_WEIGHT_COLORS = {
+    "Detection Rate (%)": "#22c55e",
+    "FPR (%)": "#ef4444",
+    "Mean Latency (ms)": "#3b82f6",
+    "Total Cost": "#f59e0b",
+}
+
+_WEIGHT_SHORT_LABELS = {
+    "Detection Rate (%)": "Detection",
+    "FPR (%)": "FPR",
+    "Mean Latency (ms)": "Latency",
+    "Total Cost": "Cost",
+}
+
+
+def sort_by_overall_score(df, weights: Dict[str, float] | None = None):
+    """Sort dataframe by weighted average rank across metrics."""
+    if weights is None:
+        weights = DEFAULT_WEIGHTS
+
+    rank_cols = []
+    for metric, (col, ascending) in _RANK_CONFIG.items():
+        w = weights.get(metric, 0.0)
+        if w == 0.0:
+            continue
+        rank_col = f"_rank_{col}"
+        df[rank_col] = df[col].rank(ascending=ascending, method="min")
+        rank_cols.append((rank_col, w))
+
+    if not rank_cols:
+        return df
+
+    df["_overall_score"] = sum(df[rc] * w for rc, w in rank_cols) / sum(w for _, w in rank_cols)
+    df = df.sort_values(by="_overall_score", ascending=True)
+    return df
+
+
+def create_weight_bar_html(weights: Dict[str, float] | None = None) -> str:
+    """Create an HTML horizontal stacked bar showing the weight distribution."""
+    if weights is None:
+        weights = DEFAULT_WEIGHTS
+
+    total = sum(max(w, 0) for w in weights.values())
+    if total == 0:
+        return '<div style="height:28px; background:#e5e7eb; border-radius:6px;"></div>'
+
+    segments = ""
+    for metric, w in weights.items():
+        if w <= 0:
+            continue
+        pct = w / total * 100
+        color = _WEIGHT_COLORS.get(metric, "#6b7280")
+        label = _WEIGHT_SHORT_LABELS.get(metric, metric)
+        text = f"{label} {pct:.0f}%" if pct >= 12 else f"{pct:.0f}%" if pct >= 5 else ""
+        segments += (
+            f'<div style="width:{pct}%; background:{color}; color:white; display:flex;'
+            f" align-items:center; justify-content:center; font-size:12px; font-weight:600;"
+            f' white-space:nowrap; overflow:hidden; text-overflow:ellipsis; padding:0 4px;">'
+            f"{text}</div>"
+        )
+
+    return (
+        f'<div style="display:flex; height:28px; border-radius:6px; overflow:hidden;'
+        f' box-shadow:0 1px 3px rgba(0,0,0,0.1);">{segments}</div>'
+    )
+
+
+def build_weights(w_det, w_fpr, w_lat, w_cost):
+    return {
+        "Detection Rate (%)": w_det if w_det is not None else 0.0,
+        "FPR (%)": w_fpr if w_fpr is not None else 0.0,
+        "Mean Latency (ms)": w_lat if w_lat is not None else 0.0,
+        "Total Cost": w_cost if w_cost is not None else 0.0,
+    }
+
+
+# ── Pareto helpers ──────────────────────────────────────────────────────
+
+def create_empty_pareto_figure(message: str = "Please select different metrics for X and Y axes"):
+    fig = go.Figure()
+    fig.add_annotation(
+        text=message,
+        x=0.5,
+        y=0.5,
+        showarrow=False,
+        font={"size": 16},
+        xref="paper",
+        yref="paper",
+    )
+    fig.update_layout(
+        xaxis={"visible": False},
+        yaxis={"visible": False},
+        template="plotly_white",
+    )
+    return fig
+
+
+def build_pareto_figure(df, x_metric, y_metric, metric_map):
+    """Build a Pareto frontier plot from a prepared leaderboard dataframe.
+
+    Args:
+        df: DataFrame with columns including Model Snapshot and internal sort columns.
+        x_metric: Display name of x metric (key in metric_map).
+        y_metric: Display name of y metric (key in metric_map).
+        metric_map: Dict mapping display metric name to (column_name, minimize_bool).
+    """
+    if df.empty:
+        return create_empty_pareto_figure("No data available for the selected dataset")
+
+    x_col, x_minimize = metric_map[x_metric]
+    y_col, y_minimize = metric_map[y_metric]
+
+    x = df[x_col]
+    y = df[y_col]
+
+    # Pareto detection
+    is_pareto = []
+    for i in range(len(df)):
+        dominated = False
+        for j in range(len(df)):
+            if i == j:
+                continue
+            better_x = (x.iloc[j] <= x.iloc[i]) if x_minimize else (x.iloc[j] >= x.iloc[i])
+            better_y = (y.iloc[j] <= y.iloc[i]) if y_minimize else (y.iloc[j] >= y.iloc[i])
+            strictly = (x.iloc[j] != x.iloc[i]) or (y.iloc[j] != y.iloc[i])
+            if better_x and better_y and strictly:
+                dominated = True
+                break
+        is_pareto.append(not dominated)
+
+    df["pareto"] = is_pareto
+
+    fig = go.Figure()
+
+    # Dominated points
+    fig.add_trace(
+        go.Scatter(
+            x=df.loc[~df.pareto, x_col],
+            y=df.loc[~df.pareto, y_col],
+            mode="markers",
+            name="Dominated",
+            marker=dict(size=10, color="gray"),
+            customdata=[
+                [model, xv, yv]
+                for model, xv, yv in zip(
+                    df.loc[~df.pareto, "Model Snapshot"], df.loc[~df.pareto, x_col], df.loc[~df.pareto, y_col]
+                )
+            ],
+            hovertemplate=(
+                "<b>%{customdata[0]}</b><br>"
+                f"{x_metric}:"
+                " %{customdata[1]:.3f}<br>"
+                f"{y_metric}:"
+                " %{customdata[2]:.3f}"
+                "<extra></extra>"
+            ),
+        )
+    )
+
+    pareto_df = df[df.pareto].reset_index(drop=True)
+    pareto_df_sorted = pareto_df.sort_values(by=x_col).reset_index(drop=True)
+
+    # Pareto frontier curve
+    if len(pareto_df_sorted) >= 2:
+        try:
+            x_pareto = np.array(pareto_df_sorted[x_col].tolist(), dtype=float)
+            y_pareto = np.array(pareto_df_sorted[y_col].tolist(), dtype=float)
+
+            if x_minimize != y_minimize:
+                def pareto_func(xv, a, b):
+                    return a * np.power(np.clip(xv, 1e-10, None), b)
+            else:
+                def pareto_func(xv, a, b):
+                    return a * np.power(np.clip(1 - xv, 1e-10, None), b)
+
+            try:
+                popt, _ = curve_fit(pareto_func, x_pareto, y_pareto, p0=[1.0, 1.0], maxfev=5000)
+
+                x_is_bounded = "Detection Rate" in x_metric or "FPR" in x_metric
+                y_is_bounded = "Detection Rate" in y_metric or "FPR" in y_metric
+
+                if x_is_bounded:
+                    x_curve = np.linspace(0, 1, 500)
+                else:
+                    x_all = np.array(df[x_col].tolist(), dtype=float)
+                    x_min_val = max(0, float(np.min(x_all)))
+                    x_max_val = float(np.max(x_all)) * 1.2
+                    x_curve = np.linspace(x_min_val, x_max_val, 500)
+
+                y_curve = pareto_func(x_curve, *popt)
+
+                if y_is_bounded:
+                    valid_indices = np.where((y_curve >= 0) & (y_curve <= 1))[0]
+                else:
+                    valid_indices = np.where((y_curve >= 0) & (y_curve <= max(df[y_col])))[0]
+
+                if len(valid_indices) > 0:
+                    x_curve_valid = x_curve[valid_indices]
+                    y_curve_valid = y_curve[valid_indices]
+
+                    fig.add_trace(
+                        go.Scatter(
+                            x=x_curve_valid.tolist(),
+                            y=y_curve_valid.tolist(),
+                            mode="lines",
+                            name="Pareto Frontier Curve",
+                            line={"color": "red", "width": 3},
+                            showlegend=True,
+                            hoverinfo="skip",
+                        )
+                    )
+            except Exception:
+                k = min(3, len(x_pareto) - 1)
+                spline = UnivariateSpline(x_pareto, y_pareto, k=k, s=0)
+                x_curve = np.linspace(0, 1, 500)
+                y_curve = np.clip(spline(x_curve), 0, 1)
+
+                fig.add_trace(
+                    go.Scatter(
+                        x=x_curve.tolist(),
+                        y=y_curve.tolist(),
+                        mode="lines",
+                        name="Pareto Frontier Curve",
+                        line={"color": "red", "width": 3},
+                        showlegend=True,
+                        hoverinfo="skip",
+                    )
+                )
+        except Exception as e:
+            print(f"Warning: Could not create Pareto frontier curve: {e}")
+
+    # Pareto points with labels
+    text_positions = ["top center", "bottom center"]
+    text_pos = [text_positions[i % 2] for i in range(len(pareto_df))]
+
+    fig.add_trace(
+        go.Scatter(
+            x=pareto_df[x_col],
+            y=pareto_df[y_col],
+            mode="markers+text",
+            name="Pareto Frontier",
+            marker=dict(size=14, color="red"),
+            text=[_short_label(m) for m in pareto_df["Model Snapshot"]],
+            textposition=text_pos,
+            customdata=[
+                [model, xv, yv]
+                for model, xv, yv in zip(pareto_df["Model Snapshot"], pareto_df[x_col], pareto_df[y_col])
+            ],
+            hovertemplate=(
+                "<b>%{customdata[0]}</b><br>"
+                f"{x_metric}:"
+                " %{customdata[1]:.3f}<br>"
+                f"{y_metric}:"
+                " %{customdata[2]:.3f}"
+                "<extra></extra>"
+            ),
+        )
+    )
+
+    fig.update_layout(
+        autosize=True,
+        height=800,
+        xaxis_title=x_metric,
+        yaxis_title=y_metric,
+        hovermode="closest",
+        template="plotly_white",
+    )
+
+    if x_minimize:
+        fig.update_xaxes(autorange="reversed")
+    if y_minimize:
+        fig.update_yaxes(autorange="reversed")
+
+    return fig
+
+
+# ── Footer ──────────────────────────────────────────────────────────────
+
+def build_footer():
+    logo_path = Path(__file__).parent / "cesia_logo.png"
+    if logo_path.exists():
+        with open(logo_path, "rb") as logo_file:
+            logo_data = base64.b64encode(logo_file.read()).decode()
+            logo_src = f"data:image/png;base64,{logo_data}"
+    else:
+        logo_src = ""
+
+    span_display = "display:inline;" if not logo_src else "display:none;"
+    footer_html = f"""
+    <div class="footer-section">
+        <div class="footer-content">
+            <p class="footer-text">This project is led by</p>
+            {"<img src='" + logo_src + "' alt='CeSIA Logo' class='footer-logo' style='height:36px;vertical-align:middle;margin-bottom:4px;'>" if logo_src else ""}
+            <span style="{span_display} color: #4338ca; font-weight: 600;">Centre pour la S&eacute;curit&eacute; de l'IA (CeSIA)</span>
+        </div>
+    </div>
+    """
+    gr.HTML(footer_html)
+
+
+# ── Global CSS ──────────────────────────────────────────────────────────
+
+GLOBAL_CSS = """
+<style>
+    .gradio-container {
+        max-width: 100% !important;
+        width: 100% !important;
+        margin: 0 !important;
+        padding-left: 0 !important;
+        padding-right: 0 !important;
+    }
+
+    .contain {
+        max-width: 100% !important;
+        width: 100% !important;
+    }
+
+    .block {
+        max-width: 100% !important;
+    }
+    .gradio-container {
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+    }
+
+    h1 {
+        font-size: 2.5rem;
+        font-weight: 700;
+        color: #111827;
+        margin-bottom: 1rem;
+        letter-spacing: -0.02em;
+    }
+
+    h2 {
+        font-size: 1.75rem;
+        font-weight: 600;
+        color: #111827;
+        margin-top: 2rem;
+        margin-bottom: 1rem;
+    }
+
+    h3 {
+        font-size: 1.25rem;
+        font-weight: 600;
+        color: #374151;
+        margin-top: 1.5rem;
+        margin-bottom: 0.75rem;
+    }
+
+    @media (max-width: 768px) {
+        h1 {
+            font-size: 1.75rem;
+        }
+        h2 {
+            font-size: 1.5rem;
+            margin-top: 1.5rem;
+        }
+        h3 {
+            font-size: 1.125rem;
+        }
+    }
+
+    @media (prefers-color-scheme: dark) {
+        .gradio-container {
+            background: #1a1a1a !important;
+            color: #e5e7eb !important;
+        }
+        h1, h2, h3 {
+            color: #f3f4f6 !important;
+        }
+    }
+
+    .footer-section {
+        text-align: center;
+        margin-top: 3rem;
+        margin-bottom: 2rem;
+        padding: 1.5rem 0;
+    }
+
+    .footer-content {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        gap: 0.5rem;
+        flex-wrap: wrap;
+    }
+
+    .footer-text {
+        margin: 0;
+        display: inline;
+    }
+
+    .footer-logo {
+        display: inline-block;
+        vertical-align: middle;
+    }
+
+    .gradio-tabs {
+        margin-top: 2rem;
+    }
+
+    .gradio-tab-nav {
+        border-bottom: 2px solid #e5e7eb;
+    }
+
+    .gradio-tab-nav button {
+        font-size: 1.1rem;
+        font-weight: 600;
+        padding: 1rem 1.5rem;
+        transition: all 0.2s ease;
+    }
+
+    .gradio-tab-nav button.selected {
+        border-bottom: 3px solid #4338ca;
+        color: #4338ca;
+    }
+
+    .center-column {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+    }
+
+    .center-column .plotly-graph-div {
+        width: auto !important;
+        max-width: 100%;
+        margin-left: auto !important;
+        margin-right: auto !important;
+    }
+</style>
+"""
+
+
+# ── Shared CSS snippets for leaderboard & category tables ───────────────
+
+LEADERBOARD_TABLE_CSS = """
+<style>
+    .leaderboard-container {
+        width: 100%;
+        overflow-x: auto;
+        margin: 20px 0;
+        -webkit-overflow-scrolling: touch;
+    }
+    .leaderboard-table {
+        width: 100%;
+        min-width: 1300px;
+        border-collapse: collapse;
+        background: white;
+        box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+        border-radius: 8px;
+        overflow: hidden;
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+    }
+    @media (max-width: 1024px) {
+        .leaderboard-table {
+            min-width: 1000px;
+            font-size: 13px;
+        }
+        .leaderboard-table th,
+        .leaderboard-table td {
+            padding: 12px 10px;
+        }
+        .rank-badge {
+            width: 28px;
+            height: 28px;
+            line-height: 28px;
+            font-size: 12px;
+        }
+    }
+
+    @media (max-width: 768px) {
+        .leaderboard-container {
+            margin: 10px 0;
+        }
+        .leaderboard-table {
+            min-width: 900px;
+            font-size: 11px;
+        }
+        .leaderboard-table th {
+            padding: 10px 8px;
+            font-size: 10px;
+        }
+        .leaderboard-table td {
+            padding: 8px 6px;
+            font-size: 11px;
+        }
+        .rank-badge {
+            width: 24px;
+            height: 24px;
+            line-height: 24px;
+            font-size: 11px;
+        }
+        .provider-badge,
+        .model-type-badge {
+            padding: 3px 8px;
+            font-size: 10px;
+        }
+        .detection-rate {
+            font-size: 13px;
+        }
+    }
+
+    @media (max-width: 480px) {
+        .leaderboard-table {
+            min-width: 800px;
+            font-size: 10px;
+        }
+        .leaderboard-table th,
+        .leaderboard-table td {
+            padding: 6px 4px;
+        }
+    }
+    .leaderboard-table thead {
+        background: #e5e7eb;
+        color: #111827;
+    }
+    .leaderboard-table th {
+        padding: 16px 12px;
+        text-align: left;
+        font-weight: 600;
+        font-size: 13px;
+        text-transform: uppercase;
+        letter-spacing: 0.5px;
+        border-bottom: 2px solid #d1d5db;
+    }
+    .leaderboard-table th:first-child {
+        padding-left: 20px;
+    }
+    .leaderboard-table th:last-child {
+        padding-right: 20px;
+    }
+    .leaderboard-table tbody tr {
+        border-bottom: 1px solid #e5e7eb;
+        transition: all 0.2s ease;
+    }
+    .leaderboard-table tbody tr:hover {
+        background-color: #f8fafc;
+        transform: scale(1.01);
+        box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+    }
+    .leaderboard-table tbody tr:last-child {
+        border-bottom: none;
+    }
+    .leaderboard-table td {
+        padding: 14px 12px;
+        font-size: 14px;
+        color: #374151;
+    }
+    .leaderboard-table td:first-child {
+        padding-left: 20px;
+        font-weight: 600;
+        color: #111827;
+    }
+    .leaderboard-table td:last-child {
+        padding-right: 20px;
+    }
+    .rank-badge {
+        display: inline-block;
+        width: 32px;
+        height: 32px;
+        line-height: 32px;
+        text-align: center;
+        border-radius: 50%;
+        font-weight: 700;
+        font-size: 14px;
+        background: #f3f4f6;
+        color: #6b7280;
+    }
+    .metric-value {
+        font-weight: 600;
+    }
+    .detection-rate {
+        font-weight: 700;
+        font-size: 15px;
+    }
+    .model-name {
+        font-weight: 600;
+        color: #1f2937;
+    }
+    .model-name a:hover {
+        color: #4338ca !important;
+        border-bottom-style: solid !important;
+    }
+    .model-developer {
+        color: #6b7280;
+        font-size: 13px;
+    }
+    .provider-badge {
+        display: inline-block;
+        padding: 4px 10px;
+        border-radius: 12px;
+        font-size: 12px;
+        font-weight: 500;
+        background: #e0e7ff;
+        color: #4338ca;
+    }
+    .model-type-badge {
+        display: inline-block;
+        padding: 4px 10px;
+        border-radius: 12px;
+        font-size: 12px;
+        font-weight: 500;
+    }
+    .type-specialized {
+        background: #dcfce7;
+        color: #166534;
+    }
+    .type-generalist {
+        background: #dbeafe;
+        color: #1e40af;
+    }
+    .sort-indicator {
+        margin-left: 4px;
+        font-size: 10px;
+    }
+</style>
+"""
+
+LEADERBOARD_DARK_MODE_CSS = """
+<style>
+    @media (prefers-color-scheme: dark) {
+        .leaderboard-table {
+            background: #2a2a2a !important;
+        }
+        .leaderboard-table thead {
+            background: #3a3a3a !important;
+            color: #f3f4f6 !important;
+        }
+        .leaderboard-table th {
+            border-bottom: 2px solid #4a4a4a !important;
+            color: #f3f4f6 !important;
+        }
+        .leaderboard-table tbody tr {
+            border-bottom: 1px solid #3a3a3a !important;
+        }
+        .leaderboard-table tbody tr:hover {
+            background-color: #333333 !important;
+        }
+        .leaderboard-table td {
+            color: #d1d5db !important;
+        }
+        .leaderboard-table td:first-child {
+            color: #f3f4f6 !important;
+        }
+        .rank-badge {
+            background: #3a3a3a !important;
+            color: #d1d5db !important;
+        }
+        .model-name {
+            color: #f3f4f6 !important;
+        }
+        .model-name a {
+            color: #f3f4f6 !important;
+        }
+        .model-name a:hover {
+            color: #a5b4fc !important;
+        }
+        .model-developer {
+            color: #9ca3af !important;
+        }
+        .provider-badge {
+            background: #3a3a3a !important;
+            color: #a5b4fc !important;
+        }
+        .type-specialized {
+            background: #1a3a2a !important;
+            color: #86efac !important;
+        }
+        .type-generalist {
+            background: #1e2a4a !important;
+            color: #93c5fd !important;
+        }
+        .footnotes-section {
+            background: #2a2a2a !important;
+            color: #d1d5db !important;
+        }
+        .footnotes-section h4 {
+            color: #f3f4f6 !important;
+        }
+    }
+</style>
+"""
+
+CATEGORY_TABLE_CSS = """
+<style>
+    .category-table-container {
+        width: 100%;
+        overflow-x: auto;
+        margin: 20px 0;
+        -webkit-overflow-scrolling: touch;
+    }
+    .category-table {
+        width: 100%;
+        min-width: 800px;
+        border-collapse: collapse;
+        background: white;
+        box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+        border-radius: 8px;
+        overflow: hidden;
+        font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+    }
+    .category-table thead {
+        background: #e5e7eb;
+        color: #111827;
+    }
+    .category-table th {
+        padding: 14px 12px;
+        text-align: left;
+        font-weight: 600;
+        font-size: 12px;
+        text-transform: uppercase;
+        letter-spacing: 0.5px;
+        border-bottom: 2px solid #d1d5db;
+    }
+    .category-table th:first-child {
+        padding-left: 20px;
+        text-align: left;
+    }
+    .category-table th:not(:first-child) {
+        text-align: center;
+    }
+    .category-table tbody tr {
+        border-bottom: 1px solid #e5e7eb;
+        transition: background-color 0.2s ease;
+    }
+    .category-table tbody tr:hover {
+        background-color: #f8fafc;
+    }
+    .category-table tbody tr:last-child {
+        border-bottom: none;
+    }
+    .category-table td {
+        padding: 12px;
+        font-size: 13px;
+        color: #374151;
+    }
+    .category-table td:first-child {
+        padding-left: 20px;
+        font-weight: 600;
+        color: #111827;
+    }
+    .category-table td:not(:first-child) {
+        text-align: center;
+        font-weight: 500;
+    }
+
+    @media (max-width: 1024px) {
+        .category-table {
+            min-width: 700px;
+            font-size: 12px;
+        }
+        .category-table th,
+        .category-table td {
+            padding: 10px 8px;
+        }
+    }
+
+    @media (max-width: 768px) {
+        .category-table-container {
+            margin: 10px 0;
+        }
+        .category-table {
+            min-width: 600px;
+            font-size: 11px;
+        }
+        .category-table th {
+            padding: 8px 6px;
+            font-size: 10px;
+        }
+        .category-table td {
+            padding: 8px 6px;
+            font-size: 11px;
+        }
+    }
+
+    @media (max-width: 480px) {
+        .category-table {
+            min-width: 500px;
+            font-size: 10px;
+        }
+        .category-table th,
+        .category-table td {
+            padding: 6px 4px;
+        }
+    }
+
+    @media (prefers-color-scheme: dark) {
+        .category-table {
+            background: #2a2a2a !important;
+        }
+        .category-table thead {
+            background: #3a3a3a !important;
+            color: #f3f4f6 !important;
+        }
+        .category-table th {
+            border-bottom: 2px solid #4a4a4a !important;
+            color: #f3f4f6 !important;
+        }
+        .category-table tbody tr {
+            border-bottom: 1px solid #3a3a3a !important;
+        }
+        .category-table tbody tr:hover {
+            background-color: #333333 !important;
+        }
+        .category-table td:first-child {
+            color: #f3f4f6 !important;
+        }
+        .category-table td:not(:first-child) {
+            color: #1f2937 !important;
+        }
+    }
+</style>
+"""