{ "allenai/centrepourlasecuriteia-content-moderation-input-dataset/wildguard": { "model_provider_use_case": "allenai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "wildguard", "accuracy": 0.9209090909090909, "accuracy_per_category": { "cyber": 0.96, "privacy": 0.93, "cbrn": 0.93, "physical harm": 0.94, "illegal activities": 0.94, "hate speech": 0.83, "integrity & quality violations": 0.91, "harm to minors": 0.98, "self-harm": 0.81, "harmful manipulation": 0.92, "sexual content": 0.98 }, "fpr": 0.01, "mean_latency": 0.16794882569994246, "latency_ci_95": { "lower": 0.1672896153698199, "upper": 0.16860803603006502, "mean": 0.16794882569994246, "std_dev": 0.012584383679289867, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "AllenAI", "model_url": "https://huggingface.co/allenai/wildguard", "cost_info": { "cost_per_1M_input_tokens": 0.7238035981627982, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.15609910299777988 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "allenai/centrepourlasecuriteia-content-moderation-output-dataset/wildguard": { "model_provider_use_case": "allenai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "wildguard", "accuracy": 0.9, "accuracy_per_category": { "illegal activities": 0.95, "privacy": 0.99, "sexual content": 0.94, "cyber": 0.9, "self-harm": 0.85, "harm to minors": 0.98, "integrity & quality violations": 0.75, "harmful manipulation": 0.83, "physical harm": 0.97, "cbrn": 0.92, "hate speech": 0.82 }, "fpr": 0.0, "mean_latency": 0.17618908030646188, "latency_ci_95": { "lower": 0.17568036971954096, "upper": 0.1766977908933828, "mean": 0.17618908030646188, "std_dev": 0.009711330231034699, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "AllenAI", "model_url": "https://huggingface.co/allenai/wildguard", "cost_info": { "cost_per_1M_input_tokens": 0.25754181310468993, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.1637579618626171 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "anthropic/centrepourlasecuriteia-content-moderation-input-dataset/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "claude-haiku-4-5", "accuracy": 0.64, "accuracy_per_category": { "cyber": 0.81, "privacy": 0.49, "cbrn": 0.74, "physical harm": 0.67, "illegal activities": 0.77, "hate speech": 0.6, "integrity & quality violations": 0.42, "harm to minors": 0.94, "self-harm": 0.38, "harmful manipulation": 0.48, "sexual content": 0.74 }, "fpr": 0.0, "mean_latency": 0.6755487293856485, "latency_ci_95": { "lower": 0.6616969457280298, "upper": 0.6894005130432671, "mean": 0.6755487293856485, "std_dev": 0.26443177878838336, "n": 1400 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.145574 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "anthropic/centrepourlasecuriteia-content-moderation-input-dataset/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "claude-sonnet-4-5", "accuracy": 0.7909090909090909, "accuracy_per_category": { "cyber": 0.95, "privacy": 0.68, "cbrn": 0.94, "physical harm": 0.83, "illegal activities": 0.87, "hate speech": 0.65, "integrity & quality violations": 0.7, "harm to minors": 0.95, "self-harm": 0.59, "harmful manipulation": 0.73, "sexual content": 0.81 }, "fpr": 0.0, "mean_latency": 2.0321146711281366, "latency_ci_95": { "lower": 1.9980452484611515, "upper": 2.0661840937951217, "mean": 2.0321146711281366, "std_dev": 0.6503883009441236, "n": 1400 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.443607 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "anthropic/centrepourlasecuriteia-content-moderation-output-dataset/claude-haiku-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "claude-haiku-4-5", "accuracy": 0.9118181818181819, "accuracy_per_category": { "illegal activities": 0.96, "privacy": 0.91, "sexual content": 0.89, "cyber": 0.97, "self-harm": 0.88, "harm to minors": 1.0, "integrity & quality violations": 0.79, "harmful manipulation": 0.88, "physical harm": 0.97, "cbrn": 1.0, "hate speech": 0.78 }, "fpr": 0.0, "mean_latency": 0.660408535855157, "latency_ci_95": { "lower": 0.6482668563627765, "upper": 0.6725502153475376, "mean": 0.660408535855157, "std_dev": 0.23178573857401796, "n": 1400 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 1.0, "cost_per_1M_output_tokens": 5.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 0.545895 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "anthropic/centrepourlasecuriteia-content-moderation-output-dataset/claude-sonnet-4-5": { "model_provider_use_case": "anthropic", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "claude-sonnet-4-5", "accuracy": 0.8690909090909091, "accuracy_per_category": { "illegal activities": 0.95, "privacy": 0.86, "sexual content": 0.86, "cyber": 0.97, "self-harm": 0.76, "harm to minors": 0.99, "integrity & quality violations": 0.75, "harmful manipulation": 0.87, "physical harm": 0.94, "cbrn": 0.96, "hate speech": 0.65 }, "fpr": 0.0, "mean_latency": 1.980507916041783, "latency_ci_95": { "lower": 1.9466451605985857, "upper": 2.01437067148498, "mean": 1.980507916041783, "std_dev": 0.6464430053089691, "n": 1400 }, "provider": "Anthropic", "model_type": "generalist", "model_developer": "Anthropic", "model_url": "https://platform.claude.com/docs/en/about-claude/models/overview", "cost_info": { "cost_per_1M_input_tokens": 3.0, "cost_per_1M_output_tokens": 15.0, "cost_per_h": "N/A", "cost_source": "Claude API pricing", "cost_additional_info": "", "total_cost": 1.6417950000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "aws/centrepourlasecuriteia-content-moderation-input-dataset/bedrock-guardrail": { "model_provider_use_case": "aws", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "bedrock-guardrail", "accuracy": 0.7318181818181818, "accuracy_per_category": { "cyber": 0.92, "privacy": 0.53, "cbrn": 0.67, "physical harm": 0.91, "illegal activities": 0.82, "hate speech": 0.8, "integrity & quality violations": 0.49, "harm to minors": 0.93, "self-harm": 0.61, "harmful manipulation": 0.44, "sexual content": 0.93 }, "fpr": 0.0, "mean_latency": 0.3027071602003915, "latency_ci_95": { "lower": 0.2893886246223259, "upper": 0.3160256957784571, "mean": 0.3027071602003915, "std_dev": 0.2542520256463307, "n": 1400 }, "provider": "AWS", "model_type": "specialized", "model_developer": "Amazon Web Services", "model_url": "https://aws.amazon.com/bedrock/guardrails/", "cost_info": { "cost_per_1M_input_tokens": 150.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Amazon Bedrock API pricing", "cost_additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$.", "total_cost": 0.2157 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "aws/centrepourlasecuriteia-content-moderation-output-dataset/bedrock-guardrail": { "model_provider_use_case": "aws", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "bedrock-guardrail", "accuracy": 0.78, "accuracy_per_category": { "illegal activities": 0.91, "privacy": 0.73, "sexual content": 0.94, "cyber": 0.89, "self-harm": 0.7, "harm to minors": 0.93, "integrity & quality violations": 0.49, "harmful manipulation": 0.49, "physical harm": 0.96, "cbrn": 0.83, "hate speech": 0.71 }, "fpr": 0.0033333333333333335, "mean_latency": 0.3212733970369612, "latency_ci_95": { "lower": 0.3112962016769463, "upper": 0.331250592396976, "mean": 0.3212733970369612, "std_dev": 0.1904655444800315, "n": 1400 }, "provider": "AWS", "model_type": "specialized", "model_developer": "Amazon Web Services", "model_url": "https://aws.amazon.com/bedrock/guardrails/", "cost_info": { "cost_per_1M_input_tokens": 150.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Amazon Bedrock API pricing", "cost_additional_info": "Usage is measured in text units. One text unit may contain up to 1000 characters. Only the input string is considered in the text unit.\n1000 text units cost $0.15. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text units * $0.15/1000 text units) = $1.2.\nCost per 1M input tokens (not units) = 1.2$.", "total_cost": 0.3981 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "azure/centrepourlasecuriteia-content-moderation-output-dataset/analyze-text": { "model_provider_use_case": "azure", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "analyze-text", "accuracy": 0.6090909090909091, "accuracy_per_category": { "illegal activities": 0.47, "privacy": 0.26, "sexual content": 0.89, "cyber": 0.16, "self-harm": 0.86, "harm to minors": 0.71, "integrity & quality violations": 0.34, "harmful manipulation": 0.38, "physical harm": 0.91, "cbrn": 0.8, "hate speech": 0.92 }, "fpr": 0.0, "mean_latency": 0.597562814269747, "latency_ci_95": { "lower": 0.5849396956294183, "upper": 0.6101859329100757, "mean": 0.597562814269747, "std_dev": 0.24097645461586636, "n": 1400 }, "provider": "Azure", "model_type": "specialized", "model_developer": "Azure", "model_url": "https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview", "cost_info": { "cost_per_1M_input_tokens": 3.04, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Azure AI Content Safety", "cost_additional_info": "Usage is measured in text records. One text record may contain up to 1000 characters. Only the input string is considered in the text record.\n1000 text records cost $0.38. Assuming an average length of 500 characters for a message, and a token length of approx. 4 characters: Estimated cost per 1M input tokens = 1M/(500/4) text records * $0.38/1000 text records) = $3.04.\nCost per 1M input tokens (not records) = 3.04$.\n\nThe benchmark was run on the free plan.", "total_cost": 1.00852 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-input-dataset/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gemini-2.5-flash", "accuracy": 0.8818181818181818, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.98, "cbrn": 0.99, "physical harm": 0.95, "illegal activities": 0.98, "hate speech": 0.86, "integrity & quality violations": 0.88, "harm to minors": 0.56, "self-harm": 0.78, "harmful manipulation": 0.95, "sexual content": 0.78 }, "fpr": 0.016666666666666666, "mean_latency": 1.4193411101613727, "latency_ci_95": { "lower": 1.3612213357967249, "upper": 1.4774608845260204, "mean": 1.4193411101613727, "std_dev": 1.1095116483118301, "n": 1400 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 0.5480927 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-input-dataset/shieldgemma-27b": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "shieldgemma-27b", "accuracy": 0.5809090909090909, "accuracy_per_category": { "cyber": 0.29, "privacy": 0.09, "cbrn": 0.91, "physical harm": 0.87, "illegal activities": 0.64, "hate speech": 0.73, "integrity & quality violations": 0.18, "harm to minors": 0.8, "self-harm": 0.72, "harmful manipulation": 0.32, "sexual content": 0.84 }, "fpr": 0.0033333333333333335, "mean_latency": 0.1099088318007333, "latency_ci_95": { "lower": 0.1090000827959994, "upper": 0.11081758080546719, "mean": 0.1099088318007333, "std_dev": 0.0173480991137042, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Google", "model_url": "https://huggingface.co/google/shieldgemma-27b", "cost_info": { "cost_per_1M_input_tokens": 0.19256129687083262, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.10215415311257045 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-input-dataset/shieldgemma-2b": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "shieldgemma-2b", "accuracy": 0.24818181818181817, "accuracy_per_category": { "cyber": 0.15, "privacy": 0.02, "cbrn": 0.48, "physical harm": 0.53, "illegal activities": 0.38, "hate speech": 0.1, "integrity & quality violations": 0.05, "harm to minors": 0.34, "self-harm": 0.39, "harmful manipulation": 0.08, "sexual content": 0.21 }, "fpr": 0.01, "mean_latency": 0.031726044927324566, "latency_ci_95": { "lower": 0.031453915259710516, "upper": 0.031998174594938616, "mean": 0.031726044927324566, "std_dev": 0.005194979494839089, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Google", "model_url": "https://huggingface.co/google/shieldgemma-2b", "cost_info": { "cost_per_1M_input_tokens": 0.05558432617011184, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.02948759620189667 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-output-dataset/gemini-2.5-flash": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gemini-2.5-flash", "accuracy": 0.8909090909090909, "accuracy_per_category": { "illegal activities": 0.91, "privacy": 0.97, "sexual content": 0.83, "cyber": 0.95, "self-harm": 0.92, "harm to minors": 0.6, "integrity & quality violations": 0.88, "harmful manipulation": 0.91, "physical harm": 0.98, "cbrn": 0.97, "hate speech": 0.88 }, "fpr": 0.0, "mean_latency": 2.131153161866324, "latency_ci_95": { "lower": 2.0531708906747133, "upper": 2.209135433057935, "mean": 2.131153161866324, "std_dev": 1.4886884747015303, "n": 1400 }, "provider": "GoogleAIStudio", "model_type": "generalist", "model_developer": "Google", "model_url": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash", "cost_info": { "cost_per_1M_input_tokens": 0.3, "cost_per_1M_output_tokens": 2.5, "cost_per_h": "N/A", "cost_source": "Gemini Developer API pricing", "cost_additional_info": "", "total_cost": 1.1270327 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-output-dataset/shieldgemma-27b": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "shieldgemma-27b", "accuracy": 0.6336363636363637, "accuracy_per_category": { "illegal activities": 0.7, "privacy": 0.32, "sexual content": 0.82, "cyber": 0.33, "self-harm": 0.74, "harm to minors": 0.75, "integrity & quality violations": 0.29, "harmful manipulation": 0.42, "physical harm": 0.91, "cbrn": 0.96, "hate speech": 0.73 }, "fpr": 0.006666666666666667, "mean_latency": 0.13334267480032785, "latency_ci_95": { "lower": 0.13205096274443018, "upper": 0.13463438685622553, "mean": 0.13334267480032785, "std_dev": 0.024658897732318774, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Google", "model_url": "https://huggingface.co/google/shieldgemma-27b", "cost_info": { "cost_per_1M_input_tokens": 0.1388315066243309, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.12393460830052694 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "google/centrepourlasecuriteia-content-moderation-output-dataset/shieldgemma-2b": { "model_provider_use_case": "google", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "shieldgemma-2b", "accuracy": 0.26636363636363636, "accuracy_per_category": { "illegal activities": 0.39, "privacy": 0.08, "sexual content": 0.22, "cyber": 0.14, "self-harm": 0.44, "harm to minors": 0.34, "integrity & quality violations": 0.05, "harmful manipulation": 0.14, "physical harm": 0.58, "cbrn": 0.38, "hate speech": 0.17 }, "fpr": 0.006666666666666667, "mean_latency": 0.037216948441096714, "latency_ci_95": { "lower": 0.036785634085487344, "upper": 0.037648262796706085, "mean": 0.037216948441096714, "std_dev": 0.008233829309629823, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Google", "model_url": "https://huggingface.co/google/shieldgemma-2b", "cost_info": { "cost_per_1M_input_tokens": 0.03874892289189925, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.03459108596775267 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "govtech/centrepourlasecuriteia-content-moderation-input-dataset/lionguard-2": { "model_provider_use_case": "govtech", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "lionguard-2", "accuracy": 0.8845454545454545, "accuracy_per_category": { "cyber": 0.94, "privacy": 0.67, "cbrn": 0.91, "physical harm": 0.92, "illegal activities": 0.96, "hate speech": 0.83, "integrity & quality violations": 0.79, "harm to minors": 0.97, "self-harm": 0.93, "harmful manipulation": 0.83, "sexual content": 0.98 }, "fpr": 0.01, "mean_latency": 0.009503273112433297, "latency_ci_95": { "lower": 0.009311582881572434, "upper": 0.00969496334329416, "mean": 0.009503273112433297, "std_dev": 0.0036593835115966812, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "GovTech", "model_url": "https://huggingface.co/govtech/lionguard-2", "cost_info": { "cost_per_1M_input_tokens": 0.1812443960764434, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.008832764398389393 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1400 }, "govtech/centrepourlasecuriteia-content-moderation-output-dataset/lionguard-2": { "model_provider_use_case": "govtech", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "lionguard-2", "accuracy": 0.9036363636363637, "accuracy_per_category": { "illegal activities": 0.97, "privacy": 0.85, "sexual content": 0.99, "cyber": 0.93, "self-harm": 0.94, "harm to minors": 0.96, "integrity & quality violations": 0.81, "harmful manipulation": 0.77, "physical harm": 0.98, "cbrn": 0.95, "hate speech": 0.79 }, "fpr": 0.01, "mean_latency": 0.009445810999189104, "latency_ci_95": { "lower": 0.009313593155526146, "upper": 0.009578028842852061, "mean": 0.009445810999189104, "std_dev": 0.0025240503643103483, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "GovTech", "model_url": "https://huggingface.co/govtech/lionguard-2", "cost_info": { "cost_per_1M_input_tokens": 0.021758170584833442, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.00877935655646854 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using the transformers Library." }, "num_samples": 1400 }, "ibm-granite/centrepourlasecuriteia-content-moderation-input-dataset/granite-guardian-3.3-8b": { "model_provider_use_case": "ibm-granite", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "granite-guardian-3.3-8b", "accuracy": 0.9272727272727272, "accuracy_per_category": { "cyber": 0.98, "privacy": 0.92, "cbrn": 1.0, "physical harm": 0.98, "illegal activities": 0.98, "hate speech": 0.84, "integrity & quality violations": 0.8, "harm to minors": 0.99, "self-harm": 0.8, "harmful manipulation": 0.94, "sexual content": 0.97 }, "fpr": 0.0033333333333333335, "mean_latency": 0.19252273099763054, "latency_ci_95": { "lower": 0.19230705557305205, "upper": 0.19273840642220902, "mean": 0.19252273099763054, "std_dev": 0.004117262987345297, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "IBM", "model_url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b", "cost_info": { "cost_per_1M_input_tokens": 0.8130975128709368, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.17893918275501994 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "ibm-granite/centrepourlasecuriteia-content-moderation-output-dataset/granite-guardian-3.3-8b": { "model_provider_use_case": "ibm-granite", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "granite-guardian-3.3-8b", "accuracy": 0.9054545454545454, "accuracy_per_category": { "illegal activities": 0.93, "privacy": 0.91, "sexual content": 0.98, "cyber": 0.94, "self-harm": 0.85, "harm to minors": 0.99, "integrity & quality violations": 0.74, "harmful manipulation": 0.78, "physical harm": 0.99, "cbrn": 0.99, "hate speech": 0.86 }, "fpr": 0.0, "mean_latency": 0.20689634186880929, "latency_ci_95": { "lower": 0.2061988732441346, "upper": 0.20759381049348397, "mean": 0.20689634186880929, "std_dev": 0.013314737916107277, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "IBM", "model_url": "https://huggingface.co/ibm-granite/granite-guardian-3.3-8b", "cost_info": { "cost_per_1M_input_tokens": 0.2968973859968493, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.1922986555258433 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "lakera/centrepourlasecuriteia-content-moderation-input-dataset/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "lakera-guard_default", "accuracy": 0.9081818181818182, "accuracy_per_category": { "cyber": 0.95, "privacy": 0.84, "cbrn": 0.97, "physical harm": 0.96, "illegal activities": 0.95, "hate speech": 0.87, "integrity & quality violations": 0.78, "harm to minors": 0.98, "self-harm": 0.79, "harmful manipulation": 0.9, "sexual content": 1.0 }, "fpr": 0.16, "mean_latency": 0.20956397243908473, "latency_ci_95": { "lower": 0.20730703245330986, "upper": 0.2118209124248596, "mean": 0.20956397243908473, "std_dev": 0.04308518453714262, "n": 1400 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "lakera/centrepourlasecuriteia-content-moderation-output-dataset/lakera-guard_default": { "model_provider_use_case": "lakera", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "lakera-guard_default", "accuracy": 0.860909090909091, "accuracy_per_category": { "illegal activities": 0.94, "privacy": 0.88, "sexual content": 0.96, "cyber": 0.87, "self-harm": 0.84, "harm to minors": 0.91, "integrity & quality violations": 0.69, "harmful manipulation": 0.76, "physical harm": 0.96, "cbrn": 0.89, "hate speech": 0.77 }, "fpr": 0.14333333333333334, "mean_latency": 0.20940651042120798, "latency_ci_95": { "lower": 0.20735007341754588, "upper": 0.21146294742487007, "mean": 0.20940651042120798, "std_dev": 0.03925756482238554, "n": 1400 }, "provider": "Lakera", "model_type": "specialized", "model_developer": "Lakera", "model_url": "https://www.lakera.ai/lakera-guard", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Lakera API pricing", "cost_additional_info": "Free for 10,000 API requests/month; Allows prompt size up to 8,000 tokens per request.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "This is the LakeraGuard API with the default policy. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "meta/centrepourlasecuriteia-content-moderation-input-dataset/llama-guard-4-12b": { "model_provider_use_case": "meta", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "llama-guard-4-12b", "accuracy": 0.7054545454545454, "accuracy_per_category": { "cyber": 0.86, "privacy": 0.69, "cbrn": 0.78, "physical harm": 0.86, "illegal activities": 0.84, "hate speech": 0.42, "integrity & quality violations": 0.47, "harm to minors": 0.87, "self-harm": 0.69, "harmful manipulation": 0.49, "sexual content": 0.79 }, "fpr": 0.03, "mean_latency": 0.2919416810785021, "latency_ci_95": { "lower": 0.2889681782255528, "upper": 0.2949151839314514, "mean": 0.2919416810785021, "std_dev": 0.05676443323638302, "n": 1400 }, "provider": "Together AI", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Together AI pricing", "cost_additional_info": "", "total_cost": 0.0671434 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "meta/centrepourlasecuriteia-content-moderation-output-dataset/llama-guard-4-12b": { "model_provider_use_case": "meta", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "llama-guard-4-12b", "accuracy": 0.8036363636363636, "accuracy_per_category": { "illegal activities": 0.9, "privacy": 0.85, "sexual content": 0.89, "cyber": 0.89, "self-harm": 0.88, "harm to minors": 0.95, "integrity & quality violations": 0.44, "harmful manipulation": 0.54, "physical harm": 0.94, "cbrn": 0.94, "hate speech": 0.62 }, "fpr": 0.04, "mean_latency": 0.3172705331870488, "latency_ci_95": { "lower": 0.29924274832923003, "upper": 0.33529831804486754, "mean": 0.3172705331870488, "std_dev": 0.3441520121450457, "n": 1400 }, "provider": "Together AI", "model_type": "specialized", "model_developer": "Meta", "model_url": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-4/", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Together AI pricing", "cost_additional_info": "", "total_cost": 0.1373716 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "mistral/bells-o-project-content-moderation-input/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "bells-o-project-content-moderation-input", "model_name": "ministral-3b-2512", "accuracy": 0.9345454545454546, "accuracy_per_category": { "harmful manipulation": 0.94, "privacy": 0.86, "hate speech": 0.9, "sexual content": 0.97, "cbrn": 0.95, "harm to minors": 0.96, "integrity & quality violations": 0.86, "illegal activities": 0.95, "self-harm": 0.96, "physical harm": 0.97, "cyber": 0.96 }, "fpr": 0.023333333333333334, "mean_latency": 0.4279132558618273, "latency_ci_95": { "lower": 0.4038809537163048, "upper": 0.45194555800734987, "mean": 0.4279132558618273, "std_dev": 0.4587787798162123, "n": 1400 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.04, "cost_per_1M_output_tokens": 0.04, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.00413752 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "mistral/bells-o-project-content-moderation-input/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "bells-o-project-content-moderation-input", "model_name": "mistral-large-3", "accuracy": 0.9072727272727272, "accuracy_per_category": { "harmful manipulation": 0.88, "privacy": 0.79, "hate speech": 0.85, "sexual content": 0.96, "cbrn": 0.96, "harm to minors": 0.97, "integrity & quality violations": 0.75, "illegal activities": 0.95, "self-harm": 0.93, "physical harm": 0.97, "cyber": 0.97 }, "fpr": 0.0033333333333333335, "mean_latency": 0.7675250434875488, "latency_ci_95": { "lower": 0.7033643424555687, "upper": 0.8316857445195289, "mean": 0.7675250434875488, "std_dev": 1.2248334742699123, "n": 1400 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 4.0, "cost_per_1M_output_tokens": 12.0, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 0.43615200000000004 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "mistral/bells-o-project-content-moderation-output/ministral-3b-2512": { "model_provider_use_case": "mistral", "dataset_name": "bells-o-project-content-moderation-output", "model_name": "ministral-3b-2512", "accuracy": 0.9518181818181818, "accuracy_per_category": { "hate speech": 0.97, "physical harm": 0.98, "harmful manipulation": 0.92, "self-harm": 0.9, "privacy": 0.91, "cbrn": 1.0, "harm to minors": 0.98, "cyber": 0.97, "sexual content": 0.93, "illegal activities": 0.99, "integrity & quality violations": 0.92 }, "fpr": 0.07, "mean_latency": 0.4416024809224265, "latency_ci_95": { "lower": 0.41586757287002707, "upper": 0.4673373889748259, "mean": 0.4416024809224265, "std_dev": 0.491281677613309, "n": 1400 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/ministral-3-3b-25-12", "cost_info": { "cost_per_1M_input_tokens": 0.04, "cost_per_1M_output_tokens": 0.04, "cost_per_h": "N/A", "cost_source": "Ministral API pricing", "cost_additional_info": "", "total_cost": 0.01877732 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "mistral/bells-o-project-content-moderation-output/mistral-large-3": { "model_provider_use_case": "mistral", "dataset_name": "bells-o-project-content-moderation-output", "model_name": "mistral-large-3", "accuracy": 0.9190909090909091, "accuracy_per_category": { "hate speech": 0.94, "physical harm": 0.99, "harmful manipulation": 0.78, "self-harm": 0.9, "privacy": 0.96, "cbrn": 0.99, "harm to minors": 1.0, "cyber": 0.94, "sexual content": 0.92, "illegal activities": 0.92, "integrity & quality violations": 0.77 }, "fpr": 0.0, "mean_latency": 0.8493295400483267, "latency_ci_95": { "lower": 0.7685226981889265, "upper": 0.930136381907727, "mean": 0.8493295400483267, "std_dev": 1.5426097793117277, "n": 1400 }, "provider": "Mistral", "model_type": "generalist", "model_developer": "Mistral AI", "model_url": "https://docs.mistral.ai/models/mistral-large-3-25-12", "cost_info": { "cost_per_1M_input_tokens": 4.0, "cost_per_1M_output_tokens": 12.0, "cost_per_h": "N/A", "cost_source": "Mistral API pricing", "cost_additional_info": "", "total_cost": 1.898772 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "nvidia/centrepourlasecuriteia-content-moderation-input-dataset/llama-3.1-nemotron-safety-guard-8b-v3": { "model_provider_use_case": "nvidia", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "llama-3.1-nemotron-safety-guard-8b-v3", "accuracy": 0.8636363636363636, "accuracy_per_category": { "cyber": 0.96, "privacy": 0.9, "cbrn": 0.94, "physical harm": 0.97, "illegal activities": 0.92, "hate speech": 0.78, "integrity & quality violations": 0.66, "harm to minors": 0.93, "self-harm": 0.85, "harmful manipulation": 0.62, "sexual content": 0.97 }, "fpr": 0.043333333333333335, "mean_latency": 0.15576645374298095, "latency_ci_95": { "lower": 0.15397129295656875, "upper": 0.15756161452939316, "mean": 0.15576645374298095, "std_dev": 0.03426977865778628, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "NVIDIA", "model_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3", "cost_info": { "cost_per_1M_input_tokens": 0.22782226487771615, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.14477626506222618 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "nvidia/centrepourlasecuriteia-content-moderation-output-dataset/llama-3.1-nemotron-safety-guard-8b-v3": { "model_provider_use_case": "nvidia", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "llama-3.1-nemotron-safety-guard-8b-v3", "accuracy": 0.9054545454545454, "accuracy_per_category": { "illegal activities": 0.96, "privacy": 0.96, "sexual content": 0.97, "cyber": 0.95, "self-harm": 0.86, "harm to minors": 0.95, "integrity & quality violations": 0.72, "harmful manipulation": 0.73, "physical harm": 0.99, "cbrn": 1.0, "hate speech": 0.87 }, "fpr": 0.056666666666666664, "mean_latency": 0.18621215888432094, "latency_ci_95": { "lower": 0.18557276212673715, "upper": 0.18685155564190473, "mean": 0.18621215888432094, "std_dev": 0.01220614082190163, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "NVIDIA", "model_url": "https://huggingface.co/nvidia/Llama-3.1-Nemotron-Safety-Guard-8B-v3", "cost_info": { "cost_per_1M_input_tokens": 0.17354468328880082, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.1730738565630383 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-5-nano", "accuracy": 0.9254545454545454, "accuracy_per_category": { "cyber": 1.0, "privacy": 0.96, "cbrn": 1.0, "physical harm": 1.0, "illegal activities": 1.0, "hate speech": 0.8, "integrity & quality violations": 0.79, "harm to minors": 0.98, "self-harm": 0.92, "harmful manipulation": 0.98, "sexual content": 0.75 }, "fpr": 0.006666666666666667, "mean_latency": 1.5597400277001516, "latency_ci_95": { "lower": 1.5193856746455585, "upper": 1.6000943807547447, "mean": 1.5597400277001516, "std_dev": 0.7703681795673553, "n": 1400 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.048563300000000004 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-5.2", "accuracy": 0.9381818181818182, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.93, "cbrn": 0.99, "physical harm": 0.98, "illegal activities": 0.98, "hate speech": 0.78, "integrity & quality violations": 0.83, "harm to minors": 0.97, "self-harm": 0.96, "harmful manipulation": 0.94, "sexual content": 0.97 }, "fpr": 0.0033333333333333335, "mean_latency": 1.0442856402056557, "latency_ci_95": { "lower": 1.0024455032847635, "upper": 1.086125777126548, "mean": 1.0442856402056557, "std_dev": 0.7987319253759664, "n": 1400 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.6885235000000001 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-oss-120b", "accuracy": 0.9318181818181818, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.96, "cbrn": 0.99, "physical harm": 0.99, "illegal activities": 0.97, "hate speech": 0.77, "integrity & quality violations": 0.84, "harm to minors": 0.98, "self-harm": 0.92, "harmful manipulation": 0.97, "sexual content": 0.87 }, "fpr": 0.0, "mean_latency": 0.7237181276934488, "latency_ci_95": { "lower": 0.708643406560615, "upper": 0.7387928488262826, "mean": 0.7237181276934488, "std_dev": 0.28777776367461705, "n": 1400 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.032422900000000004 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-oss-20b", "accuracy": 0.6218181818181818, "accuracy_per_category": { "cyber": 0.61, "privacy": 0.69, "cbrn": 0.58, "physical harm": 0.62, "illegal activities": 0.69, "hate speech": 0.69, "integrity & quality violations": 0.47, "harm to minors": 0.61, "self-harm": 0.72, "harmful manipulation": 0.54, "sexual content": 0.62 }, "fpr": 0.02, "mean_latency": 1.7985665757315499, "latency_ci_95": { "lower": 1.7140022692418668, "upper": 1.883130882221233, "mean": 1.7985665757315499, "std_dev": 1.6143401124211116, "n": 1400 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.06419470000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-oss-safeguard-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-oss-safeguard-120b", "accuracy": 0.8781818181818182, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.94, "cbrn": 1.0, "physical harm": 0.98, "illegal activities": 0.95, "hate speech": 0.7, "integrity & quality violations": 0.69, "harm to minors": 0.96, "self-harm": 0.9, "harmful manipulation": 0.82, "sexual content": 0.73 }, "fpr": 0.0, "mean_latency": 0.9912308720179966, "latency_ci_95": { "lower": 0.9414346608201876, "upper": 1.0410270832158057, "mean": 0.9912308720179966, "std_dev": 0.950614089089987, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-120b", "cost_info": { "cost_per_1M_input_tokens": 5.898870113479002, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 3.07, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 1.1834195244259302 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/gpt-oss-safeguard-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "gpt-oss-safeguard-20b", "accuracy": 0.8881818181818182, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.93, "cbrn": 0.99, "physical harm": 0.95, "illegal activities": 0.94, "hate speech": 0.71, "integrity & quality violations": 0.71, "harm to minors": 0.98, "self-harm": 0.88, "harmful manipulation": 0.9, "sexual content": 0.79 }, "fpr": 0.0, "mean_latency": 0.5077587234973907, "latency_ci_95": { "lower": 0.4910731057974469, "upper": 0.5244443411973345, "mean": 0.5077587234973907, "std_dev": 0.3185299220402077, "n": 1400 }, "provider": "OpenRouter", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-20b", "cost_info": { "cost_per_1M_input_tokens": 0.07, "cost_per_1M_output_tokens": 0.3, "cost_per_h": "N/A", "cost_source": "OpenRouter API pricing", "cost_additional_info": "", "total_cost": 0.10993276 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-input-dataset/omni-moderation": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "omni-moderation", "accuracy": 0.74, "accuracy_per_category": { "cyber": 0.9, "privacy": 0.46, "cbrn": 0.88, "physical harm": 0.95, "illegal activities": 0.84, "hate speech": 0.76, "integrity & quality violations": 0.47, "harm to minors": 0.95, "self-harm": 0.87, "harmful manipulation": 0.38, "sexual content": 0.68 }, "fpr": 0.023333333333333334, "mean_latency": 0.3429920823233468, "latency_ci_95": { "lower": 0.33798123785984197, "upper": 0.3480029267868517, "mean": 0.3429920823233468, "std_dev": 0.09565746531044984, "n": 1400 }, "provider": "OpenAI", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/omni-moderation-latest", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "OpenAI Moderation API pricing", "cost_additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-5-nano": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-5-nano", "accuracy": 0.9027272727272727, "accuracy_per_category": { "illegal activities": 0.94, "privacy": 0.96, "sexual content": 0.67, "cyber": 0.96, "self-harm": 0.93, "harm to minors": 0.99, "integrity & quality violations": 0.75, "harmful manipulation": 0.83, "physical harm": 0.99, "cbrn": 0.99, "hate speech": 0.92 }, "fpr": 0.0033333333333333335, "mean_latency": 1.4641724479198457, "latency_ci_95": { "lower": 1.4271079360623544, "upper": 1.501236959777337, "mean": 1.4641724479198457, "std_dev": 0.7075648192793513, "n": 1400 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5-nano", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.4, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 0.05068805 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to minimal; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-5.2": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-5.2", "accuracy": 0.9518181818181818, "accuracy_per_category": { "illegal activities": 0.98, "privacy": 0.97, "sexual content": 0.97, "cyber": 0.98, "self-harm": 0.96, "harm to minors": 0.98, "integrity & quality violations": 0.85, "harmful manipulation": 0.92, "physical harm": 0.99, "cbrn": 0.99, "hate speech": 0.88 }, "fpr": 0.0, "mean_latency": 1.1199401940618243, "latency_ci_95": { "lower": 1.0988383402519575, "upper": 1.1410420478716912, "mean": 1.1199401940618243, "std_dev": 0.40283626113424703, "n": 1400 }, "provider": "OpenAI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/gpt-5.2", "cost_info": { "cost_per_1M_input_tokens": 1.75, "cost_per_1M_output_tokens": 14.0, "cost_per_h": "N/A", "cost_source": "OpenAI API pricing", "cost_additional_info": "", "total_cost": 1.3295257500000002 }, "execution_specifications": { "type": "API", "details": "Reasoning effort set to low; see OpenAI API docs. REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-oss-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-oss-120b", "accuracy": 0.9281818181818182, "accuracy_per_category": { "illegal activities": 0.96, "privacy": 0.98, "sexual content": 0.85, "cyber": 0.99, "self-harm": 0.93, "harm to minors": 0.99, "integrity & quality violations": 0.85, "harmful manipulation": 0.9, "physical harm": 0.97, "cbrn": 0.98, "hate speech": 0.81 }, "fpr": 0.0, "mean_latency": 0.7561430590493339, "latency_ci_95": { "lower": 0.7382452744998518, "upper": 0.774040843598816, "mean": 0.7561430590493339, "std_dev": 0.34167029472682847, "n": 1400 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-120b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.051996850000000004 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-oss-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-oss-20b", "accuracy": 0.7181818181818181, "accuracy_per_category": { "illegal activities": 0.65, "privacy": 0.72, "sexual content": 0.68, "cyber": 0.76, "self-harm": 0.79, "harm to minors": 0.7, "integrity & quality violations": 0.66, "harmful manipulation": 0.72, "physical harm": 0.76, "cbrn": 0.7, "hate speech": 0.76 }, "fpr": 0.0033333333333333335, "mean_latency": 1.757279393843242, "latency_ci_95": { "lower": 1.6735166402449926, "upper": 1.8410421474414915, "mean": 1.757279393843242, "std_dev": 1.5990383966194697, "n": 1400 }, "provider": "Together AI", "model_type": "generalist", "model_developer": "OpenAI", "model_url": "https://www.together.ai/models/gpt-oss-20b", "cost_info": { "cost_per_1M_input_tokens": 0.05, "cost_per_1M_output_tokens": 0.2, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.08121905 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-oss-safeguard-120b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-oss-safeguard-120b", "accuracy": 0.8418181818181818, "accuracy_per_category": { "illegal activities": 0.95, "privacy": 0.92, "sexual content": 0.56, "cyber": 0.97, "self-harm": 0.9, "harm to minors": 0.96, "integrity & quality violations": 0.68, "harmful manipulation": 0.72, "physical harm": 0.93, "cbrn": 0.99, "hate speech": 0.68 }, "fpr": 0.0, "mean_latency": 1.1339499698366438, "latency_ci_95": { "lower": 1.0827253500645733, "upper": 1.1851745896087142, "mean": 1.1339499698366438, "std_dev": 0.9778825354839475, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-120b", "cost_info": { "cost_per_1M_input_tokens": 2.437501498074139, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 3.07, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 1.3538102695438596 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (94GB NVL) on RunPod using vLLM." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/gpt-oss-safeguard-20b": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "gpt-oss-safeguard-20b", "accuracy": 0.8927272727272727, "accuracy_per_category": { "illegal activities": 0.92, "privacy": 0.94, "sexual content": 0.78, "cyber": 0.97, "self-harm": 0.85, "harm to minors": 0.97, "integrity & quality violations": 0.77, "harmful manipulation": 0.93, "physical harm": 0.96, "cbrn": 0.98, "hate speech": 0.75 }, "fpr": 0.0, "mean_latency": 0.5182382569994245, "latency_ci_95": { "lower": 0.5020243010997294, "upper": 0.5344522128991196, "mean": 0.5182382569994245, "std_dev": 0.309525856433271, "n": 1400 }, "provider": "OpenRouter", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://huggingface.co/openai/gpt-oss-safeguard-20b", "cost_info": { "cost_per_1M_input_tokens": 0.07, "cost_per_1M_output_tokens": 0.3, "cost_per_h": "N/A", "cost_source": "OpenRouter API pricing", "cost_additional_info": "", "total_cost": 0.13713153 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "openai/centrepourlasecuriteia-content-moderation-output-dataset/omni-moderation": { "model_provider_use_case": "openai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "omni-moderation", "accuracy": 0.7090909090909091, "accuracy_per_category": { "illegal activities": 0.76, "privacy": 0.72, "sexual content": 0.64, "cyber": 0.75, "self-harm": 0.84, "harm to minors": 0.79, "integrity & quality violations": 0.4, "harmful manipulation": 0.32, "physical harm": 0.93, "cbrn": 0.84, "hate speech": 0.81 }, "fpr": 0.006666666666666667, "mean_latency": 0.3155126539298466, "latency_ci_95": { "lower": 0.3097630727618276, "upper": 0.32126223509786567, "mean": 0.3155126539298466, "std_dev": 0.1097600145315037, "n": 1400 }, "provider": "OpenAI", "model_type": "specialized", "model_developer": "OpenAI", "model_url": "https://platform.openai.com/docs/models/omni-moderation-latest", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "OpenAI Moderation API pricing", "cost_additional_info": "The Omni Moderation endpoint is free to use but has low rate limits in the free tier. Your usage tier on the OpenAI API determines these rate limits.", "total_cost": 0.0 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "qwen/centrepourlasecuriteia-content-moderation-input-dataset/qwen3guard-gen-0.6b": { "model_provider_use_case": "qwen", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "qwen3guard-gen-0.6b", "accuracy": 0.9245454545454546, "accuracy_per_category": { "cyber": 0.98, "privacy": 0.93, "cbrn": 0.95, "physical harm": 0.97, "illegal activities": 0.96, "hate speech": 0.84, "integrity & quality violations": 0.85, "harm to minors": 0.97, "self-harm": 0.8, "harmful manipulation": 0.92, "sexual content": 1.0 }, "fpr": 0.02666666666666667, "mean_latency": 0.19419640813555036, "latency_ci_95": { "lower": 0.19107999562025785, "upper": 0.19731282065084288, "mean": 0.19419640813555036, "std_dev": 0.05949259136775017, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Qwen", "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B", "cost_info": { "cost_per_1M_input_tokens": 0.38936574188219714, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.1804947726726532 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "qwen/centrepourlasecuriteia-content-moderation-input-dataset/qwen3guard-gen-8b": { "model_provider_use_case": "qwen", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "qwen3guard-gen-8b", "accuracy": 0.9454545454545454, "accuracy_per_category": { "cyber": 1.0, "privacy": 0.94, "cbrn": 1.0, "physical harm": 0.98, "illegal activities": 0.97, "hate speech": 0.85, "integrity & quality violations": 0.84, "harm to minors": 0.99, "self-harm": 0.9, "harmful manipulation": 0.95, "sexual content": 0.98 }, "fpr": 0.006666666666666667, "mean_latency": 0.19008652653012956, "latency_ci_95": { "lower": 0.18743975410113897, "upper": 0.19273329895912014, "mean": 0.19008652653012956, "std_dev": 0.05052712045939928, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Qwen", "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "cost_info": { "cost_per_1M_input_tokens": 0.38112538813051666, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.17667486604717045 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "qwen/centrepourlasecuriteia-content-moderation-output-dataset/qwen3guard-gen-0.6b": { "model_provider_use_case": "qwen", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "qwen3guard-gen-0.6b", "accuracy": 0.9227272727272727, "accuracy_per_category": { "illegal activities": 0.94, "privacy": 0.96, "sexual content": 0.96, "cyber": 0.96, "self-harm": 0.86, "harm to minors": 0.99, "integrity & quality violations": 0.8, "harmful manipulation": 0.83, "physical harm": 0.99, "cbrn": 0.94, "hate speech": 0.92 }, "fpr": 0.016666666666666666, "mean_latency": 0.048780372994286676, "latency_ci_95": { "lower": 0.0480823539105477, "upper": 0.04947839207802565, "mean": 0.048780372994286676, "std_dev": 0.013325246228475014, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Qwen", "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B", "cost_info": { "cost_per_1M_input_tokens": 0.05528456019474252, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.04533864667746756 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "qwen/centrepourlasecuriteia-content-moderation-output-dataset/qwen3guard-gen-8b": { "model_provider_use_case": "qwen", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "qwen3guard-gen-8b", "accuracy": 0.9545454545454546, "accuracy_per_category": { "illegal activities": 0.99, "privacy": 0.98, "sexual content": 0.99, "cyber": 0.97, "self-harm": 0.91, "harm to minors": 1.0, "integrity & quality violations": 0.83, "harmful manipulation": 0.93, "physical harm": 1.0, "cbrn": 0.98, "hate speech": 0.92 }, "fpr": 0.016666666666666666, "mean_latency": 0.13238392829895018, "latency_ci_95": { "lower": 0.13105481108714015, "upper": 0.13371304551076021, "mean": 0.13238392829895018, "std_dev": 0.025372965476823234, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "Qwen", "model_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "cost_info": { "cost_per_1M_input_tokens": 0.1500354918097283, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.12304350669119093 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "rakancorle1/centrepourlasecuriteia-content-moderation-input-dataset/thinkguard": { "model_provider_use_case": "rakancorle1", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "thinkguard", "accuracy": 0.7781818181818182, "accuracy_per_category": { "cyber": 0.86, "privacy": 0.73, "cbrn": 0.84, "physical harm": 0.9, "illegal activities": 0.83, "hate speech": 0.72, "integrity & quality violations": 0.63, "harm to minors": 0.86, "self-harm": 0.64, "harmful manipulation": 0.69, "sexual content": 0.86 }, "fpr": 0.08, "mean_latency": 0.1785528426510947, "latency_ci_95": { "lower": 0.17834486711263356, "upper": 0.17876081818955583, "mean": 0.1785528426510947, "std_dev": 0.003970271478323348, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "RakanCorle1", "model_url": "https://huggingface.co/Rakancorle1/ThinkGuard", "cost_info": { "cost_per_1M_input_tokens": 0.2906798634516623, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.16595494764182303 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "rakancorle1/centrepourlasecuriteia-content-moderation-output-dataset/thinkguard": { "model_provider_use_case": "rakancorle1", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "thinkguard", "accuracy": 0.8145454545454546, "accuracy_per_category": { "illegal activities": 0.91, "privacy": 0.87, "sexual content": 0.86, "cyber": 0.84, "self-harm": 0.69, "harm to minors": 0.92, "integrity & quality violations": 0.62, "harmful manipulation": 0.74, "physical harm": 0.94, "cbrn": 0.86, "hate speech": 0.71 }, "fpr": 0.08333333333333333, "mean_latency": 0.18875198875154767, "latency_ci_95": { "lower": 0.1882683485403946, "upper": 0.18923562896270074, "mean": 0.18875198875154767, "std_dev": 0.00923273453368272, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "RakanCorle1", "model_url": "https://huggingface.co/Rakancorle1/ThinkGuard", "cost_info": { "cost_per_1M_input_tokens": 0.19000934406913317, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.17543448732296626 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "saillab/centrepourlasecuriteia-content-moderation-input-dataset/xguard": { "model_provider_use_case": "saillab", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "xguard", "accuracy": 0.8281818181818181, "accuracy_per_category": { "cyber": 0.89, "privacy": 0.82, "cbrn": 0.92, "physical harm": 0.9, "illegal activities": 0.91, "hate speech": 0.76, "integrity & quality violations": 0.65, "harm to minors": 0.9, "self-harm": 0.72, "harmful manipulation": 0.77, "sexual content": 0.87 }, "fpr": 0.0, "mean_latency": 1.438086905990328, "latency_ci_95": { "lower": 1.4220830667384288, "upper": 1.4540907452422274, "mean": 1.438086905990328, "std_dev": 0.30551471098781485, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "SAIL Lab", "model_url": "https://huggingface.co/saillab/x-guard", "cost_info": { "cost_per_1M_input_tokens": 11.345572408123338, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 1.3366218854010106 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "saillab/centrepourlasecuriteia-content-moderation-output-dataset/xguard": { "model_provider_use_case": "saillab", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "xguard", "accuracy": 0.9009090909090909, "accuracy_per_category": { "illegal activities": 0.96, "privacy": 0.94, "sexual content": 0.95, "cyber": 0.91, "self-harm": 0.9, "harm to minors": 0.98, "integrity & quality violations": 0.72, "harmful manipulation": 0.77, "physical harm": 0.98, "cbrn": 0.94, "hate speech": 0.86 }, "fpr": 0.0033333333333333335, "mean_latency": 0.4900521091052464, "latency_ci_95": { "lower": 0.4689487805620818, "upper": 0.511155437648411, "mean": 0.4900521091052464, "std_dev": 0.40286441392372, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "SAIL Lab", "model_url": "https://huggingface.co/saillab/x-guard", "cost_info": { "cost_per_1M_input_tokens": 0.9604441053405044, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.4554762102961541 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "toxicityprompts/centrepourlasecuriteia-content-moderation-input-dataset/polyguard-ministral": { "model_provider_use_case": "toxicityprompts", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "polyguard-ministral", "accuracy": 0.92, "accuracy_per_category": { "cyber": 0.98, "privacy": 0.84, "cbrn": 1.0, "physical harm": 0.97, "illegal activities": 0.95, "hate speech": 0.82, "integrity & quality violations": 0.8, "harm to minors": 0.99, "self-harm": 0.89, "harmful manipulation": 0.89, "sexual content": 0.99 }, "fpr": 0.0033333333333333335, "mean_latency": 0.17871589626584733, "latency_ci_95": { "lower": 0.17849986671699192, "upper": 0.17893192581470274, "mean": 0.17871589626584733, "std_dev": 0.0041240232512055235, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ToxicityPrompts", "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral", "cost_info": { "cost_per_1M_input_tokens": 0.4409996758824333, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.16610649691820145 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "toxicityprompts/centrepourlasecuriteia-content-moderation-input-dataset/polyguard-qwen": { "model_provider_use_case": "toxicityprompts", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "polyguard-qwen", "accuracy": 0.9354545454545454, "accuracy_per_category": { "cyber": 0.98, "privacy": 0.93, "cbrn": 0.99, "physical harm": 0.97, "illegal activities": 0.98, "hate speech": 0.83, "integrity & quality violations": 0.84, "harm to minors": 0.98, "self-harm": 0.91, "harmful manipulation": 0.88, "sexual content": 1.0 }, "fpr": 0.0, "mean_latency": 0.16735810961042133, "latency_ci_95": { "lower": 0.167181707969697, "upper": 0.16753451125114566, "mean": 0.16735810961042133, "std_dev": 0.0033675229696693413, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ToxicityPrompts", "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen", "cost_info": { "cost_per_1M_input_tokens": 0.42708542860944143, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.15555006521013048 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "toxicityprompts/centrepourlasecuriteia-content-moderation-output-dataset/polyguard-ministral": { "model_provider_use_case": "toxicityprompts", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "polyguard-ministral", "accuracy": 0.9318181818181818, "accuracy_per_category": { "illegal activities": 0.97, "privacy": 0.97, "sexual content": 0.97, "cyber": 0.96, "self-harm": 0.88, "harm to minors": 0.98, "integrity & quality violations": 0.83, "harmful manipulation": 0.88, "physical harm": 0.99, "cbrn": 0.99, "hate speech": 0.83 }, "fpr": 0.013333333333333334, "mean_latency": 0.3748241812842233, "latency_ci_95": { "lower": 0.37309410657377223, "upper": 0.37655425599467435, "mean": 0.3748241812842233, "std_dev": 0.03302727969403025, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ToxicityPrompts", "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Ministral", "cost_info": { "cost_per_1M_input_tokens": 0.4686604178361155, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.34837825293805863 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "toxicityprompts/centrepourlasecuriteia-content-moderation-output-dataset/polyguard-qwen": { "model_provider_use_case": "toxicityprompts", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "polyguard-qwen", "accuracy": 0.9572727272727273, "accuracy_per_category": { "illegal activities": 1.0, "privacy": 0.98, "sexual content": 0.98, "cyber": 0.98, "self-harm": 0.93, "harm to minors": 1.0, "integrity & quality violations": 0.86, "harmful manipulation": 0.88, "physical harm": 1.0, "cbrn": 1.0, "hate speech": 0.92 }, "fpr": 0.013333333333333334, "mean_latency": 0.3705882680416107, "latency_ci_95": { "lower": 0.3683706245632213, "upper": 0.37280591152000014, "mean": 0.3705882680416107, "std_dev": 0.04233501072523893, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "ToxicityPrompts", "model_url": "https://huggingface.co/ToxicityPrompts/PolyGuard-Qwen", "cost_info": { "cost_per_1M_input_tokens": 0.47809043060129436, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 2.39, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.34444120690756375 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an H100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 }, "virtue-ai/centrepourlasecuriteia-content-moderation-input-dataset/virtueguard-text-lite": { "model_provider_use_case": "virtue-ai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "virtueguard-text-lite", "accuracy": 0.7290909090909091, "accuracy_per_category": { "cyber": 0.83, "privacy": 0.61, "cbrn": 0.86, "physical harm": 0.84, "illegal activities": 0.86, "hate speech": 0.57, "integrity & quality violations": 0.61, "harm to minors": 0.9, "self-harm": 0.62, "harmful manipulation": 0.68, "sexual content": 0.64 }, "fpr": 0.0, "mean_latency": 0.24810820971216474, "latency_ci_95": { "lower": 0.22206087989181125, "upper": 0.27415553953251826, "mean": 0.24810820971216474, "std_dev": 0.4972458368778715, "n": 1400 }, "provider": "Together AI", "model_type": "specialized", "model_developer": "Virtue AI", "model_url": "https://www.together.ai/models/virtueguard-text-lite", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.0098006 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "virtue-ai/centrepourlasecuriteia-content-moderation-output-dataset/virtueguard-text-lite": { "model_provider_use_case": "virtue-ai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "virtueguard-text-lite", "accuracy": 0.11181818181818182, "accuracy_per_category": { "illegal activities": 0.14, "privacy": 0.08, "sexual content": 0.11, "cyber": 0.27, "self-harm": 0.0, "harm to minors": 0.1, "integrity & quality violations": 0.13, "harmful manipulation": 0.12, "physical harm": 0.01, "cbrn": 0.25, "hate speech": 0.02 }, "fpr": 0.0033333333333333335, "mean_latency": 0.31551449128559655, "latency_ci_95": { "lower": 0.2464019144914514, "upper": 0.3846270680797417, "mean": 0.31551449128559655, "std_dev": 1.3193652218407883, "n": 1400 }, "provider": "Together AI", "model_type": "specialized", "model_developer": "Virtue AI", "model_url": "https://www.together.ai/models/virtueguard-text-lite", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.0, "cost_per_h": "N/A", "cost_source": "Together AI API pricing", "cost_additional_info": "", "total_cost": 0.0810904 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "x-ai/centrepourlasecuriteia-content-moderation-input-dataset/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.8263636363636364, "accuracy_per_category": { "cyber": 0.99, "privacy": 0.76, "cbrn": 0.93, "physical harm": 0.97, "illegal activities": 0.97, "hate speech": 0.48, "integrity & quality violations": 0.73, "harm to minors": 0.98, "self-harm": 0.66, "harmful manipulation": 0.9, "sexual content": 0.72 }, "fpr": 0.0, "mean_latency": 0.9228259367602212, "latency_ci_95": { "lower": 0.6722643846420433, "upper": 1.173387488878399, "mean": 0.9228259367602212, "std_dev": 4.783242256757777, "n": 1400 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.06539510000000001 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "x-ai/centrepourlasecuriteia-content-moderation-output-dataset/grok-4-1-fast-non-reasoning": { "model_provider_use_case": "x-ai", "dataset_name": "centrepourlasecuriteia-content-moderation-output-dataset", "model_name": "grok-4-1-fast-non-reasoning", "accuracy": 0.8363636363636363, "accuracy_per_category": { "illegal activities": 0.95, "privacy": 0.91, "sexual content": 0.63, "cyber": 0.96, "self-harm": 0.9, "harm to minors": 0.96, "integrity & quality violations": 0.72, "harmful manipulation": 0.83, "physical harm": 0.95, "cbrn": 0.75, "hate speech": 0.64 }, "fpr": 0.0, "mean_latency": 0.8401098469325474, "latency_ci_95": { "lower": 0.6645747595715246, "upper": 1.0156449342935703, "mean": 0.8401098469325474, "std_dev": 3.350980389093776, "n": 1400 }, "provider": "xAI", "model_type": "generalist", "model_developer": "X-AI", "model_url": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", "cost_info": { "cost_per_1M_input_tokens": 0.2, "cost_per_1M_output_tokens": 0.5, "cost_per_h": "N/A", "cost_source": "X-AI API pricing", "cost_additional_info": "", "total_cost": 0.1310404 }, "execution_specifications": { "type": "API", "details": "REST API accessed through a 'Memory-Optimized 32GB RAM' CPU RunPod instance from US-KS-2." }, "num_samples": 1400 }, "bells-o/centrepourlasecuriteia-content-moderation-input-dataset/opencc-cm-escalation": { "model_provider_use_case": "bells-o", "dataset_name": "centrepourlasecuriteia-content-moderation-input-dataset", "model_name": "opencc-cm-escalation", "accuracy": 0.9609090909090909, "accuracy_per_category": { "harmful manipulation": 0.98, "privacy": 0.76, "hate speech": 0.96, "sexual content": 0.98, "cbrn": 0.99, "harm to minors": 1.0, "integrity & quality violations": 0.94, "illegal activities": 0.99, "self-harm": 0.99, "physical harm": 1.0, "cyber": 0.98 }, "fpr": 0.17, "mean_latency": 0.12831727709089008, "latency_ci_95": { "lower": 0.1266602779288333, "upper": 0.12997427625294686, "mean": 0.12831727709089008, "std_dev": 0.031632260992795695, "n": 1400 }, "provider": "RunPod", "model_type": "specialized", "model_developer": "CeSIA", "model_url": "https://huggingface.co/centrepourlasecuriteia/opencc-cm-escalation", "cost_info": { "cost_per_1M_input_tokens": 0.0, "cost_per_1M_output_tokens": 0.0, "cost_per_h": 1.49, "cost_source": "RunPod", "cost_additional_info": "Output token cost is disregarded. The cost per 1M input tokens is estimated as total_cost * (1,000,000 / total_input_tokens).", "total_cost": 0.07435273333655464 }, "execution_specifications": { "type": "Local", "details": "This model was ran on an A100 (80GB PCIe) on RunPod using vLLM." }, "num_samples": 1400 } }