| { |
| "model_key": "sentinel-mb-c-d11", |
| "encoder_model": "answerdotai/ModernBERT-base", |
| "encoder_params_millions": 149.7, |
| "head_type": "columnar", |
| "head_code": "c", |
| "head_variant": "d11", |
| "head_dropout": 0.1, |
| "head_div": 1, |
| "head_mul": 1, |
| "head_skip": true, |
| "head_architecture": "funnel", |
| "model_family": "modernbert-base", |
| "projection_size": 640, |
| "trainable_head_params": 14325653, |
| "dataset_counts": { |
| "train": 900, |
| "dev": 150, |
| "test": 150 |
| }, |
| "dataset_signature": { |
| "generator_version": "2026-04-07-final-audit-clear-v1", |
| "counts": { |
| "train": 900, |
| "dev": 150, |
| "test": 150 |
| }, |
| "distribution": { |
| "train": { |
| "risky": 603, |
| "clean": 297 |
| }, |
| "dev": { |
| "risky": 142, |
| "clean": 8 |
| }, |
| "test": { |
| "risky": 142, |
| "clean": 8 |
| } |
| } |
| }, |
| "label_groups": { |
| "severity": [ |
| "sev_0_compliant_or_ok", |
| "sev_1_minor", |
| "sev_2_moderate", |
| "sev_3_high" |
| ], |
| "domain": [ |
| "performance_claims_forecasting", |
| "investment_advice_suitability", |
| "conflicts_inducements", |
| "marketing_solicitation_advertising", |
| "selective_disclosure_fair_access", |
| "mnpi_insider_trading", |
| "recordkeeping_supervision", |
| "ai_automation_capability_claims", |
| "privacy_confidentiality", |
| "cybersecurity_internal_controls", |
| "employment_favoritism_role_conflict", |
| "aml_and_suspicious_activity", |
| "other_unknown" |
| ], |
| "subtype": [ |
| "speculative_outcomes_unqualified", |
| "implicit_or_explicit_guarantee", |
| "risk_context_omitted_or_unbalanced", |
| "unregistered_personalized_investment_advice", |
| "undisclosed_economic_conflict_or_referral", |
| "pressure_or_coercion", |
| "selective_disclosure", |
| "mnpi_misuse_or_encouragement", |
| "recordkeeping_or_preapproval_evasion", |
| "ai_autonomy_or_safety_overstatement", |
| "credentials_validation_or_compliance_misrepresentation", |
| "confidential_data_leakage", |
| "internal_controls_or_exception_process_leakage", |
| "academic_commercial_role_blurring_or_quid_pro_quo", |
| "improper_solicitation_offering_pressure", |
| "excessive_trading_or_account_churning", |
| "product_switching_without_cost_benefit_analysis", |
| "dual_registrant_capacity_or_wrap_fee_conflict_confusion", |
| "elder_exploitation_or_vulnerable_client_signal", |
| "suspicious_activity_indicator_or_structuring", |
| "influencer_or_social_media_promotion_compliance_failure", |
| "crypto_asset_misrepresentation_or_inadequate_disclosure", |
| "other_unknown" |
| ], |
| "jurisdiction": [ |
| "US", |
| "EU", |
| "UK", |
| "Other", |
| "Unknown" |
| ], |
| "why": [ |
| "forward_looking_statement_unqualified", |
| "guarantee_or_assurance_language", |
| "omits_material_risk_or_downside", |
| "implies_downside_protection_or_no_drawdown", |
| "cherry_picks_performance_period", |
| "omits_performance_methodology_or_gross_net_context", |
| "personalized_trade_or_allocation_recommendation", |
| "timing_or_sizing_guidance", |
| "creates_implied_advisory_relationship", |
| "conflict_not_disclosed", |
| "referral_relationship_not_disclosed", |
| "omits_fees_costs_or_reasonably_available_alternatives", |
| "selective_private_performance_or_fundraising_update", |
| "off_the_record_or_not_in_writing_language", |
| "mnpi_possession_indicated", |
| "encourages_action_before_public_release", |
| "avoid_recordkeeping_channel_shift", |
| "bypasses_required_preapproval", |
| "pressure_scarcity_urgency", |
| "unsubstantiated_social_proof_or_validation", |
| "omits_testimonial_endorsement_or_rating_disclosure", |
| "obscures_required_disclosure_or_form_crs", |
| "minimizes_need_for_diligence_or_compliance", |
| "overstates_ai_capability_or_removes_human_oversight", |
| "claims_compliance_risk_eliminated", |
| "shares_sensitive_personal_or_financial_data", |
| "violates_need_to_know_data_minimization", |
| "shares_sensitive_internal_controls_or_exceptions", |
| "role_power_imbalance_or_favoritism", |
| "excessive_trading_cost_to_equity", |
| "inadequate_customer_profile_or_suitability_basis", |
| "exploits_vulnerable_or_elderly_client", |
| "aml_suspicious_activity_indicator", |
| "omits_switching_costs_and_product_comparison", |
| "conflict_language_understates_actual_relationship", |
| "omits_influencer_compensation_or_affiliation_disclosure", |
| "misrepresents_sipc_or_regulatory_protection_for_crypto", |
| "data_breach_notification_obligation_triggered", |
| "impedes_regulatory_reporting_or_whistleblower_rights" |
| ], |
| "impacted_principles": [ |
| "truthful_non_misleading_communications", |
| "balanced_risk_reward_presentation", |
| "no_performance_guarantees_or_promissory_language", |
| "registration_and_scope_of_advice", |
| "duty_of_loyalty_conflict_disclosure", |
| "fair_access_to_material_information", |
| "insider_trading_and_mnpi_controls", |
| "supervision_and_books_records", |
| "privacy_confidentiality_and_secure_handling", |
| "security_control_integrity", |
| "role_separation_and_fair_access_in_academia", |
| "non_coercion_and_no_undue_influence", |
| "accurate_ai_capability_and_human_oversight", |
| "client_vulnerability_and_exploitation_prevention", |
| "aml_and_sanctions_compliance" |
| ], |
| "remediation_actions": [ |
| "add_forward_looking_disclaimer", |
| "reframe_as_scenarios_not_expectations", |
| "add_balanced_risk_and_downside_section", |
| "remove_or_soften_guarantee_language", |
| "remove_personalized_recommendations", |
| "add_registered_advice_boundary_language", |
| "disclose_conflicts_and_compensation", |
| "add_fees_costs_and_alternatives_comparison", |
| "use_standardized_approved_performance_materials", |
| "add_performance_methodology_and_gross_net_context", |
| "avoid_selective_disclosure_share_broadly", |
| "escalate_mnpi_to_compliance_and_halt", |
| "keep_discussion_on_retained_channels", |
| "require_formal_preapproval_before_send", |
| "remove_pressure_scarcity_and_use_factual_timeline", |
| "substantiation_or_remove_credibility_claims", |
| "add_testimonial_endorsement_and_rating_disclosure", |
| "make_required_disclosure_clear_and_prominent", |
| "avoid_minimizing_compliance_or_diligence", |
| "clarify_ai_is_assistive_with_human_review", |
| "remove_claims_that_ai_eliminates_risk", |
| "redact_and_minimize_sensitive_data", |
| "use_secure_transfer_and_limit_access", |
| "avoid_sharing_internal_controls_or_sanitize", |
| "route_academic_opportunities_through_institution", |
| "separate_recommendation_letters_from_work", |
| "assess_cost_to_equity_against_client_profile", |
| "flag_for_elder_exploitation_review_and_hold", |
| "assess_sar_filing_obligation_and_escalate", |
| "initiate_breach_notification_review_and_timeline", |
| "remove_provisions_impeding_regulatory_communications" |
| ] |
| }, |
| "metadata_groups": { |
| "content_type": [ |
| "email", |
| "message" |
| ], |
| "audience_segment": [ |
| "client", |
| "internal", |
| "prospect_or_investor", |
| "public", |
| "third_party" |
| ], |
| "detection_difficulty": [ |
| "obvious", |
| "moderate", |
| "subtle" |
| ], |
| "aggravating_factors": [ |
| "intentional", |
| "reckless", |
| "negligent", |
| "concealment_present", |
| "customer_harm_potential", |
| "financial_benefit_to_respondent", |
| "vulnerable_client", |
| "pattern_or_duration" |
| ] |
| }, |
| "output_signature": { |
| "violation": { |
| "type": "binary" |
| }, |
| "severity": { |
| "type": "multiclass", |
| "labels": [ |
| "sev_0_compliant_or_ok", |
| "sev_1_minor", |
| "sev_2_moderate", |
| "sev_3_high" |
| ] |
| }, |
| "domain": { |
| "type": "multiclass", |
| "labels": [ |
| "performance_claims_forecasting", |
| "investment_advice_suitability", |
| "conflicts_inducements", |
| "marketing_solicitation_advertising", |
| "selective_disclosure_fair_access", |
| "mnpi_insider_trading", |
| "recordkeeping_supervision", |
| "ai_automation_capability_claims", |
| "privacy_confidentiality", |
| "cybersecurity_internal_controls", |
| "employment_favoritism_role_conflict", |
| "aml_and_suspicious_activity", |
| "other_unknown" |
| ] |
| }, |
| "subtype": { |
| "type": "multiclass", |
| "labels": [ |
| "speculative_outcomes_unqualified", |
| "implicit_or_explicit_guarantee", |
| "risk_context_omitted_or_unbalanced", |
| "unregistered_personalized_investment_advice", |
| "undisclosed_economic_conflict_or_referral", |
| "pressure_or_coercion", |
| "selective_disclosure", |
| "mnpi_misuse_or_encouragement", |
| "recordkeeping_or_preapproval_evasion", |
| "ai_autonomy_or_safety_overstatement", |
| "credentials_validation_or_compliance_misrepresentation", |
| "confidential_data_leakage", |
| "internal_controls_or_exception_process_leakage", |
| "academic_commercial_role_blurring_or_quid_pro_quo", |
| "improper_solicitation_offering_pressure", |
| "excessive_trading_or_account_churning", |
| "product_switching_without_cost_benefit_analysis", |
| "dual_registrant_capacity_or_wrap_fee_conflict_confusion", |
| "elder_exploitation_or_vulnerable_client_signal", |
| "suspicious_activity_indicator_or_structuring", |
| "influencer_or_social_media_promotion_compliance_failure", |
| "crypto_asset_misrepresentation_or_inadequate_disclosure", |
| "other_unknown" |
| ] |
| }, |
| "jurisdiction": { |
| "type": "multiclass", |
| "labels": [ |
| "US", |
| "EU", |
| "UK", |
| "Other", |
| "Unknown" |
| ] |
| }, |
| "why": { |
| "type": "multilabel", |
| "labels": [ |
| "forward_looking_statement_unqualified", |
| "guarantee_or_assurance_language", |
| "omits_material_risk_or_downside", |
| "implies_downside_protection_or_no_drawdown", |
| "cherry_picks_performance_period", |
| "omits_performance_methodology_or_gross_net_context", |
| "personalized_trade_or_allocation_recommendation", |
| "timing_or_sizing_guidance", |
| "creates_implied_advisory_relationship", |
| "conflict_not_disclosed", |
| "referral_relationship_not_disclosed", |
| "omits_fees_costs_or_reasonably_available_alternatives", |
| "selective_private_performance_or_fundraising_update", |
| "off_the_record_or_not_in_writing_language", |
| "mnpi_possession_indicated", |
| "encourages_action_before_public_release", |
| "avoid_recordkeeping_channel_shift", |
| "bypasses_required_preapproval", |
| "pressure_scarcity_urgency", |
| "unsubstantiated_social_proof_or_validation", |
| "omits_testimonial_endorsement_or_rating_disclosure", |
| "obscures_required_disclosure_or_form_crs", |
| "minimizes_need_for_diligence_or_compliance", |
| "overstates_ai_capability_or_removes_human_oversight", |
| "claims_compliance_risk_eliminated", |
| "shares_sensitive_personal_or_financial_data", |
| "violates_need_to_know_data_minimization", |
| "shares_sensitive_internal_controls_or_exceptions", |
| "role_power_imbalance_or_favoritism", |
| "excessive_trading_cost_to_equity", |
| "inadequate_customer_profile_or_suitability_basis", |
| "exploits_vulnerable_or_elderly_client", |
| "aml_suspicious_activity_indicator", |
| "omits_switching_costs_and_product_comparison", |
| "conflict_language_understates_actual_relationship", |
| "omits_influencer_compensation_or_affiliation_disclosure", |
| "misrepresents_sipc_or_regulatory_protection_for_crypto", |
| "data_breach_notification_obligation_triggered", |
| "impedes_regulatory_reporting_or_whistleblower_rights" |
| ] |
| }, |
| "impacted_principles": { |
| "type": "multilabel", |
| "labels": [ |
| "truthful_non_misleading_communications", |
| "balanced_risk_reward_presentation", |
| "no_performance_guarantees_or_promissory_language", |
| "registration_and_scope_of_advice", |
| "duty_of_loyalty_conflict_disclosure", |
| "fair_access_to_material_information", |
| "insider_trading_and_mnpi_controls", |
| "supervision_and_books_records", |
| "privacy_confidentiality_and_secure_handling", |
| "security_control_integrity", |
| "role_separation_and_fair_access_in_academia", |
| "non_coercion_and_no_undue_influence", |
| "accurate_ai_capability_and_human_oversight", |
| "client_vulnerability_and_exploitation_prevention", |
| "aml_and_sanctions_compliance" |
| ] |
| }, |
| "remediation_actions": { |
| "type": "multilabel", |
| "labels": [ |
| "add_forward_looking_disclaimer", |
| "reframe_as_scenarios_not_expectations", |
| "add_balanced_risk_and_downside_section", |
| "remove_or_soften_guarantee_language", |
| "remove_personalized_recommendations", |
| "add_registered_advice_boundary_language", |
| "disclose_conflicts_and_compensation", |
| "add_fees_costs_and_alternatives_comparison", |
| "use_standardized_approved_performance_materials", |
| "add_performance_methodology_and_gross_net_context", |
| "avoid_selective_disclosure_share_broadly", |
| "escalate_mnpi_to_compliance_and_halt", |
| "keep_discussion_on_retained_channels", |
| "require_formal_preapproval_before_send", |
| "remove_pressure_scarcity_and_use_factual_timeline", |
| "substantiation_or_remove_credibility_claims", |
| "add_testimonial_endorsement_and_rating_disclosure", |
| "make_required_disclosure_clear_and_prominent", |
| "avoid_minimizing_compliance_or_diligence", |
| "clarify_ai_is_assistive_with_human_review", |
| "remove_claims_that_ai_eliminates_risk", |
| "redact_and_minimize_sensitive_data", |
| "use_secure_transfer_and_limit_access", |
| "avoid_sharing_internal_controls_or_sanitize", |
| "route_academic_opportunities_through_institution", |
| "separate_recommendation_letters_from_work", |
| "assess_cost_to_equity_against_client_profile", |
| "flag_for_elder_exploitation_review_and_hold", |
| "assess_sar_filing_obligation_and_escalate", |
| "initiate_breach_notification_review_and_timeline", |
| "remove_provisions_impeding_regulatory_communications" |
| ] |
| }, |
| "content_type": { |
| "type": "multiclass", |
| "labels": [ |
| "email", |
| "message" |
| ] |
| }, |
| "audience_segment": { |
| "type": "multiclass", |
| "labels": [ |
| "client", |
| "internal", |
| "prospect_or_investor", |
| "public", |
| "third_party" |
| ] |
| }, |
| "detection_difficulty": { |
| "type": "multiclass", |
| "labels": [ |
| "obvious", |
| "moderate", |
| "subtle" |
| ] |
| }, |
| "aggravating_factors": { |
| "type": "multilabel", |
| "labels": [ |
| "intentional", |
| "reckless", |
| "negligent", |
| "concealment_present", |
| "customer_harm_potential", |
| "financial_benefit_to_respondent", |
| "vulnerable_client", |
| "pattern_or_duration" |
| ] |
| } |
| }, |
| "device_info": { |
| "device": "cuda", |
| "torch_cuda_version": "11.8", |
| "gpu_count": 1, |
| "gpu_name": "NVIDIA GeForce RTX 2080 Ti", |
| "gpu_memory_gb": 11.0, |
| "gpu_capability": "7.5", |
| "nvidia_smi": [ |
| "NVIDIA GeForce RTX 2080 Ti, 591.74, 11264 MiB" |
| ] |
| }, |
| "timings": { |
| "encoding_seconds": 0.0, |
| "training_seconds": 82.25, |
| "total_seconds": 86.5 |
| }, |
| "cycles": [ |
| { |
| "loss": 7.7748064517974855, |
| "violation_accuracy": 0.9933333333333333, |
| "violation_precision": 1.0, |
| "violation_recall": 0.9929577464788732, |
| "violation_f1": 0.9964664310954063, |
| "severity_accuracy": 0.68, |
| "severity_precision_macro": 0.6277301315037164, |
| "severity_recall_macro": 0.707563025210084, |
| "severity_f1_macro": 0.6574701673088821, |
| "domain_accuracy": 0.8466666666666667, |
| "domain_precision_macro": 0.8728019516325967, |
| "domain_recall_macro": 0.8525462962962963, |
| "domain_f1_macro": 0.8494220062066961, |
| "subtype_accuracy": 0.7733333333333333, |
| "subtype_precision_macro": 0.8113931523022433, |
| "subtype_recall_macro": 0.7490964843237571, |
| "subtype_f1_macro": 0.7508012065714375, |
| "jurisdiction_accuracy": 0.6933333333333334, |
| "jurisdiction_precision_macro": 0.39713131313131317, |
| "jurisdiction_recall_macro": 0.4161038961038961, |
| "jurisdiction_f1_macro": 0.3923395902343271, |
| "why_precision_micro": 0.5229591836734694, |
| "why_precision_macro": 0.5766391767639499, |
| "why_recall_micro": 0.779467680608365, |
| "why_recall_macro": 0.7375322683014991, |
| "why_f1_micro": 0.6259541984732825, |
| "why_f1_macro": 0.6266278969973275, |
| "impacted_principles_precision_micro": 0.714859437751004, |
| "impacted_principles_precision_macro": 0.7255202728514017, |
| "impacted_principles_recall_micro": 0.8127853881278538, |
| "impacted_principles_recall_macro": 0.7848574654881022, |
| "impacted_principles_f1_micro": 0.7606837606837606, |
| "impacted_principles_f1_macro": 0.7439683431383844, |
| "remediation_actions_precision_micro": 0.6126760563380281, |
| "remediation_actions_precision_macro": 0.5838607852720756, |
| "remediation_actions_recall_micro": 0.7733333333333333, |
| "remediation_actions_recall_macro": 0.7043368620792969, |
| "remediation_actions_f1_micro": 0.6836935166994106, |
| "remediation_actions_f1_macro": 0.6242740328903318, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.44666666666666666, |
| "detection_difficulty_precision_macro": 0.4471819645732689, |
| "detection_difficulty_recall_macro": 0.46915306915306915, |
| "detection_difficulty_f1_macro": 0.4404195664321677, |
| "aggravating_factors_precision_micro": 0.5650793650793651, |
| "aggravating_factors_precision_macro": 0.550085885667087, |
| "aggravating_factors_recall_micro": 0.7574468085106383, |
| "aggravating_factors_recall_macro": 0.7552521514727553, |
| "aggravating_factors_f1_micro": 0.6472727272727272, |
| "aggravating_factors_f1_macro": 0.6306325855261203, |
| "stage_a_selection_score": 0.7456116562791146, |
| "selection_score": 0.7500419326212061, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9933333333333333, |
| "violation_accuracy_scenario_macro_risky": 0.9929577464788732, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1371", |
| "violation_worst_scenario_label": "risky", |
| "cycle": 1, |
| "best_epoch": 15, |
| "epochs_ran": 21, |
| "lr": 0.003, |
| "head_dropout": 0.1, |
| "weight_decay": 0.01, |
| "cycle_seconds": 20.43 |
| }, |
| { |
| "loss": 11.207931518554688, |
| "violation_accuracy": 0.9933333333333333, |
| "violation_precision": 1.0, |
| "violation_recall": 0.9929577464788732, |
| "violation_f1": 0.9964664310954063, |
| "severity_accuracy": 0.7133333333333334, |
| "severity_precision_macro": 0.5736714975845411, |
| "severity_recall_macro": 0.5810399159663866, |
| "severity_f1_macro": 0.577203237410072, |
| "domain_accuracy": 0.8733333333333333, |
| "domain_precision_macro": 0.9152304502304504, |
| "domain_recall_macro": 0.9037037037037038, |
| "domain_f1_macro": 0.8981829715276235, |
| "subtype_accuracy": 0.82, |
| "subtype_precision_macro": 0.8295979273252001, |
| "subtype_recall_macro": 0.8100452577725306, |
| "subtype_f1_macro": 0.8046637752590468, |
| "jurisdiction_accuracy": 0.6933333333333334, |
| "jurisdiction_precision_macro": 0.41350649350649354, |
| "jurisdiction_recall_macro": 0.4179220779220779, |
| "jurisdiction_f1_macro": 0.4076005906238464, |
| "why_precision_micro": 0.6041666666666666, |
| "why_precision_macro": 0.5994839193351778, |
| "why_recall_micro": 0.7718631178707225, |
| "why_recall_macro": 0.7338144761221683, |
| "why_f1_micro": 0.67779632721202, |
| "why_f1_macro": 0.6489732285249087, |
| "impacted_principles_precision_micro": 0.7204724409448819, |
| "impacted_principles_precision_macro": 0.7509759521524227, |
| "impacted_principles_recall_micro": 0.8356164383561644, |
| "impacted_principles_recall_macro": 0.7889626527134941, |
| "impacted_principles_f1_micro": 0.773784355179704, |
| "impacted_principles_f1_macro": 0.7575593568585874, |
| "remediation_actions_precision_micro": 0.6105263157894737, |
| "remediation_actions_precision_macro": 0.5976390453783973, |
| "remediation_actions_recall_micro": 0.7733333333333333, |
| "remediation_actions_recall_macro": 0.690795299444056, |
| "remediation_actions_f1_micro": 0.6823529411764706, |
| "remediation_actions_f1_macro": 0.6264413385705756, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.41333333333333333, |
| "detection_difficulty_precision_macro": 0.4076248313090418, |
| "detection_difficulty_recall_macro": 0.4146464646464647, |
| "detection_difficulty_f1_macro": 0.41032213795594075, |
| "aggravating_factors_precision_micro": 0.6544715447154471, |
| "aggravating_factors_precision_macro": 0.6429940120546376, |
| "aggravating_factors_recall_micro": 0.6851063829787234, |
| "aggravating_factors_recall_macro": 0.6755889259203152, |
| "aggravating_factors_f1_micro": 0.6694386694386694, |
| "aggravating_factors_f1_macro": 0.6555896631040743, |
| "stage_a_selection_score": 0.767369221062852, |
| "selection_score": 0.7671909669032824, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9933333333333333, |
| "violation_accuracy_scenario_macro_risky": 0.9929577464788732, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1371", |
| "violation_worst_scenario_label": "risky", |
| "cycle": 2, |
| "best_epoch": 28, |
| "epochs_ran": 34, |
| "lr": 0.001, |
| "head_dropout": 0.1, |
| "weight_decay": 0.0, |
| "cycle_seconds": 32.74 |
| }, |
| { |
| "loss": 8.636670589447021, |
| "violation_accuracy": 0.9866666666666667, |
| "violation_precision": 0.9929577464788732, |
| "violation_recall": 0.9929577464788732, |
| "violation_f1": 0.9929577464788732, |
| "severity_accuracy": 0.66, |
| "severity_precision_macro": 0.4967107870333677, |
| "severity_recall_macro": 0.5516281512605042, |
| "severity_f1_macro": 0.5189861673414305, |
| "domain_accuracy": 0.8666666666666667, |
| "domain_precision_macro": 0.8598119380377445, |
| "domain_recall_macro": 0.9199074074074075, |
| "domain_f1_macro": 0.8685643227768131, |
| "subtype_accuracy": 0.7866666666666666, |
| "subtype_precision_macro": 0.8163114663114664, |
| "subtype_recall_macro": 0.775937950937951, |
| "subtype_f1_macro": 0.7646749863327403, |
| "jurisdiction_accuracy": 0.7733333333333333, |
| "jurisdiction_precision_macro": 0.5372131147540984, |
| "jurisdiction_recall_macro": 0.4397402597402597, |
| "jurisdiction_f1_macro": 0.47636711947056776, |
| "why_precision_micro": 0.4528301886792453, |
| "why_precision_macro": 0.48296837652041275, |
| "why_recall_micro": 0.8212927756653993, |
| "why_recall_macro": 0.7714144117990271, |
| "why_f1_micro": 0.5837837837837838, |
| "why_f1_macro": 0.5733734446440217, |
| "impacted_principles_precision_micro": 0.6332179930795848, |
| "impacted_principles_precision_macro": 0.6573170966740058, |
| "impacted_principles_recall_micro": 0.8356164383561644, |
| "impacted_principles_recall_macro": 0.8099590558905407, |
| "impacted_principles_f1_micro": 0.7204724409448819, |
| "impacted_principles_f1_macro": 0.7112060969947187, |
| "remediation_actions_precision_micro": 0.5207756232686981, |
| "remediation_actions_precision_macro": 0.5043151896069353, |
| "remediation_actions_recall_micro": 0.8355555555555556, |
| "remediation_actions_recall_macro": 0.7390617197643065, |
| "remediation_actions_f1_micro": 0.6416382252559728, |
| "remediation_actions_f1_macro": 0.5815208256530966, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.41333333333333333, |
| "detection_difficulty_precision_macro": 0.39879147137211657, |
| "detection_difficulty_recall_macro": 0.40982905982905987, |
| "detection_difficulty_f1_macro": 0.4012269618676941, |
| "aggravating_factors_precision_micro": 0.5627009646302251, |
| "aggravating_factors_precision_macro": 0.5604147213071698, |
| "aggravating_factors_recall_micro": 0.7446808510638298, |
| "aggravating_factors_recall_macro": 0.7464349207339351, |
| "aggravating_factors_f1_micro": 0.6410256410256411, |
| "aggravating_factors_f1_macro": 0.6206138817554251, |
| "stage_a_selection_score": 0.7407038445266995, |
| "selection_score": 0.7436551178025155, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9866666666666667, |
| "violation_accuracy_scenario_macro_risky": 0.9929577464788732, |
| "violation_accuracy_scenario_macro_clean": 0.875, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1371", |
| "violation_worst_scenario_label": "risky", |
| "cycle": 3, |
| "best_epoch": 23, |
| "epochs_ran": 29, |
| "lr": 0.0005, |
| "head_dropout": 0.1, |
| "weight_decay": 0.02, |
| "cycle_seconds": 27.33 |
| } |
| ], |
| "best_cycle": { |
| "loss": 11.207931518554688, |
| "violation_accuracy": 0.9933333333333333, |
| "violation_precision": 1.0, |
| "violation_recall": 0.9929577464788732, |
| "violation_f1": 0.9964664310954063, |
| "severity_accuracy": 0.7133333333333334, |
| "severity_precision_macro": 0.5736714975845411, |
| "severity_recall_macro": 0.5810399159663866, |
| "severity_f1_macro": 0.577203237410072, |
| "domain_accuracy": 0.8733333333333333, |
| "domain_precision_macro": 0.9152304502304504, |
| "domain_recall_macro": 0.9037037037037038, |
| "domain_f1_macro": 0.8981829715276235, |
| "subtype_accuracy": 0.82, |
| "subtype_precision_macro": 0.8295979273252001, |
| "subtype_recall_macro": 0.8100452577725306, |
| "subtype_f1_macro": 0.8046637752590468, |
| "jurisdiction_accuracy": 0.6933333333333334, |
| "jurisdiction_precision_macro": 0.41350649350649354, |
| "jurisdiction_recall_macro": 0.4179220779220779, |
| "jurisdiction_f1_macro": 0.4076005906238464, |
| "why_precision_micro": 0.6041666666666666, |
| "why_precision_macro": 0.5994839193351778, |
| "why_recall_micro": 0.7718631178707225, |
| "why_recall_macro": 0.7338144761221683, |
| "why_f1_micro": 0.67779632721202, |
| "why_f1_macro": 0.6489732285249087, |
| "impacted_principles_precision_micro": 0.7204724409448819, |
| "impacted_principles_precision_macro": 0.7509759521524227, |
| "impacted_principles_recall_micro": 0.8356164383561644, |
| "impacted_principles_recall_macro": 0.7889626527134941, |
| "impacted_principles_f1_micro": 0.773784355179704, |
| "impacted_principles_f1_macro": 0.7575593568585874, |
| "remediation_actions_precision_micro": 0.6105263157894737, |
| "remediation_actions_precision_macro": 0.5976390453783973, |
| "remediation_actions_recall_micro": 0.7733333333333333, |
| "remediation_actions_recall_macro": 0.690795299444056, |
| "remediation_actions_f1_micro": 0.6823529411764706, |
| "remediation_actions_f1_macro": 0.6264413385705756, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.41333333333333333, |
| "detection_difficulty_precision_macro": 0.4076248313090418, |
| "detection_difficulty_recall_macro": 0.4146464646464647, |
| "detection_difficulty_f1_macro": 0.41032213795594075, |
| "aggravating_factors_precision_micro": 0.6544715447154471, |
| "aggravating_factors_precision_macro": 0.6429940120546376, |
| "aggravating_factors_recall_micro": 0.6851063829787234, |
| "aggravating_factors_recall_macro": 0.6755889259203152, |
| "aggravating_factors_f1_micro": 0.6694386694386694, |
| "aggravating_factors_f1_macro": 0.6555896631040743, |
| "stage_a_selection_score": 0.767369221062852, |
| "selection_score": 0.7671909669032824, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9933333333333333, |
| "violation_accuracy_scenario_macro_risky": 0.9929577464788732, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1371", |
| "violation_worst_scenario_label": "risky", |
| "cycle": 2, |
| "best_epoch": 28, |
| "epochs_ran": 34, |
| "lr": 0.001, |
| "head_dropout": 0.1, |
| "weight_decay": 0.0, |
| "cycle_seconds": 32.74 |
| }, |
| "train": { |
| "loss": 0.29129520431160927, |
| "violation_accuracy": 1.0, |
| "violation_precision": 1.0, |
| "violation_recall": 1.0, |
| "violation_f1": 1.0, |
| "severity_accuracy": 0.9733333333333334, |
| "severity_precision_macro": 0.979737423027768, |
| "severity_recall_macro": 0.9795454017784588, |
| "severity_f1_macro": 0.9795522630686699, |
| "domain_accuracy": 0.9911111111111112, |
| "domain_precision_macro": 0.9922987117552334, |
| "domain_recall_macro": 0.9966329966329965, |
| "domain_f1_macro": 0.9943418090318922, |
| "subtype_accuracy": 1.0, |
| "subtype_precision_macro": 1.0, |
| "subtype_recall_macro": 1.0, |
| "subtype_f1_macro": 1.0, |
| "jurisdiction_accuracy": 0.9711111111111111, |
| "jurisdiction_precision_macro": 0.8754392733703078, |
| "jurisdiction_recall_macro": 0.9931847968545217, |
| "jurisdiction_f1_macro": 0.9279309415166559, |
| "why_precision_micro": 0.7692307692307693, |
| "why_precision_macro": 0.7629469993662027, |
| "why_recall_micro": 1.0, |
| "why_recall_macro": 0.9487179487179487, |
| "why_f1_micro": 0.8695652173913044, |
| "why_f1_macro": 0.8392609851418487, |
| "impacted_principles_precision_micro": 0.9257028112449799, |
| "impacted_principles_precision_macro": 0.8949360744911738, |
| "impacted_principles_recall_micro": 0.9913978494623656, |
| "impacted_principles_recall_macro": 0.9304394224733208, |
| "impacted_principles_f1_micro": 0.9574247144340603, |
| "impacted_principles_f1_macro": 0.9116715954228238, |
| "remediation_actions_precision_micro": 0.8031128404669261, |
| "remediation_actions_precision_macro": 0.743125500508932, |
| "remediation_actions_recall_micro": 0.9990319457889641, |
| "remediation_actions_recall_macro": 0.9029428409734013, |
| "remediation_actions_f1_micro": 0.8904227782571181, |
| "remediation_actions_f1_macro": 0.8109253028549341, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.9944444444444445, |
| "detection_difficulty_precision_macro": 0.9945552657437111, |
| "detection_difficulty_recall_macro": 0.9945552657437111, |
| "detection_difficulty_f1_macro": 0.9945552657437111, |
| "aggravating_factors_precision_micro": 0.9263862332695985, |
| "aggravating_factors_precision_macro": 0.9236474949570554, |
| "aggravating_factors_recall_micro": 0.9979402677651905, |
| "aggravating_factors_recall_macro": 0.9992537313432837, |
| "aggravating_factors_f1_micro": 0.9608329201784829, |
| "aggravating_factors_f1_macro": 0.9592327500257608, |
| "stage_a_selection_score": 0.937580517111993, |
| "selection_score": 0.947753814478068, |
| "scenario_key_count": 900, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 1.0, |
| "violation_accuracy_scenario_macro_risky": 1.0, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 1.0, |
| "violation_worst_scenario_key": "train_1001", |
| "violation_worst_scenario_label": "risky" |
| }, |
| "dev": { |
| "loss": 11.207931518554688, |
| "violation_accuracy": 0.9933333333333333, |
| "violation_precision": 1.0, |
| "violation_recall": 0.9929577464788732, |
| "violation_f1": 0.9964664310954063, |
| "severity_accuracy": 0.7133333333333334, |
| "severity_precision_macro": 0.5736714975845411, |
| "severity_recall_macro": 0.5810399159663866, |
| "severity_f1_macro": 0.577203237410072, |
| "domain_accuracy": 0.8733333333333333, |
| "domain_precision_macro": 0.9152304502304504, |
| "domain_recall_macro": 0.9037037037037038, |
| "domain_f1_macro": 0.8981829715276235, |
| "subtype_accuracy": 0.82, |
| "subtype_precision_macro": 0.8295979273252001, |
| "subtype_recall_macro": 0.8100452577725306, |
| "subtype_f1_macro": 0.8046637752590468, |
| "jurisdiction_accuracy": 0.6933333333333334, |
| "jurisdiction_precision_macro": 0.41350649350649354, |
| "jurisdiction_recall_macro": 0.4179220779220779, |
| "jurisdiction_f1_macro": 0.4076005906238464, |
| "why_precision_micro": 0.616822429906542, |
| "why_precision_macro": 0.6160081633765844, |
| "why_recall_micro": 0.752851711026616, |
| "why_recall_macro": 0.7186333609410531, |
| "why_f1_micro": 0.678082191780822, |
| "why_f1_macro": 0.6517414247029207, |
| "impacted_principles_precision_micro": 0.7631578947368421, |
| "impacted_principles_precision_macro": 0.7874420024420025, |
| "impacted_principles_recall_micro": 0.7945205479452054, |
| "impacted_principles_recall_macro": 0.7614157289194307, |
| "impacted_principles_f1_micro": 0.7785234899328859, |
| "impacted_principles_f1_macro": 0.7660467655075498, |
| "remediation_actions_precision_micro": 0.6105263157894737, |
| "remediation_actions_precision_macro": 0.5976390453783973, |
| "remediation_actions_recall_micro": 0.7733333333333333, |
| "remediation_actions_recall_macro": 0.690795299444056, |
| "remediation_actions_f1_micro": 0.6823529411764706, |
| "remediation_actions_f1_macro": 0.6264413385705756, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.41333333333333333, |
| "detection_difficulty_precision_macro": 0.4076248313090418, |
| "detection_difficulty_recall_macro": 0.4146464646464647, |
| "detection_difficulty_f1_macro": 0.41032213795594075, |
| "aggravating_factors_precision_micro": 0.6404494382022472, |
| "aggravating_factors_precision_macro": 0.6351122397339503, |
| "aggravating_factors_recall_micro": 0.7276595744680852, |
| "aggravating_factors_recall_macro": 0.7164210015443564, |
| "aggravating_factors_f1_micro": 0.6812749003984064, |
| "aggravating_factors_f1_macro": 0.6705742793431082, |
| "stage_a_selection_score": 0.7687761716662238, |
| "selection_score": 0.7690657581979315, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9933333333333333, |
| "violation_accuracy_scenario_macro_risky": 0.9929577464788732, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1371", |
| "violation_worst_scenario_label": "risky" |
| }, |
| "test": { |
| "loss": 10.207207107543946, |
| "violation_accuracy": 0.9866666666666667, |
| "violation_precision": 1.0, |
| "violation_recall": 0.9859154929577465, |
| "violation_f1": 0.9929078014184397, |
| "severity_accuracy": 0.7266666666666667, |
| "severity_precision_macro": 0.7056742540613509, |
| "severity_recall_macro": 0.6917853651724619, |
| "severity_f1_macro": 0.6937461494861875, |
| "domain_accuracy": 0.82, |
| "domain_precision_macro": 0.8639371000239372, |
| "domain_recall_macro": 0.7870126705653021, |
| "domain_f1_macro": 0.8032142065328451, |
| "subtype_accuracy": 0.7733333333333333, |
| "subtype_precision_macro": 0.7708825265643447, |
| "subtype_recall_macro": 0.7368260527351436, |
| "subtype_f1_macro": 0.7383595011385061, |
| "jurisdiction_accuracy": 0.74, |
| "jurisdiction_precision_macro": 0.5511805026656511, |
| "jurisdiction_recall_macro": 0.5755799755799755, |
| "jurisdiction_f1_macro": 0.5608646466716769, |
| "why_precision_micro": 0.6408045977011494, |
| "why_precision_macro": 0.6228897802851919, |
| "why_recall_micro": 0.8228782287822878, |
| "why_recall_macro": 0.7797228098698687, |
| "why_f1_micro": 0.7205169628432957, |
| "why_f1_macro": 0.6837887640406874, |
| "impacted_principles_precision_micro": 0.7368421052631579, |
| "impacted_principles_precision_macro": 0.7691853878810401, |
| "impacted_principles_recall_micro": 0.7636363636363637, |
| "impacted_principles_recall_macro": 0.6710974322869485, |
| "impacted_principles_f1_micro": 0.7499999999999999, |
| "impacted_principles_f1_macro": 0.7030370589130892, |
| "remediation_actions_precision_micro": 0.6188811188811189, |
| "remediation_actions_precision_macro": 0.5923653065256482, |
| "remediation_actions_recall_micro": 0.7695652173913043, |
| "remediation_actions_recall_macro": 0.684497765569872, |
| "remediation_actions_f1_micro": 0.686046511627907, |
| "remediation_actions_f1_macro": 0.6175714466344578, |
| "content_type_accuracy": 1.0, |
| "content_type_precision_macro": 1.0, |
| "content_type_recall_macro": 1.0, |
| "content_type_f1_macro": 1.0, |
| "audience_segment_accuracy": 1.0, |
| "audience_segment_precision_macro": 1.0, |
| "audience_segment_recall_macro": 1.0, |
| "audience_segment_f1_macro": 1.0, |
| "detection_difficulty_accuracy": 0.47333333333333333, |
| "detection_difficulty_precision_macro": 0.46757744378508614, |
| "detection_difficulty_recall_macro": 0.471182412358883, |
| "detection_difficulty_f1_macro": 0.46490073858516184, |
| "aggravating_factors_precision_micro": 0.6641509433962264, |
| "aggravating_factors_precision_macro": 0.6283313196161129, |
| "aggravating_factors_recall_micro": 0.7333333333333333, |
| "aggravating_factors_recall_macro": 0.6949052211781471, |
| "aggravating_factors_f1_micro": 0.697029702970297, |
| "aggravating_factors_f1_macro": 0.6546016914120363, |
| "stage_a_selection_score": 0.7506931806680867, |
| "selection_score": 0.7565296660343293, |
| "scenario_key_count": 150, |
| "rows_per_scenario_min": 1, |
| "rows_per_scenario_median": 1.0, |
| "rows_per_scenario_max": 1, |
| "violation_accuracy_scenario_macro": 0.9866666666666667, |
| "violation_accuracy_scenario_macro_risky": 0.9859154929577465, |
| "violation_accuracy_scenario_macro_clean": 1.0, |
| "violation_accuracy_scenario_min": 0.0, |
| "violation_worst_scenario_key": "train_1843", |
| "violation_worst_scenario_label": "risky" |
| }, |
| "thresholds": { |
| "violation": 0.5, |
| "why": 0.55, |
| "impacted_principles": 0.7, |
| "remediation_actions": 0.5, |
| "aggravating_factors": 0.4 |
| }, |
| "log_path": "_cache/logs/legacy/stage-a-grid-v3-gpu/raw/260424_135746_sentinel-mb-c-d11.log", |
| "prior_poc_inflation_factors": [ |
| "The previous PoC reused the same 17 synthetic families across train, dev, and test, so the model mostly learned family signatures rather than broad compliance reasoning.", |
| "Every prior observation carried extra structural cues such as source metadata, evidence snippets, and explicit jurisdiction sentences appended to the text.", |
| "A later dataset refactor silently dropped jurisdiction, impacted-principle, and remediation heads, which made the reported Stage A contract narrower than the product actually promises.", |
| "Reported micro metrics on dense negative label maps made performance look cleaner than a realistic class-by-class review would suggest." |
| ], |
| "mitigations": [ |
| "The data pipeline now uses a 150-row agent-authored pilot plus a hard human-review gate before any 1000/100/100 release split is allowed to exist on disk.", |
| "The generation workflow now keeps Python limited to validation, formatting, duplicate review, and statistics while the agent authors and labels each observation directly.", |
| "The encoder default still uses a 512-token window, which comfortably covers the current 1000-character manual-authoring ceiling.", |
| "The full Stage A diagnose/prescribe contract is restored in both dataset and model outputs: jurisdiction, why, impacted principles, remediation actions, detection difficulty, and aggravating factors are all explicit.", |
| "Dataset generation now validates the mock contract keys directly and requires a human-reviewed approval hash before contract changes can pass validation.", |
| "The model factory now constructs full model bundles, while checkpoints store the trained projection and heads plus the frozen encoder reference instead of duplicating immutable backbone weights.", |
| "Evaluation artifacts now report scenario-family macro violation metrics and worst-family binary performance so repeated rows inside a narrow split cannot hide behind a flattering row-average alone.", |
| "Cross-checkpoint comparison artifacts are only kept when they are refreshed against the current dataset, preventing stale benchmark reports from masquerading as current evidence." |
| ], |
| "artifact_format": "checkpoint_only", |
| "end_to_end_serialized": false, |
| "transformers_bundle_dir": null, |
| "checkpoint_dir": "_models/stage-a-grid-v3-gpu/sentinel-mb-c-d11/260424_135913_sentinel-mb-c-d11", |
| "display_name": "sentinel-mb-c-d11@260424_135913" |
| } |
|
|
