Phase 5: Fairness Assessment & Mitigation - logistic_regression (Combined Results): phase5_complete_summary.json
078f816 verified | { | |
| "method": "logistic_regression", | |
| "phase": "Phase 5: Fairness Assessment & Mitigation", | |
| "deployment_config": { | |
| "method": "logistic_regression", | |
| "is_mitigated": true, | |
| "use_group_thresholds": false, | |
| "threshold_configuration": { | |
| "type": "global", | |
| "source": "phase4_roi_optimization", | |
| "global_threshold": 0.028621862186218622, | |
| "group_thresholds": {} | |
| }, | |
| "fairness_status": { | |
| "bias_detected": true, | |
| "mitigation_applied": true, | |
| "fairness_compliant": true | |
| }, | |
| "phase6_instructions": { | |
| "load_thresholds_from": "phase4", | |
| "apply_group_specific": false, | |
| "evaluation_type": "global_threshold_system" | |
| } | |
| }, | |
| "evaluation": { | |
| "output_dir": "/kaggle/working/hospital-readmission-risk/phase-5-fairness-assessment-mitigation/outputs/logistic_regression/evaluation", | |
| "bias_detected": true, | |
| "overall_performance": { | |
| "confusion_matrix": { | |
| "TP": 1678, | |
| "FP": 11056, | |
| "TN": 2505, | |
| "FN": 26 | |
| }, | |
| "accuracy": 0.2740255486406813, | |
| "precision": 0.1317732055913303, | |
| "recall": 0.9847417840375586, | |
| "f1_score": 0.2324421665050561, | |
| "tpr": 0.9847417840375586, | |
| "fpr": 0.8152791092102353, | |
| "tnr": 0.18472089078976478, | |
| "fnr": 0.015258215962441314, | |
| "roc_auc": 0.8075357980787905, | |
| "brier_score": 0.08689973498374315, | |
| "intervention_rate": 0.8341958729118899 | |
| }, | |
| "fairness_summary": { | |
| "phase": 5, | |
| "evaluation_name": "Fairness Evaluation & Deployment Readiness", | |
| "model": "Logistic Regression with Platt Calibration", | |
| "optimal_threshold": 0.028621862186218622, | |
| "overall_performance": { | |
| "confusion_matrix": { | |
| "TP": 1678, | |
| "FP": 11056, | |
| "TN": 2505, | |
| "FN": 26 | |
| }, | |
| "accuracy": 0.2740255486406813, | |
| "precision": 0.1317732055913303, | |
| "recall": 0.9847417840375586, | |
| "f1_score": 0.2324421665050561, | |
| "tpr": 0.9847417840375586, | |
| "fpr": 0.8152791092102353, | |
| "tnr": 0.18472089078976478, | |
| "fnr": 0.015258215962441314, | |
| "roc_auc": 0.8075357980787905, | |
| "brier_score": 0.08689973498374315, | |
| "intervention_rate": 0.8341958729118899 | |
| }, | |
| "fairness_evaluation": { | |
| "race": { | |
| "demographic_parity": { | |
| "metric": "demographic_parity", | |
| "definition": "Intervention rate should be similar across groups", | |
| "intervention_rates": { | |
| "AfricanAmerican": 0.7755946225439504, | |
| "Caucasian": 0.8510765041273083, | |
| "Hispanic": 0.728125, | |
| "Other": 0.8217821782178217, | |
| "Asian": 0.9230769230769231 | |
| }, | |
| "max_rate": 0.9230769230769231, | |
| "min_rate": 0.728125, | |
| "gap": 0.1949519230769231, | |
| "std": 0.06634245509090504, | |
| "passed": false | |
| }, | |
| "equalized_odds": { | |
| "metric": "equalized_odds", | |
| "definition": "TPR and FPR should be similar across groups", | |
| "tpr": { | |
| "AfricanAmerican": 0.9847094801223242, | |
| "Caucasian": 0.9840304182509506, | |
| "Hispanic": 1.0, | |
| "Other": 1.0, | |
| "Asian": 1.0 | |
| }, | |
| "fpr": { | |
| "AfricanAmerican": 0.749028749028749, | |
| "Caucasian": 0.8343234955921809, | |
| "Hispanic": 0.6892857142857143, | |
| "Other": 0.8074866310160428, | |
| "Asian": 0.9166666666666666 | |
| }, | |
| "tpr_gap": 0.01596958174904939, | |
| "fpr_gap": 0.22738095238095235, | |
| "tpr_std": 0.0076601403202099785, | |
| "fpr_std": 0.07707000114790657, | |
| "passed": false | |
| }, | |
| "equal_opportunity": { | |
| "metric": "equal_opportunity", | |
| "definition": "TPR should be similar across groups", | |
| "tpr": { | |
| "AfricanAmerican": 0.9847094801223242, | |
| "Caucasian": 0.9840304182509506, | |
| "Hispanic": 1.0, | |
| "Other": 1.0, | |
| "Asian": 1.0 | |
| }, | |
| "max_tpr": 1.0, | |
| "min_tpr": 0.9840304182509506, | |
| "gap": 0.01596958174904939, | |
| "std": 0.0076601403202099785, | |
| "passed": true | |
| }, | |
| "all_passed": false | |
| }, | |
| "gender": { | |
| "demographic_parity": { | |
| "metric": "demographic_parity", | |
| "definition": "Intervention rate should be similar across groups", | |
| "intervention_rates": { | |
| "Male": 0.8443063012150325, | |
| "Female": 0.8254549896176866 | |
| }, | |
| "max_rate": 0.8443063012150325, | |
| "min_rate": 0.8254549896176866, | |
| "gap": 0.018851311597345877, | |
| "std": 0.009425655798672938, | |
| "passed": true | |
| }, | |
| "equalized_odds": { | |
| "metric": "equalized_odds", | |
| "definition": "TPR and FPR should be similar across groups", | |
| "tpr": { | |
| "Male": 0.9869621903520208, | |
| "Female": 0.9829242262540021 | |
| }, | |
| "fpr": { | |
| "Male": 0.8269687846617018, | |
| "Female": 0.8051034482758621 | |
| }, | |
| "tpr_gap": 0.004037964098018709, | |
| "fpr_gap": 0.02186533638583965, | |
| "tpr_std": 0.0020189820490093546, | |
| "fpr_std": 0.010932668192919826, | |
| "passed": true | |
| }, | |
| "equal_opportunity": { | |
| "metric": "equal_opportunity", | |
| "definition": "TPR should be similar across groups", | |
| "tpr": { | |
| "Male": 0.9869621903520208, | |
| "Female": 0.9829242262540021 | |
| }, | |
| "max_tpr": 0.9869621903520208, | |
| "min_tpr": 0.9829242262540021, | |
| "gap": 0.004037964098018709, | |
| "std": 0.0020189820490093546, | |
| "passed": true | |
| }, | |
| "all_passed": true | |
| }, | |
| "age": { | |
| "demographic_parity": { | |
| "metric": "demographic_parity", | |
| "definition": "Intervention rate should be similar across groups", | |
| "intervention_rates": { | |
| "[50-60)": 0.7212355212355213, | |
| "[20-30)": 0.7677165354330708, | |
| "[40-50)": 0.7301808066759388, | |
| "[70-80)": 0.8945744407302648, | |
| "[60-70)": 0.8530528775927549, | |
| "[80-90)": 0.9359625146427177, | |
| "[90-100)": 0.929384965831435, | |
| "[30-40)": 0.6666666666666666, | |
| "[10-20)": 0.5252525252525253 | |
| }, | |
| "max_rate": 0.9359625146427177, | |
| "min_rate": 0.5252525252525253, | |
| "gap": 0.41070998939019243, | |
| "std": 0.12863782814965471, | |
| "passed": false | |
| }, | |
| "equalized_odds": { | |
| "metric": "equalized_odds", | |
| "definition": "TPR and FPR should be similar across groups", | |
| "tpr": { | |
| "[50-60)": 0.9683257918552036, | |
| "[20-30)": 1.0, | |
| "[40-50)": 0.952054794520548, | |
| "[70-80)": 0.9852941176470589, | |
| "[60-70)": 0.9873096446700508, | |
| "[80-90)": 1.0, | |
| "[90-100)": 1.0, | |
| "[30-40)": 1.0, | |
| "[10-20)": 1.0 | |
| }, | |
| "fpr": { | |
| "[50-60)": 0.698184888138455, | |
| "[20-30)": 0.7318181818181818, | |
| "[40-50)": 0.705108359133127, | |
| "[70-80)": 0.8819220627014357, | |
| "[60-70)": 0.8355893034004622, | |
| "[80-90)": 0.9272727272727272, | |
| "[90-100)": 0.9184210526315789, | |
| "[30-40)": 0.6213389121338913, | |
| "[10-20)": 0.5104166666666666 | |
| }, | |
| "tpr_gap": 0.047945205479452024, | |
| "fpr_gap": 0.4168560606060606, | |
| "tpr_std": 0.016353127436580347, | |
| "fpr_std": 0.13449177122867922, | |
| "passed": false | |
| }, | |
| "equal_opportunity": { | |
| "metric": "equal_opportunity", | |
| "definition": "TPR should be similar across groups", | |
| "tpr": { | |
| "[50-60)": 0.9683257918552036, | |
| "[20-30)": 1.0, | |
| "[40-50)": 0.952054794520548, | |
| "[70-80)": 0.9852941176470589, | |
| "[60-70)": 0.9873096446700508, | |
| "[80-90)": 1.0, | |
| "[90-100)": 1.0, | |
| "[30-40)": 1.0, | |
| "[10-20)": 1.0 | |
| }, | |
| "max_tpr": 1.0, | |
| "min_tpr": 0.952054794520548, | |
| "gap": 0.047945205479452024, | |
| "std": 0.016353127436580347, | |
| "passed": true | |
| }, | |
| "all_passed": false | |
| } | |
| }, | |
| "bias_detected": true, | |
| "recommendations": [ | |
| "\u26a0\ufe0f Fairness violations detected. Consider group-specific thresholds or recalibration.", | |
| "Review group-specific metrics and statistical tests for detailed analysis.", | |
| "Consult with clinical and ethics teams before deployment.", | |
| "Phase 6 mitigation priority: HIGH" | |
| ], | |
| "phase6_inputs": { | |
| "requires_mitigation": true, | |
| "group_metrics_summary": { | |
| "race": [ | |
| { | |
| "attribute": "race", | |
| "group": "AfricanAmerican", | |
| "n_samples": 2901, | |
| "n_positive": 327, | |
| "positive_rate": 0.11271975180972078, | |
| "TP": 322, | |
| "FP": 1928, | |
| "TN": 646, | |
| "FN": 5, | |
| "accuracy": 0.333678042054464, | |
| "precision": 0.1431111111111111, | |
| "recall": 0.9847094801223242, | |
| "f1_score": 0.2499029879705083, | |
| "tpr": 0.9847094801223242, | |
| "fpr": 0.749028749028749, | |
| "tnr": 0.25097125097125095, | |
| "fnr": 0.01529051987767584, | |
| "intervention_rate": 0.7755946225439504, | |
| "roc_auc": 0.8262429042245556, | |
| "brier_score": 0.08627390938893648 | |
| }, | |
| { | |
| "attribute": "race", | |
| "group": "Caucasian", | |
| "n_samples": 11751, | |
| "n_positive": 1315, | |
| "positive_rate": 0.11190536975576547, | |
| "TP": 1294, | |
| "FP": 8707, | |
| "TN": 1729, | |
| "FN": 21, | |
| "accuracy": 0.2572547017275125, | |
| "precision": 0.12938706129387062, | |
| "recall": 0.9840304182509506, | |
| "f1_score": 0.2287027218098268, | |
| "tpr": 0.9840304182509506, | |
| "fpr": 0.8343234955921809, | |
| "tnr": 0.1656765044078191, | |
| "fnr": 0.01596958174904943, | |
| "intervention_rate": 0.8510765041273083, | |
| "roc_auc": 0.8011558410707598, | |
| "brier_score": 0.08747023215778671 | |
| }, | |
| { | |
| "attribute": "race", | |
| "group": "Hispanic", | |
| "n_samples": 320, | |
| "n_positive": 40, | |
| "positive_rate": 0.125, | |
| "TP": 40, | |
| "FP": 193, | |
| "TN": 87, | |
| "FN": 0, | |
| "accuracy": 0.396875, | |
| "precision": 0.17167381974248927, | |
| "recall": 1.0, | |
| "f1_score": 0.29304029304029305, | |
| "tpr": 1.0, | |
| "fpr": 0.6892857142857143, | |
| "tnr": 0.3107142857142857, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.728125, | |
| "roc_auc": 0.8291071428571428, | |
| "brier_score": 0.09583137143869011 | |
| }, | |
| { | |
| "attribute": "race", | |
| "group": "Other", | |
| "n_samples": 202, | |
| "n_positive": 15, | |
| "positive_rate": 0.07425742574257425, | |
| "TP": 15, | |
| "FP": 151, | |
| "TN": 36, | |
| "FN": 0, | |
| "accuracy": 0.2524752475247525, | |
| "precision": 0.09036144578313253, | |
| "recall": 1.0, | |
| "f1_score": 0.16574585635359115, | |
| "tpr": 1.0, | |
| "fpr": 0.8074866310160428, | |
| "tnr": 0.1925133689839572, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.8217821782178217, | |
| "roc_auc": 0.8181818181818182, | |
| "brier_score": 0.061681916452139746 | |
| }, | |
| { | |
| "attribute": "race", | |
| "group": "Asian", | |
| "n_samples": 91, | |
| "n_positive": 7, | |
| "positive_rate": 0.07692307692307693, | |
| "TP": 7, | |
| "FP": 77, | |
| "TN": 7, | |
| "FN": 0, | |
| "accuracy": 0.15384615384615385, | |
| "precision": 0.08333333333333333, | |
| "recall": 1.0, | |
| "f1_score": 0.15384615384615385, | |
| "tpr": 1.0, | |
| "fpr": 0.9166666666666666, | |
| "tnr": 0.08333333333333333, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.9230769230769231, | |
| "roc_auc": 0.8673469387755102, | |
| "brier_score": 0.057751201315052084 | |
| } | |
| ], | |
| "gender": [ | |
| { | |
| "attribute": "gender", | |
| "group": "Male", | |
| "n_samples": 7078, | |
| "n_positive": 767, | |
| "positive_rate": 0.10836394461712348, | |
| "TP": 757, | |
| "FP": 5219, | |
| "TN": 1092, | |
| "FN": 10, | |
| "accuracy": 0.2612319864368466, | |
| "precision": 0.12667336010709504, | |
| "recall": 0.9869621903520208, | |
| "f1_score": 0.22452914133175145, | |
| "tpr": 0.9869621903520208, | |
| "fpr": 0.8269687846617018, | |
| "tnr": 0.1730312153382982, | |
| "fnr": 0.01303780964797914, | |
| "intervention_rate": 0.8443063012150325, | |
| "roc_auc": 0.8003597534736332, | |
| "brier_score": 0.0853545345056348 | |
| }, | |
| { | |
| "attribute": "gender", | |
| "group": "Female", | |
| "n_samples": 8187, | |
| "n_positive": 937, | |
| "positive_rate": 0.1144497373885428, | |
| "TP": 921, | |
| "FP": 5837, | |
| "TN": 1413, | |
| "FN": 16, | |
| "accuracy": 0.285086112128985, | |
| "precision": 0.13628292394199468, | |
| "recall": 0.9829242262540021, | |
| "f1_score": 0.23937621832358674, | |
| "tpr": 0.9829242262540021, | |
| "fpr": 0.8051034482758621, | |
| "tnr": 0.19489655172413792, | |
| "fnr": 0.017075773745997867, | |
| "intervention_rate": 0.8254549896176866, | |
| "roc_auc": 0.8130260184742206, | |
| "brier_score": 0.08823562468498301 | |
| } | |
| ], | |
| "age": [ | |
| { | |
| "attribute": "age", | |
| "group": "[50-60)", | |
| "n_samples": 2590, | |
| "n_positive": 221, | |
| "positive_rate": 0.08532818532818533, | |
| "TP": 214, | |
| "FP": 1654, | |
| "TN": 715, | |
| "FN": 7, | |
| "accuracy": 0.3586872586872587, | |
| "precision": 0.1145610278372591, | |
| "recall": 0.9683257918552036, | |
| "f1_score": 0.20488271900430827, | |
| "tpr": 0.9683257918552036, | |
| "fpr": 0.698184888138455, | |
| "tnr": 0.30181511186154497, | |
| "fnr": 0.03167420814479638, | |
| "intervention_rate": 0.7212355212355213, | |
| "roc_auc": 0.8363515162859637, | |
| "brier_score": 0.06721122767044026 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[20-30)", | |
| "n_samples": 254, | |
| "n_positive": 34, | |
| "positive_rate": 0.13385826771653545, | |
| "TP": 34, | |
| "FP": 161, | |
| "TN": 59, | |
| "FN": 0, | |
| "accuracy": 0.3661417322834646, | |
| "precision": 0.17435897435897435, | |
| "recall": 1.0, | |
| "f1_score": 0.296943231441048, | |
| "tpr": 1.0, | |
| "fpr": 0.7318181818181818, | |
| "tnr": 0.2681818181818182, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.7677165354330708, | |
| "roc_auc": 0.8954545454545454, | |
| "brier_score": 0.08474998571634562 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[40-50)", | |
| "n_samples": 1438, | |
| "n_positive": 146, | |
| "positive_rate": 0.10152990264255911, | |
| "TP": 139, | |
| "FP": 911, | |
| "TN": 381, | |
| "FN": 7, | |
| "accuracy": 0.3616133518776078, | |
| "precision": 0.13238095238095238, | |
| "recall": 0.952054794520548, | |
| "f1_score": 0.2324414715719064, | |
| "tpr": 0.952054794520548, | |
| "fpr": 0.705108359133127, | |
| "tnr": 0.2948916408668731, | |
| "fnr": 0.04794520547945205, | |
| "intervention_rate": 0.7301808066759388, | |
| "roc_auc": 0.8127995250010602, | |
| "brier_score": 0.079934175740362 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[70-80)", | |
| "n_samples": 3889, | |
| "n_positive": 476, | |
| "positive_rate": 0.12239650295705837, | |
| "TP": 469, | |
| "FP": 3010, | |
| "TN": 403, | |
| "FN": 7, | |
| "accuracy": 0.22422216508099768, | |
| "precision": 0.13480885311871227, | |
| "recall": 0.9852941176470589, | |
| "f1_score": 0.23716814159292035, | |
| "tpr": 0.9852941176470589, | |
| "fpr": 0.8819220627014357, | |
| "tnr": 0.1180779372985643, | |
| "fnr": 0.014705882352941176, | |
| "intervention_rate": 0.8945744407302648, | |
| "roc_auc": 0.772059131299751, | |
| "brier_score": 0.09720942286539178 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[60-70)", | |
| "n_samples": 3423, | |
| "n_positive": 394, | |
| "positive_rate": 0.11510371019573473, | |
| "TP": 389, | |
| "FP": 2531, | |
| "TN": 498, | |
| "FN": 5, | |
| "accuracy": 0.2591294186386211, | |
| "precision": 0.13321917808219177, | |
| "recall": 0.9873096446700508, | |
| "f1_score": 0.23476161738080867, | |
| "tpr": 0.9873096446700508, | |
| "fpr": 0.8355893034004622, | |
| "tnr": 0.1644106965995378, | |
| "fnr": 0.012690355329949238, | |
| "intervention_rate": 0.8530528775927549, | |
| "roc_auc": 0.8185945337205658, | |
| "brier_score": 0.0876884182999776 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[80-90)", | |
| "n_samples": 2561, | |
| "n_positive": 306, | |
| "positive_rate": 0.11948457633736821, | |
| "TP": 306, | |
| "FP": 2091, | |
| "TN": 164, | |
| "FN": 0, | |
| "accuracy": 0.18352206169465052, | |
| "precision": 0.1276595744680851, | |
| "recall": 1.0, | |
| "f1_score": 0.22641509433962262, | |
| "tpr": 1.0, | |
| "fpr": 0.9272727272727272, | |
| "tnr": 0.07272727272727272, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.9359625146427177, | |
| "roc_auc": 0.7776183644189383, | |
| "brier_score": 0.09528812202797123 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[90-100)", | |
| "n_samples": 439, | |
| "n_positive": 59, | |
| "positive_rate": 0.13439635535307518, | |
| "TP": 59, | |
| "FP": 349, | |
| "TN": 31, | |
| "FN": 0, | |
| "accuracy": 0.20501138952164008, | |
| "precision": 0.14460784313725492, | |
| "recall": 1.0, | |
| "f1_score": 0.25267665952890794, | |
| "tpr": 1.0, | |
| "fpr": 0.9184210526315789, | |
| "tnr": 0.08157894736842106, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.929384965831435, | |
| "roc_auc": 0.7963871543264942, | |
| "brier_score": 0.09999089048596788 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[30-40)", | |
| "n_samples": 543, | |
| "n_positive": 65, | |
| "positive_rate": 0.11970534069981584, | |
| "TP": 65, | |
| "FP": 297, | |
| "TN": 181, | |
| "FN": 0, | |
| "accuracy": 0.4530386740331492, | |
| "precision": 0.17955801104972377, | |
| "recall": 1.0, | |
| "f1_score": 0.30444964871194385, | |
| "tpr": 1.0, | |
| "fpr": 0.6213389121338913, | |
| "tnr": 0.3786610878661088, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.6666666666666666, | |
| "roc_auc": 0.8595751528805922, | |
| "brier_score": 0.0868983446028739 | |
| }, | |
| { | |
| "attribute": "age", | |
| "group": "[10-20)", | |
| "n_samples": 99, | |
| "n_positive": 3, | |
| "positive_rate": 0.030303030303030304, | |
| "TP": 3, | |
| "FP": 49, | |
| "TN": 47, | |
| "FN": 0, | |
| "accuracy": 0.5050505050505051, | |
| "precision": 0.057692307692307696, | |
| "recall": 1.0, | |
| "f1_score": 0.1090909090909091, | |
| "tpr": 1.0, | |
| "fpr": 0.5104166666666666, | |
| "tnr": 0.4895833333333333, | |
| "fnr": 0.0, | |
| "intervention_rate": 0.5252525252525253, | |
| "roc_auc": 0.9479166666666666, | |
| "brier_score": 0.026620728050508028 | |
| } | |
| ] | |
| }, | |
| "worst_violations": [ | |
| { | |
| "attribute": "race", | |
| "violations": [ | |
| { | |
| "metric": "demographic_parity", | |
| "gap": 0.1949519230769231, | |
| "threshold": 0.05 | |
| }, | |
| { | |
| "metric": "equalized_odds", | |
| "tpr_gap": 0.01596958174904939, | |
| "fpr_gap": 0.22738095238095235, | |
| "threshold": 0.05 | |
| } | |
| ] | |
| }, | |
| { | |
| "attribute": "age", | |
| "violations": [ | |
| { | |
| "metric": "demographic_parity", | |
| "gap": 0.41070998939019243, | |
| "threshold": 0.05 | |
| }, | |
| { | |
| "metric": "equalized_odds", | |
| "tpr_gap": 0.047945205479452024, | |
| "fpr_gap": 0.4168560606060606, | |
| "threshold": 0.05 | |
| } | |
| ] | |
| } | |
| ], | |
| "mitigation_priority": "high" | |
| } | |
| } | |
| }, | |
| "mitigation": { | |
| "status": "applied", | |
| "output_dir": "/kaggle/working/hospital-readmission-risk/phase-5-fairness-assessment-mitigation/outputs/logistic_regression/mitigation", | |
| "impact": { | |
| "phase": 6, | |
| "mitigation_strategy": "equalized_odds", | |
| "baseline_metrics": { | |
| "global_threshold": 0.028621862186218622, | |
| "overall_metrics": { | |
| "accuracy": 0.2740255486406813, | |
| "precision": 0.1317732055913303, | |
| "recall": 0.9847417840375586, | |
| "tpr": 0.9847417840375586, | |
| "fpr": 0.8152791092102353, | |
| "f1_score": 0.2324421665050561, | |
| "roc_auc": 0.8075357980787905, | |
| "intervention_rate": 0.8341958729118899, | |
| "n_interventions": 12734, | |
| "tp": 1678, | |
| "fp": 11056, | |
| "tn": 2505, | |
| "fn": 26 | |
| }, | |
| "group_metrics": { | |
| "race": { | |
| "AfricanAmerican": { | |
| "n_samples": 2901, | |
| "tpr": 0.9847094801223242, | |
| "fpr": 0.749028749028749, | |
| "precision": 0.1431111111111111, | |
| "recall": 0.9847094801223242, | |
| "intervention_rate": 0.7755946225439504 | |
| }, | |
| "Caucasian": { | |
| "n_samples": 11751, | |
| "tpr": 0.9840304182509506, | |
| "fpr": 0.8343234955921809, | |
| "precision": 0.12938706129387062, | |
| "recall": 0.9840304182509506, | |
| "intervention_rate": 0.8510765041273083 | |
| }, | |
| "Hispanic": { | |
| "n_samples": 320, | |
| "tpr": 1.0, | |
| "fpr": 0.6892857142857143, | |
| "precision": 0.17167381974248927, | |
| "recall": 1.0, | |
| "intervention_rate": 0.728125 | |
| }, | |
| "Other": { | |
| "n_samples": 202, | |
| "tpr": 1.0, | |
| "fpr": 0.8074866310160428, | |
| "precision": 0.09036144578313253, | |
| "recall": 1.0, | |
| "intervention_rate": 0.8217821782178217 | |
| }, | |
| "Asian": { | |
| "n_samples": 91, | |
| "tpr": 1.0, | |
| "fpr": 0.9166666666666666, | |
| "precision": 0.08333333333333333, | |
| "recall": 1.0, | |
| "intervention_rate": 0.9230769230769231 | |
| } | |
| }, | |
| "gender": { | |
| "Male": { | |
| "n_samples": 7078, | |
| "tpr": 0.9869621903520208, | |
| "fpr": 0.8269687846617018, | |
| "precision": 0.12667336010709504, | |
| "recall": 0.9869621903520208, | |
| "intervention_rate": 0.8443063012150325 | |
| }, | |
| "Female": { | |
| "n_samples": 8187, | |
| "tpr": 0.9829242262540021, | |
| "fpr": 0.8051034482758621, | |
| "precision": 0.13628292394199468, | |
| "recall": 0.9829242262540021, | |
| "intervention_rate": 0.8254549896176866 | |
| } | |
| }, | |
| "age": { | |
| "[50-60)": { | |
| "n_samples": 2590, | |
| "tpr": 0.9683257918552036, | |
| "fpr": 0.698184888138455, | |
| "precision": 0.1145610278372591, | |
| "recall": 0.9683257918552036, | |
| "intervention_rate": 0.7212355212355213 | |
| }, | |
| "[20-30)": { | |
| "n_samples": 254, | |
| "tpr": 1.0, | |
| "fpr": 0.7318181818181818, | |
| "precision": 0.17435897435897435, | |
| "recall": 1.0, | |
| "intervention_rate": 0.7677165354330708 | |
| }, | |
| "[40-50)": { | |
| "n_samples": 1438, | |
| "tpr": 0.952054794520548, | |
| "fpr": 0.705108359133127, | |
| "precision": 0.13238095238095238, | |
| "recall": 0.952054794520548, | |
| "intervention_rate": 0.7301808066759388 | |
| }, | |
| "[70-80)": { | |
| "n_samples": 3889, | |
| "tpr": 0.9852941176470589, | |
| "fpr": 0.8819220627014357, | |
| "precision": 0.13480885311871227, | |
| "recall": 0.9852941176470589, | |
| "intervention_rate": 0.8945744407302648 | |
| }, | |
| "[60-70)": { | |
| "n_samples": 3423, | |
| "tpr": 0.9873096446700508, | |
| "fpr": 0.8355893034004622, | |
| "precision": 0.13321917808219177, | |
| "recall": 0.9873096446700508, | |
| "intervention_rate": 0.8530528775927549 | |
| }, | |
| "[80-90)": { | |
| "n_samples": 2561, | |
| "tpr": 1.0, | |
| "fpr": 0.9272727272727272, | |
| "precision": 0.1276595744680851, | |
| "recall": 1.0, | |
| "intervention_rate": 0.9359625146427177 | |
| }, | |
| "[90-100)": { | |
| "n_samples": 439, | |
| "tpr": 1.0, | |
| "fpr": 0.9184210526315789, | |
| "precision": 0.14460784313725492, | |
| "recall": 1.0, | |
| "intervention_rate": 0.929384965831435 | |
| }, | |
| "[30-40)": { | |
| "n_samples": 543, | |
| "tpr": 1.0, | |
| "fpr": 0.6213389121338913, | |
| "precision": 0.17955801104972377, | |
| "recall": 1.0, | |
| "intervention_rate": 0.6666666666666666 | |
| }, | |
| "[10-20)": { | |
| "n_samples": 99, | |
| "tpr": 1.0, | |
| "fpr": 0.5104166666666666, | |
| "precision": 0.057692307692307696, | |
| "recall": 1.0, | |
| "intervention_rate": 0.5252525252525253 | |
| }, | |
| "[0-10)": { | |
| "n_samples": 29, | |
| "tpr": 0.0, | |
| "fpr": 0.10344827586206896, | |
| "precision": 0.0, | |
| "recall": 0.0, | |
| "intervention_rate": 0.10344827586206896 | |
| } | |
| } | |
| }, | |
| "fairness_metrics": { | |
| "race": { | |
| "tpr_gap": 0.01596958174904939, | |
| "fpr_gap": 0.22738095238095235, | |
| "intervention_rate_gap": 0.1949519230769231, | |
| "tpr_max": 1.0, | |
| "tpr_min": 0.9840304182509506, | |
| "fpr_max": 0.9166666666666666, | |
| "fpr_min": 0.6892857142857143 | |
| }, | |
| "gender": { | |
| "tpr_gap": 0.004037964098018709, | |
| "fpr_gap": 0.02186533638583965, | |
| "intervention_rate_gap": 0.018851311597345877, | |
| "tpr_max": 0.9869621903520208, | |
| "tpr_min": 0.9829242262540021, | |
| "fpr_max": 0.8269687846617018, | |
| "fpr_min": 0.8051034482758621 | |
| }, | |
| "age": { | |
| "tpr_gap": 1.0, | |
| "fpr_gap": 0.8238244514106583, | |
| "intervention_rate_gap": 0.8325142387806488, | |
| "tpr_max": 1.0, | |
| "tpr_min": 0.0, | |
| "fpr_max": 0.9272727272727272, | |
| "fpr_min": 0.10344827586206896 | |
| } | |
| }, | |
| "roi_metrics": { | |
| "expected_value": 18413000, | |
| "net_benefit": 18413000, | |
| "cost_per_intervention": 500, | |
| "benefit_per_true_positive": 14500 | |
| } | |
| }, | |
| "mitigated_metrics": { | |
| "group_thresholds": { | |
| "race": { | |
| "AfricanAmerican": { | |
| "threshold": 0.026465646564656435, | |
| "tpr": 0.9908256880733946, | |
| "fpr": 0.817016317016317, | |
| "tpr_gap": 0.006083904035835919, | |
| "fpr_gap": 0.0017372078060817753, | |
| "score": 0.007821111841917694, | |
| "n_samples": 2901, | |
| "improvement": true | |
| }, | |
| "Caucasian": { | |
| "threshold": 0.029209920992099174, | |
| "tpr": 0.9787072243346008, | |
| "fpr": 0.8160214641625144, | |
| "tpr_gap": 0.0060345597029578535, | |
| "fpr_gap": 0.000742354952279145, | |
| "score": 0.0067769146552369985, | |
| "n_samples": 11751, | |
| "improvement": true | |
| }, | |
| "Hispanic": { | |
| "threshold": 0.025681568156815653, | |
| "tpr": 1.0, | |
| "fpr": 0.8178571428571428, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.0025780336469075626, | |
| "score": 0.01783624960934893, | |
| "n_samples": 320, | |
| "improvement": true | |
| }, | |
| "Other": { | |
| "threshold": 0.02852385238523849, | |
| "tpr": 1.0, | |
| "fpr": 0.8181818181818182, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.002902708971582957, | |
| "score": 0.018160924934024325, | |
| "n_samples": 202, | |
| "improvement": true | |
| }, | |
| "Asian": { | |
| "threshold": 0.032346234623462304, | |
| "tpr": 1.0, | |
| "fpr": 0.8095238095238095, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.005755299686425741, | |
| "score": 0.02101351564886711, | |
| "n_samples": 91, | |
| "improvement": true | |
| } | |
| }, | |
| "gender": { | |
| "Male": { | |
| "threshold": 0.02891589158915888, | |
| "tpr": 0.984354628422425, | |
| "fpr": 0.8173031215338298, | |
| "tpr_gap": 0.0003871556151335831, | |
| "fpr_gap": 0.002024012323594504, | |
| "score": 0.002411167938728087, | |
| "n_samples": 7078, | |
| "improvement": true | |
| }, | |
| "Female": { | |
| "threshold": 0.028229822982298196, | |
| "tpr": 0.9850586979722519, | |
| "fpr": 0.8164137931034483, | |
| "tpr_gap": 0.00031691393469324147, | |
| "fpr_gap": 0.0011346838932130288, | |
| "score": 0.0014515978279062702, | |
| "n_samples": 8187, | |
| "improvement": true | |
| } | |
| }, | |
| "age": { | |
| "[50-60)": { | |
| "threshold": 0.025975597559755946, | |
| "tpr": 0.9819004524886877, | |
| "fpr": 0.8151118615449556, | |
| "tpr_gap": 0.0028413315488708957, | |
| "fpr_gap": 0.00016724766527964618, | |
| "score": 0.003008579214150542, | |
| "n_samples": 2590, | |
| "improvement": true | |
| }, | |
| "[20-30)": { | |
| "threshold": 0.025975597559755946, | |
| "tpr": 1.0, | |
| "fpr": 0.8136363636363636, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.0016427455738716734, | |
| "score": 0.01690096153631304, | |
| "n_samples": 254, | |
| "improvement": true | |
| }, | |
| "[40-50)": { | |
| "threshold": 0.026073607360736044, | |
| "tpr": 0.958904109589041, | |
| "fpr": 0.8142414860681114, | |
| "tpr_gap": 0.02583767444851759, | |
| "fpr_gap": 0.0010376231421238558, | |
| "score": 0.026875297590641445, | |
| "n_samples": 1438, | |
| "improvement": true | |
| }, | |
| "[70-80)": { | |
| "threshold": 0.030876087608760837, | |
| "tpr": 0.9600840336134454, | |
| "fpr": 0.8157046586580721, | |
| "tpr_gap": 0.024657750424113223, | |
| "fpr_gap": 0.0004255494478367927, | |
| "score": 0.025083299871950016, | |
| "n_samples": 3889, | |
| "improvement": true | |
| }, | |
| "[60-70)": { | |
| "threshold": 0.029307930793079272, | |
| "tpr": 0.9847715736040609, | |
| "fpr": 0.8154506437768241, | |
| "tpr_gap": 2.9789566502236298e-05, | |
| "fpr_gap": 0.00017153456658880462, | |
| "score": 0.0002013241330910409, | |
| "n_samples": 3423, | |
| "improvement": true | |
| }, | |
| "[80-90)": { | |
| "threshold": 0.03342434243424338, | |
| "tpr": 0.9705882352941176, | |
| "fpr": 0.8159645232815964, | |
| "tpr_gap": 0.014153548743440991, | |
| "fpr_gap": 0.0006854140713611345, | |
| "score": 0.014838962814802126, | |
| "n_samples": 2561, | |
| "improvement": true | |
| }, | |
| "[90-100)": { | |
| "threshold": 0.0326402640264026, | |
| "tpr": 0.9661016949152542, | |
| "fpr": 0.8105263157894737, | |
| "tpr_gap": 0.018640089122304415, | |
| "fpr_gap": 0.004752793420761625, | |
| "score": 0.02339288254306604, | |
| "n_samples": 439, | |
| "improvement": true | |
| }, | |
| "[30-40)": { | |
| "threshold": 0.024211421142114185, | |
| "tpr": 1.0, | |
| "fpr": 0.8138075313807531, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.0014715778294821513, | |
| "score": 0.01672979379192352, | |
| "n_samples": 543, | |
| "improvement": true | |
| }, | |
| "[10-20)": { | |
| "threshold": 0.022545254525452522, | |
| "tpr": 1.0, | |
| "fpr": 0.8125, | |
| "tpr_gap": 0.015258215962441368, | |
| "fpr_gap": 0.0027791092102352755, | |
| "score": 0.018037325172676644, | |
| "n_samples": 99, | |
| "improvement": true | |
| } | |
| } | |
| }, | |
| "overall_metrics": { | |
| "accuracy": 0.2741565673108418, | |
| "precision": 0.13063346990230065, | |
| "recall": 0.9730046948356808, | |
| "tpr": 0.9730046948356808, | |
| "fpr": 0.8136568099697662, | |
| "f1_score": 0.23034176160044456, | |
| "roc_auc": 0.8075357980787905, | |
| "intervention_rate": 0.8314444808385195, | |
| "n_interventions": 12692, | |
| "tp": 1658, | |
| "fp": 11034, | |
| "tn": 2527, | |
| "fn": 46 | |
| }, | |
| "group_metrics": { | |
| "race": { | |
| "AfricanAmerican": { | |
| "n_samples": 2901, | |
| "tpr": 0.9847094801223242, | |
| "fpr": 0.7676767676767676, | |
| "precision": 0.14012184508268058, | |
| "recall": 0.9847094801223242, | |
| "intervention_rate": 0.7921406411582212 | |
| }, | |
| "Caucasian": { | |
| "n_samples": 11751, | |
| "tpr": 0.9695817490494296, | |
| "fpr": 0.826274434649291, | |
| "precision": 0.1288139017983431, | |
| "recall": 0.9695817490494296, | |
| "intervention_rate": 0.8423112926559442 | |
| }, | |
| "Hispanic": { | |
| "n_samples": 320, | |
| "tpr": 1.0, | |
| "fpr": 0.7285714285714285, | |
| "precision": 0.16393442622950818, | |
| "recall": 1.0, | |
| "intervention_rate": 0.7625 | |
| }, | |
| "Other": { | |
| "n_samples": 202, | |
| "tpr": 0.9333333333333333, | |
| "fpr": 0.8288770053475936, | |
| "precision": 0.08284023668639054, | |
| "recall": 0.9333333333333333, | |
| "intervention_rate": 0.8366336633663366 | |
| }, | |
| "Asian": { | |
| "n_samples": 91, | |
| "tpr": 1.0, | |
| "fpr": 0.9047619047619048, | |
| "precision": 0.08433734939759036, | |
| "recall": 1.0, | |
| "intervention_rate": 0.9120879120879121 | |
| } | |
| }, | |
| "gender": { | |
| "Male": { | |
| "n_samples": 7078, | |
| "tpr": 0.9739243807040417, | |
| "fpr": 0.8333069244176834, | |
| "precision": 0.12437562437562437, | |
| "recall": 0.9739243807040417, | |
| "intervention_rate": 0.8485447866628991 | |
| }, | |
| "Female": { | |
| "n_samples": 8187, | |
| "tpr": 0.9722518676627535, | |
| "fpr": 0.7965517241379311, | |
| "precision": 0.1362548609033802, | |
| "recall": 0.9722518676627535, | |
| "intervention_rate": 0.8166605594234763 | |
| } | |
| }, | |
| "age": { | |
| "[50-60)": { | |
| "n_samples": 2590, | |
| "tpr": 0.9819004524886877, | |
| "fpr": 0.8151118615449556, | |
| "precision": 0.10102420856610801, | |
| "recall": 0.9819004524886877, | |
| "intervention_rate": 0.8293436293436294 | |
| }, | |
| "[20-30)": { | |
| "n_samples": 254, | |
| "tpr": 1.0, | |
| "fpr": 0.8136363636363636, | |
| "precision": 0.1596244131455399, | |
| "recall": 1.0, | |
| "intervention_rate": 0.8385826771653543 | |
| }, | |
| "[40-50)": { | |
| "n_samples": 1438, | |
| "tpr": 0.958904109589041, | |
| "fpr": 0.8142414860681114, | |
| "precision": 0.1174496644295302, | |
| "recall": 0.958904109589041, | |
| "intervention_rate": 0.8289290681502086 | |
| }, | |
| "[70-80)": { | |
| "n_samples": 3889, | |
| "tpr": 0.9600840336134454, | |
| "fpr": 0.8157046586580721, | |
| "precision": 0.1410058623881518, | |
| "recall": 0.9600840336134454, | |
| "intervention_rate": 0.8333761892517356 | |
| }, | |
| "[60-70)": { | |
| "n_samples": 3423, | |
| "tpr": 0.9847715736040609, | |
| "fpr": 0.8154506437768241, | |
| "precision": 0.1357592722183345, | |
| "recall": 0.9847715736040609, | |
| "intervention_rate": 0.8349401110137307 | |
| }, | |
| "[80-90)": { | |
| "n_samples": 2561, | |
| "tpr": 0.9705882352941176, | |
| "fpr": 0.8159645232815964, | |
| "precision": 0.13897987833411324, | |
| "recall": 0.9705882352941176, | |
| "intervention_rate": 0.8344396720031237 | |
| }, | |
| "[90-100)": { | |
| "n_samples": 439, | |
| "tpr": 0.9661016949152542, | |
| "fpr": 0.8105263157894737, | |
| "precision": 0.15616438356164383, | |
| "recall": 0.9661016949152542, | |
| "intervention_rate": 0.8314350797266514 | |
| }, | |
| "[30-40)": { | |
| "n_samples": 543, | |
| "tpr": 1.0, | |
| "fpr": 0.8138075313807531, | |
| "precision": 0.14317180616740088, | |
| "recall": 1.0, | |
| "intervention_rate": 0.8360957642725598 | |
| }, | |
| "[10-20)": { | |
| "n_samples": 99, | |
| "tpr": 1.0, | |
| "fpr": 0.8125, | |
| "precision": 0.037037037037037035, | |
| "recall": 1.0, | |
| "intervention_rate": 0.8181818181818182 | |
| }, | |
| "[0-10)": { | |
| "n_samples": 29, | |
| "tpr": 0.0, | |
| "fpr": 0.10344827586206896, | |
| "precision": 0.0, | |
| "recall": 0.0, | |
| "intervention_rate": 0.10344827586206896 | |
| } | |
| } | |
| }, | |
| "fairness_metrics": { | |
| "race": { | |
| "tpr_gap": 0.06666666666666665, | |
| "fpr_gap": 0.17619047619047623, | |
| "intervention_rate_gap": 0.1495879120879121, | |
| "tpr_max": 1.0, | |
| "tpr_min": 0.9333333333333333, | |
| "fpr_max": 0.9047619047619048, | |
| "fpr_min": 0.7285714285714285 | |
| }, | |
| "gender": { | |
| "tpr_gap": 0.0016725130412882372, | |
| "fpr_gap": 0.03675520027975232, | |
| "intervention_rate_gap": 0.03188422723942286, | |
| "tpr_max": 0.9739243807040417, | |
| "tpr_min": 0.9722518676627535, | |
| "fpr_max": 0.8333069244176834, | |
| "fpr_min": 0.7965517241379311 | |
| }, | |
| "age": { | |
| "tpr_gap": 1.0, | |
| "fpr_gap": 0.7125162474195275, | |
| "intervention_rate_gap": 0.7351344013032853, | |
| "tpr_max": 1.0, | |
| "tpr_min": 0.0, | |
| "fpr_max": 0.8159645232815964, | |
| "fpr_min": 0.10344827586206896 | |
| } | |
| }, | |
| "roi_metrics": { | |
| "expected_value": 17834000, | |
| "net_benefit": 17834000, | |
| "cost_per_intervention": 500, | |
| "benefit_per_true_positive": 14500 | |
| } | |
| }, | |
| "improvements": { | |
| "fairness_improvements": { | |
| "race": { | |
| "tpr_gap_before": 0.01596958174904939, | |
| "tpr_gap_after": 0.06666666666666665, | |
| "tpr_gap_reduction": -0.05069708491761726, | |
| "tpr_gap_reduction_pct": -317.4603174603185, | |
| "fpr_gap_before": 0.22738095238095235, | |
| "fpr_gap_after": 0.17619047619047623, | |
| "fpr_gap_reduction": 0.05119047619047612, | |
| "fpr_gap_reduction_pct": 22.513089005235575 | |
| }, | |
| "gender": { | |
| "tpr_gap_before": 0.004037964098018709, | |
| "tpr_gap_after": 0.0016725130412882372, | |
| "tpr_gap_reduction": 0.002365451056730472, | |
| "tpr_gap_reduction_pct": 58.580289455548105, | |
| "fpr_gap_before": 0.02186533638583965, | |
| "fpr_gap_after": 0.03675520027975232, | |
| "fpr_gap_reduction": -0.01488986389391267, | |
| "fpr_gap_reduction_pct": -68.09803257157108 | |
| }, | |
| "age": { | |
| "tpr_gap_before": 1.0, | |
| "tpr_gap_after": 1.0, | |
| "tpr_gap_reduction": 0.0, | |
| "tpr_gap_reduction_pct": 0.0, | |
| "fpr_gap_before": 0.8238244514106583, | |
| "fpr_gap_after": 0.7125162474195275, | |
| "fpr_gap_reduction": 0.11130820399113084, | |
| "fpr_gap_reduction_pct": 13.511155659501803 | |
| } | |
| }, | |
| "performance_changes": { | |
| "tpr_change": -0.011737089201877882, | |
| "fpr_change": -0.0016222992404690828, | |
| "accuracy_change": 0.00013101867016052093, | |
| "roc_auc_change": 0.0, | |
| "intervention_rate_change": -0.0027513920733703845 | |
| }, | |
| "roi_changes": { | |
| "expected_value_before": 18413000, | |
| "expected_value_after": 17834000, | |
| "expected_value_change": -579000, | |
| "roi_reduction_pct": 3.1445174604898716 | |
| }, | |
| "summary": { | |
| "avg_fairness_improvement_pct": -48.49230265193402, | |
| "performance_drop_acceptable": true, | |
| "roi_reduction_acceptable": true, | |
| "fairness_targets_met": false | |
| } | |
| }, | |
| "summary": { | |
| "fairness_targets_met": false, | |
| "performance_drop_acceptable": true, | |
| "roi_reduction_acceptable": true, | |
| "recommended_for_deployment": false | |
| } | |
| } | |
| } | |
| } |