Upload aggregate.py with huggingface_hub

aggregate.py

@@ -4,11 +4,66 @@ import argparse
 import contextlib
 import csv
 import json
-from collections import defaultdict
+import logging
+from collections import Counter, defaultdict
 from pathlib import Path
 
 import numpy as np
 
+logger = logging.getLogger(__name__)
+
+
+def write_validation_results_csv(
+    output_path: Path,
+    pairs: list[dict],
+    reviews_by_reviewer: dict[str, list[dict]],
+    exclusions: dict,
+) -> None:
+    pair_strata = {idx: classify_transformation(pair) for idx, pair in enumerate(pairs)}
+    implausible_any = set(exclusions["implausible_any_rater"])
+    implausible_majority = set(exclusions["implausible_majority"])
+    disputed_quality = set(exclusions["disputed_quality"])
+    final_excluded = set(exclusions["all_excluded"])
+
+    fieldnames = [
+        "pair_index",
+        "reviewer_id",
+        "clinically_plausible",
+        "pathology_preserved",
+        "quality_score",
+        "comments",
+        "timestamp",
+        "transform_type",
+        "excluded_implausible_any",
+        "excluded_implausible_majority",
+        "excluded_disputed_quality",
+        "excluded_final",
+    ]
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        writer.writeheader()
+        for reviewer, rows in sorted(reviews_by_reviewer.items()):
+            for row in sorted(rows, key=lambda r: int(r["pair_index"])):
+                pair_index = int(row["pair_index"])
+                writer.writerow(
+                    {
+                        "pair_index": pair_index,
+                        "reviewer_id": row.get("reviewer_id", reviewer),
+                        "clinically_plausible": row.get("clinically_plausible", ""),
+                        "pathology_preserved": row.get("pathology_preserved", ""),
+                        "quality_score": row.get("quality_score", ""),
+                        "comments": row.get("comments", ""),
+                        "timestamp": row.get("timestamp", ""),
+                        "transform_type": pair_strata.get(pair_index, "unknown"),
+                        "excluded_implausible_any": int(pair_index in implausible_any),
+                        "excluded_implausible_majority": int(pair_index in implausible_majority),
+                        "excluded_disputed_quality": int(pair_index in disputed_quality),
+                        "excluded_final": int(pair_index in final_excluded),
+                    }
+                )
+
 
 def load_all_reviews(results_dir: Path) -> dict[str, list[dict]]:
     reviews_by_reviewer: dict[str, list[dict]] = {}
@@ -83,6 +138,74 @@ def fleiss_kappa(matrix: np.ndarray, categories: list[int | float]) -> float:
     return float((p_bar - p_e) / (1.0 - p_e))
 
 
+def build_stratum_rating_matrix(
+    reviews_by_reviewer: dict[str, list[dict]],
+    field: str,
+    indices: list[int],
+    value_map: dict[str, float] | None = None,
+) -> np.ndarray:
+    index_set = set(indices)
+    reviewers = sorted(reviews_by_reviewer.keys())
+    idx_to_row = {idx: row_num for row_num, idx in enumerate(sorted(indices))}
+    matrix = np.full((len(indices), len(reviewers)), np.nan)
+    for j, reviewer in enumerate(reviewers):
+        for row in reviews_by_reviewer[reviewer]:
+            pair_idx = int(row["pair_index"])
+            if pair_idx in index_set and row.get(field):
+                val = row[field]
+                matrix_row = idx_to_row[pair_idx]
+                if value_map and val in value_map:
+                    matrix[matrix_row, j] = value_map[val]
+                else:
+                    with contextlib.suppress(ValueError, TypeError):
+                        matrix[matrix_row, j] = float(val)
+    return matrix
+
+
+def compute_stratum_kappa(
+    pairs: list[dict],
+    reviews_by_reviewer: dict[str, list[dict]],
+) -> dict[str, dict[str, float | None]]:
+    pair_strata: dict[int, str] = {}
+    for i, pair in enumerate(pairs):
+        pair_strata[i] = classify_transformation(pair)
+
+    strata = ["age_only", "sex_only", "intersectional"]
+    result: dict[str, dict[str, float | None]] = {}
+
+    for stratum in strata:
+        indices = [i for i, s in pair_strata.items() if s == stratum]
+        if not indices:
+            result[stratum] = {
+                "fleiss_kappa_quality": None,
+                "fleiss_kappa_plausibility": None,
+                "fleiss_kappa_pathology_preservation": None,
+            }
+            continue
+
+        q_matrix = build_stratum_rating_matrix(reviews_by_reviewer, "quality_score", indices)
+        p_matrix = build_stratum_rating_matrix(
+            reviews_by_reviewer, "clinically_plausible", indices,
+            value_map={"Yes": 1.0, "No": 0.0},
+        )
+        pp_matrix = build_stratum_rating_matrix(
+            reviews_by_reviewer, "pathology_preserved", indices,
+            value_map={"Yes": 1.0, "No": 0.0, "Uncertain": 0.5},
+        )
+
+        kq = fleiss_kappa(q_matrix, categories=[1, 2, 3, 4, 5])
+        kp = fleiss_kappa(p_matrix, categories=[0, 1])
+        kpp = fleiss_kappa(pp_matrix, categories=[0, 0.5, 1])
+
+        result[stratum] = {
+            "fleiss_kappa_quality": round(kq, 4) if not np.isnan(kq) else None,
+            "fleiss_kappa_plausibility": round(kp, 4) if not np.isnan(kp) else None,
+            "fleiss_kappa_pathology_preservation": round(kpp, 4) if not np.isnan(kpp) else None,
+        }
+
+    return result
+
+
 def compute_exclusions(
     reviews_by_reviewer: dict[str, list[dict]],
     num_pairs: int,
@@ -119,7 +242,10 @@ def compute_exclusions(
 
         scores = quality_scores.get(idx, [])
         if len(scores) >= 3 and (max(scores) - min(scores)) >= 3 and idx not in excluded_implausible:
-            excluded_disputed.append(idx)
+            score_counts = Counter(scores)
+            has_majority = any(c > len(scores) / 2 for c in score_counts.values())
+            if not has_majority:
+                excluded_disputed.append(idx)
 
     all_excluded = sorted(set(excluded_implausible + excluded_disputed))
 
@@ -197,10 +323,12 @@ def compute_stratum_stats(
 
 
 def main() -> None:
+    logging.basicConfig(level=logging.INFO)
     parser = argparse.ArgumentParser(description="Aggregate radiologist reviews")
     parser.add_argument("--pairs", type=str, required=True)
     parser.add_argument("--results-dir", type=str, default="results")
     parser.add_argument("--output", type=str, default="validation_report.json")
+    parser.add_argument("--csv-output", type=str, default=None)
     args = parser.parse_args()
 
     with open(args.pairs) as f:
@@ -211,7 +339,7 @@ def main() -> None:
     reviewers = sorted(reviews_by_reviewer.keys())
 
     if not reviewers:
-        print("No review files found.")
+        logger.warning("No review files found.")
         return
 
     quality_matrix = build_rating_matrix(
@@ -232,6 +360,7 @@ def main() -> None:
 
     exclusions = compute_exclusions(reviews_by_reviewer, num_pairs)
     stratum_stats = compute_stratum_stats(pairs, reviews_by_reviewer, exclusions)
+    stratum_kappa = compute_stratum_kappa(pairs, reviews_by_reviewer)
 
     total_reviewed = sum(len(rows) for rows in reviews_by_reviewer.values())
     per_reviewer: dict[str, dict] = {}
@@ -259,9 +388,12 @@ def main() -> None:
         "reviewers": reviewers,
         "total_reviews": total_reviewed,
         "inter_rater_reliability": {
-            "fleiss_kappa_quality": round(kappa_quality, 4) if not np.isnan(kappa_quality) else None,
-            "fleiss_kappa_plausibility": round(kappa_plausible, 4) if not np.isnan(kappa_plausible) else None,
-            "fleiss_kappa_pathology_preservation": round(kappa_preserved, 4) if not np.isnan(kappa_preserved) else None,
+            "overall": {
+                "fleiss_kappa_quality": round(kappa_quality, 4) if not np.isnan(kappa_quality) else None,
+                "fleiss_kappa_plausibility": round(kappa_plausible, 4) if not np.isnan(kappa_plausible) else None,
+                "fleiss_kappa_pathology_preservation": round(kappa_preserved, 4) if not np.isnan(kappa_preserved) else None,
+            },
+            "by_transformation_type": stratum_kappa,
         },
         "overall": {
             "mean_quality_score": round(mean_quality, 2) if mean_quality else None,
@@ -296,18 +428,33 @@ def main() -> None:
     with open(exclusion_path, "w") as f:
         json.dump({"excluded_pair_indices": exclusions["all_excluded"]}, f, indent=2)
 
-    print(f"Reviewers: {len(reviewers)}")
-    print(f"Total reviews: {total_reviewed}")
-    print(f"Fleiss kappa (quality): {report['inter_rater_reliability']['fleiss_kappa_quality']}")
-    print(f"Fleiss kappa (plausibility): {report['inter_rater_reliability']['fleiss_kappa_plausibility']}")
-    print(f"Fleiss kappa (pathology): {report['inter_rater_reliability']['fleiss_kappa_pathology_preservation']}")
-    print(f"Overall pass rate: {report['overall']['pass_rate']}")
-    print(f"Total excluded: {report['exclusions']['total_excluded']}")
+    csv_output_path = (
+        Path(args.csv_output)
+        if args.csv_output
+        else output_path.parent / "validation_results.csv"
+    )
+    write_validation_results_csv(csv_output_path, pairs, reviews_by_reviewer, exclusions)
+
+    logger.info("Reviewers: %d", len(reviewers))
+    logger.info("Total reviews: %d", total_reviewed)
+    irr = report["inter_rater_reliability"]["overall"]
+    logger.info("Fleiss kappa (quality): %s", irr["fleiss_kappa_quality"])
+    logger.info("Fleiss kappa (plausibility): %s", irr["fleiss_kappa_plausibility"])
+    logger.info("Fleiss kappa (pathology): %s", irr["fleiss_kappa_pathology_preservation"])
+    for stratum, sk in stratum_kappa.items():
+        logger.info(
+            "  %s kappa: quality=%s plausibility=%s pathology=%s",
+            stratum, sk["fleiss_kappa_quality"], sk["fleiss_kappa_plausibility"],
+            sk["fleiss_kappa_pathology_preservation"],
+        )
+    logger.info("Overall pass rate: %s", report["overall"]["pass_rate"])
+    logger.info("Total excluded: %d", report["exclusions"]["total_excluded"])
     for stratum, stats in stratum_stats.items():
         if stats.get("total", 0) > 0:
-            print(f"  {stratum}: {stats['passed']}/{stats['total']} passed ({stats['pass_rate']})")
-    print(f"Report: {output_path}")
-    print(f"Exclusion list: {exclusion_path}")
+            logger.info("  %s: %d/%d passed (%s)", stratum, stats["passed"], stats["total"], stats["pass_rate"])
+    logger.info("Report: %s", output_path)
+    logger.info("Exclusion list: %s", exclusion_path)
+    logger.info("Validation CSV: %s", csv_output_path)
 
 
 if __name__ == "__main__":