Update bias_detector.py

bias_detector.py

@@ -451,11 +451,16 @@ Output: "They pursue excellence and they work hard. They have delivered results.
         # company names (ORG) and locations (LOC) from person names (PER),
         # eliminating false positives like "Luminary Analytics" or "Machine Learning".
         ner_results = self.ner(anonymized)
-        person_spans = [
-            (ent["start"], ent["end"], ent["word"])
-            for ent in ner_results
-            if ent["entity_group"] == "PER"
-        ]
+        person_spans = []
+        for ent in ner_results:
+            if ent["entity_group"] != "PER":
+                continue
+            # Clean BERT subword artifacts (## prefixes from tokenizer)
+            word = ent["word"].replace("##", "").strip()
+            # Skip if too short to be a real name token (avoids partial matches)
+            if len(word) < 3:
+                continue
+            person_spans.append((ent["start"], ent["end"], word))
 
         # Count frequency by normalised name (lower-case first token)
         name_freq: Counter = Counter()
@@ -523,7 +528,7 @@ Output: "They pursue excellence and they work hard. They have delivered results.
                 full_name_label[name.strip()] = label
                 # Also register individual tokens (first name, last name separately)
                 for token in name.strip().split():
-                    if len(token) > 2:
+                    if len(token) >= 4:
                         full_name_label.setdefault(token, label)
             # Also add email-derived names that NER missed
             for first, last, full in email_names:
@@ -540,7 +545,7 @@ Output: "They pursue excellence and they work hard. They have delivered results.
             token_label: dict = {}
             for full_name, label in full_name_label.items():
                 for token in full_name.split():
-                    if len(token) > 2 and token not in token_label:
+                    if len(token) >= 4 and token not in token_label:
                         token_label[token] = label
 
             # Replace full names first (longest first), then individual tokens
@@ -837,4 +842,4 @@ I, Prof. Michael Davies, am delighted to recommend her for this position."""
     result = bd.anonymize_document(test_cv)
     print(f"SURFACE:\n{result['surface_anonymized']}\n")
     print(f"FULLY ANONYMIZED:\n{result['fully_anonymized']}")
-    print(f"Sustainability: {result['sustainability']}")
+    print(f"Sustainability: {result['sustainability']}")