bias_detector.py

# ============================================
# Nubias: Gender Bias Detector & CV Anonymizer
# ============================================
# Usage:
#   from bias_detector import BiasDetector
#   bd = BiasDetector()
#   bias_result = bd.analyze_job_posting("We need an aggressive go-getter.")
#   rewrite_result = bd.rewrite_job_posting("We need an aggressive go-getter.")
#   anon_result = bd.anonymize_document("Sarah is a warm and nurturing leader.")
#
# Each public method returns a dict that includes a "sustainability" key:
#   {
#     "energy_kwh":  float,   # estimated kWh consumed by this request
#     "co2_grams":   float,   # gCO2e for this request
#     "water_liters": float,  # estimated water used for cooling
#   }
# ============================================

import re
import torch
from collections import Counter
from transformers import (
    pipeline,
    AutoTokenizer,
    AutoModelForCausalLM,
)
from codecarbon import EmissionsTracker


# ---------------------------------------------------------------------------
# Water Usage Effectiveness constant.
# Industry average for hyperscale data centres ≈ 1.6 L / kWh of IT load.
# Source: Mytton (2021), "Hiding greenhouse gas emissions in the cloud".
# ---------------------------------------------------------------------------
_WUE_LITERS_PER_KWH = 1.6


def _build_sustainability(tracker: EmissionsTracker) -> dict:
    """
    Stop the tracker and convert its output into the three metrics we report.
    CodeCarbon gives energy in kWh and emissions in kg CO2e.
    """
    tracker.stop()
    energy_kwh  = tracker.final_emissions_data.energy_consumed   # kWh
    co2_kg      = tracker.final_emissions_data.emissions          # kg CO2e
    co2_grams   = co2_kg * 1000
    water_liters = energy_kwh * _WUE_LITERS_PER_KWH
    return {
        "energy_kwh":   round(energy_kwh,   6),
        "co2_grams":    round(co2_grams,    4),
        "water_liters": round(water_liters, 6),
    }


class BiasDetector:
    def __init__(self, llm_model_name: str = "Qwen/Qwen2.5-1.5B-Instruct"):
        """
        Initialize all models. Call once at startup, reuse for every request.

        Args:
            llm_model_name: HuggingFace instruction-tuned model for rewriting.
                            Defaults to Qwen2.5-1.5B-Instruct (float16, CPU).
        """
        print("Loading bias detection model...")
        self.classifier = pipeline(
            "text-classification",
            model="valurank/distilroberta-bias",
            device=0 if torch.cuda.is_available() else -1,
        )

        print("Loading transformer NER model...")
        self.ner = pipeline(
            "ner",
            model="dslim/bert-base-NER",
            aggregation_strategy="simple",
            device=0 if torch.cuda.is_available() else -1,
        )

        print(f"Loading {llm_model_name}...")
        self.tokenizer = AutoTokenizer.from_pretrained(llm_model_name)
        self.llm = AutoModelForCausalLM.from_pretrained(
            llm_model_name,
            torch_dtype=torch.float16,   # float16 halves memory to ~14 GB vs ~28 GB for float32
            device_map="cpu",
            low_cpu_mem_usage=True,      # load shards one at a time to avoid RAM spike during loading
        )

        print("All models loaded successfully!")

        # ----- Detection Lexicon (stem-based) -----

        self.detection_lexicon = {
            "masculine": [
                "active", "adventurous", "aggress", "ambitio", "analy", "assert",
                "autonom", "challeng", "compet", "confident", "courag", "decisive",
                "determin", "dominant", "force", "independen", "individual",
                "intellect", "lead", "logic", "objective", "outspoken", "persist",
                "self-confiden", "self-relian", "self-sufficien", "superior",
                "rockstar", "ninja", "manpower", "chairman", "he/him",
                "fearless", "strong-willed", "ruthless", "hustle", "crush",
                "conquer", "warrior", "battle-tested", "go-getter",
            ],
            "feminine": [
                "affectionate", "cheer", "commit", "communal", "compassion",
                "connect", "considerate", "cooperat", "depend", "empath",
                "gentle", "honest", "interpersonal", "interdependen", "kind",
                "loyal", "nurtur", "pleasant", "polite", "respon", "sensitiv",
                "support", "sympath", "tender", "trust", "understand", "warm",
                "yield", "collaborative", "caring", "welcoming", "friendly",
                "helpful", "people-oriented", "team-oriented", "encouraging",
                "agreeable", "devoted", "gracious", "modest",
            ],
        }

        # ----- Word-level replacements for job posting rewriting -----

        self.word_replacements = {
            "aggressive": "proactive",
            "aggressively": "proactively",
            "ambitious": "motivated",
            "dominant": "experienced",
            "dominate": "excel in",
            "manpower": "workforce",
            "chairman": "chairperson",
            "ninja": "specialist",
            "rockstar": "top performer",
            "fearless": "confident",
            "ruthless": "results-driven",
            "crush": "achieve",
            "conquer": "succeed in",
            "warrior": "professional",
            "hustle": "dedication",
            "go-getter": "self-starter",
            "nurturing": "developing",  # verb form: "nurturing their growth" → "developing their growth"
            "warm": "approachable",
            "sympathetic": "understanding",
            "gentle": "thoughtful",
            "tender": "considerate",
            "yielding": "flexible",
            "caring": "attentive",
            "agreeable": "cooperative",
            "devoted": "dedicated",
            "modest": "professional",
        }

        # ----- Gendered style word replacements for CV anonymization -----

        self.style_replacements = {
            # Feminine-coded → neutral
            "warm": "effective",
            "nurturing": "developing",  # verb form: "nurturing their growth" → "developing their growth"
            "nurture": "develop",
            "nurtured": "developed",
            "gentle": "measured",
            "caring": "attentive",
            "compassionate": "considerate",
            "compassion": "consideration",
            "sympathetic": "understanding",
            "sympathy": "understanding",
            "affectionate": "personable",
            "tender": "thoughtful",
            "devoted": "dedicated",
            "gracious": "professional",
            "motherly": "mentoring",
            "sweet": "pleasant",
            "yielding": "flexible",
            "submissive": "cooperative",
            "emotional": "perceptive",
            "cheerful": "positive",
            "pleasant": "professional",
            "polite": "courteous",
            "agreeable": "cooperative",
            "modest": "understated",
            "friendly": "collegial",
            "welcoming": "inclusive",
            # Masculine-coded → neutral
            "aggressive": "proactive",
            "aggressively": "proactively",
            "dominant": "strong",
            "dominated": "excelled in",
            "forceful": "effective",
            "forcefully": "effectively",
            "ambitious": "motivated",
            "fierce": "determined",
            "fiercely": "with determination",
            "ruthless": "results-oriented",
            "bold": "decisive",
            "boldly": "decisively",
            "fearless": "confident",
            "fearlessly": "confidently",
            "commanding": "authoritative",
            "headstrong": "resolute",
        }

        # ----- System prompts -----

        self.JOB_REWRITE_PROMPT = """You are a job posting editor. Your ONLY job is to rewrite job postings to be gender-neutral.

Rules:
1. Replace masculine-coded words (aggressive, dominant, fearless, ninja, rockstar) with neutral alternatives
2. Replace feminine-coded words (nurturing, warm, sympathetic) with neutral alternatives
3. Keep ALL job requirements, qualifications, and responsibilities intact
4. Keep the same professional tone and structure
5. Do NOT add or remove job requirements
6. Output ONLY the rewritten job posting, nothing else

Example:
Input: "We need an aggressive go-getter who can crush the competition and dominate the market."
Output: "We need a motivated professional who can deliver strong results and excel in the market."
"""

        self.CV_GRAMMAR_PROMPT = """You are a grammar correction assistant.
A document has had all gendered pronouns replaced with singular "they/their/them/themselves".
This sometimes breaks subject-verb agreement (e.g. "they pursues" instead of "they pursue", "they works" instead of "they work").

Your ONLY job is to fix subject-verb agreement where "they" is the subject followed by a third-person singular verb.

Rules:
1. Fix ONLY verb agreement errors caused by "they" replacing "he/she" (e.g. "they pursues" → "they pursue", "they works" → "they work", "they has" → "they have")
2. Do NOT change [CANDIDATE], [PERSON_2], [PERSON_3], [EMAIL] or any other placeholder
3. Do NOT rephrase, rewrite, summarize, or change anything else
4. Do NOT add or remove any content
5. Output ONLY the corrected text, nothing else

Example:
Input:  "They pursues excellence and they works hard. They has delivered results."
Output: "They pursue excellence and they work hard. They have delivered results."
"""

    # =============================================
    # HELPER: LLM generation
    # =============================================

    def _generate(self, system_prompt: str, user_message: str, max_extra_tokens: int = 300) -> str:
        """Run a single LLM inference call. Used by both rewrite and anonymize."""
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user",   "content": user_message},
        ]

        input_text = self.tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        inputs = self.tokenizer(input_text, return_tensors="pt").to(self.llm.device)

        with torch.no_grad():
            outputs = self.llm.generate(
                **inputs,
                max_new_tokens=max_extra_tokens,
                temperature=0.3,
                do_sample=True,
                top_p=0.9,
                repetition_penalty=1.2,
            )

        generated = outputs[0][inputs["input_ids"].shape[1]:]
        result = self.tokenizer.decode(generated, skip_special_tokens=True).strip()

        # Strip LLM commentary that sometimes follows the output
        cutoff_markers = [
            "\nI made", "\nI changed", "\nI replaced", "\nI also",
            "\nNote:", "\nChanges:", "\nExplanation:",
            "\nHere's", "\nThe revised", "\nThis version",
            "\nIn this", "\nBy replacing",
        ]
        for marker in cutoff_markers:
            if marker in result:
                result = result[:result.index(marker)].strip()

        # Strip wrapping quotes
        if result.startswith('"') and result.endswith('"'):
            result = result[1:-1].strip()

        return result

    # =============================================
    # HELPER: Lexicon-based style word replacement
    # =============================================

    def _replace_style_words(self, text: str, replacements_dict: dict) -> str:
        """Replace gendered style words using a dictionary. Preserves capitalization."""
        result = text
        for old_word, new_word in replacements_dict.items():
            pattern = re.compile(r"\b" + re.escape(old_word) + r"\b", re.IGNORECASE)

            def replace_keep_case(match, replacement=new_word):
                original = match.group(0)
                if original[0].isupper():
                    return replacement[0].upper() + replacement[1:]
                return replacement

            result = pattern.sub(replace_keep_case, result)
        return result

    # =============================================
    # PHASE 1A: BIAS DETECTION IN JOB POSTINGS
    # =============================================

    def _run_bias_analysis(self, text: str) -> dict:
        """
        Core bias detection logic (no tracker). Called internally by both
        analyze_job_posting() and rewrite_job_posting() to avoid nested trackers.
        """
        # 1. ML model score
        ml_result = self.classifier(text)[0]

        # 2. Lexicon scan using stem matching
        flagged_words = []
        words_in_text = text.lower().split()

        for category, stems in self.detection_lexicon.items():
            for stem in stems:
                for word in words_in_text:
                    if word.startswith(stem) or stem in word:
                        flagged_words.append({
                            "word":     word.strip(".,;:!?()\"'"),
                            "stem":     stem,
                            "category": category + "-coded",
                        })

        # Remove duplicates
        seen = set()
        unique_flags = []
        for f in flagged_words:
            if f["word"] not in seen:
                seen.add(f["word"])
                unique_flags.append(f)

        # 3. Combined bias score
        ml_bias_score = (
            1 - ml_result["score"]
            if ml_result["label"] == "NEUTRAL"
            else ml_result["score"]
        )
        lexicon_boost  = min(len(unique_flags) * 0.15, 0.5)
        combined_score = min(ml_bias_score + lexicon_boost, 1.0)

        # 4. Overall label
        overall_label = "BIASED" if combined_score >= 0.5 else "NEUTRAL"

        return {
            "overall_label":   overall_label,
            "bias_score":      round(combined_score, 3),
            "ml_model_result": ml_result,
            "flagged_words":   unique_flags,
            "masculine_count": sum(1 for f in unique_flags if f["category"] == "masculine-coded"),
            "feminine_count":  sum(1 for f in unique_flags if f["category"] == "feminine-coded"),
        }

    def analyze_job_posting(self, text: str) -> dict:
        """
        Hybrid bias detector: ML model + lexicon scan.
        Tracks energy, CO2, and water consumption for the full request.

        Returns dict with: overall_label, bias_score, ml_model_result,
        flagged_words, masculine_count, feminine_count, sustainability.
        """
        tracker = EmissionsTracker(
            project_name="nubias_bias_detection",
            log_level="error",
            save_to_file=False,
        )
        tracker.start()

        result = self._run_bias_analysis(text)
        sustainability = _build_sustainability(tracker)

        return {**result, "sustainability": sustainability}

    # =============================================
    # PHASE 1B: JOB POSTING REWRITER
    # =============================================

    def rewrite_job_posting(self, text: str) -> dict:
        """
        Rewrites a job posting to remove gendered language.
        Combines lexicon-based replacements with Qwen2.5-7B rewriting.
        Sustainability covers the full pipeline (classifier + LLM).

        Returns dict with: original, analysis, lexicon_fixed,
        fully_rewritten, sustainability.
        """
        tracker = EmissionsTracker(
            project_name="nubias_job_rewrite",
            log_level="error",
            save_to_file=False,
        )
        tracker.start()

        # 1. Bias detection (use private method to avoid nested trackers)
        analysis = self._run_bias_analysis(text)

        # 2. Lexicon-based quick fix
        lexicon_fixed = self._replace_style_words(text, self.word_replacements)

        # 3. LLM rewrite for deeper neutralization
        rewritten = self._generate(
            self.JOB_REWRITE_PROMPT,
            f"Rewrite this job posting to be gender-neutral. Output ONLY the rewritten text:\n\n{lexicon_fixed}",
            max_extra_tokens=300,
        )

        # Fallback if LLM output looks broken
        if len(rewritten) < 10 or len(rewritten) > len(text) * 3:
            rewritten = lexicon_fixed

        sustainability = _build_sustainability(tracker)

        return {
            "original":        text,
            "analysis":        analysis,
            "lexicon_fixed":   lexicon_fixed,
            "fully_rewritten": rewritten,
            "sustainability":  sustainability,
        }

    # =============================================
    # PHASE 2: CV / LETTER GENDER ANONYMIZER
    # =============================================

    def _surface_anonymize(self, text: str) -> str:
        """
        Step A: Replace emails with [EMAIL].
        Step B: Transformer NER (dslim/bert-base-NER) → frequency-based person labelling.
                Most-mentioned person → [CANDIDATE].
                Others → [PERSON_2], [PERSON_3], … in order of first appearance.
        Step C: Rule-based pronoun / title / gendered-noun replacement.
        """

        # --- Step A: Extract name from email BEFORE masking it ---
        # e.g. "vanessa.collard@ses.com" → Vanessa Collard
        # Must run on original text before email is replaced with [EMAIL].
        email_names = []
        header_text_orig = "\n".join(text.splitlines()[:3])
        email_match = re.search(r"([a-zA-Z]+)[._]([a-zA-Z]+)@", header_text_orig)
        if email_match:
            first = email_match.group(1).capitalize()
            last  = email_match.group(2).capitalize()
            email_names.append((first, last, f"{first} {last}"))

        # --- Step A: Email addresses ---
        anonymized = re.sub(
            r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}",
            "[EMAIL]",
            text,
        )

        # --- Step B: Person names via transformer NER (dslim/bert-base-NER) ---
        # Returns entity_group PER/ORG/LOC/MISC — we only keep PER.
        # Unlike spaCy en_core_web_sm, this model correctly distinguishes
        # company names (ORG) and locations (LOC) from person names (PER),
        # eliminating false positives like "Luminary Analytics" or "Machine Learning".
        ner_results = self.ner(anonymized)
        person_spans = []
        for ent in ner_results:
            if ent["entity_group"] != "PER":
                continue
            # Clean BERT subword artifacts (## prefixes from tokenizer)
            word = ent["word"].replace("##", "").strip()
            # Skip if too short to be a real name token (avoids partial matches)
            if len(word) < 3:
                continue
            person_spans.append((ent["start"], ent["end"], word))

        # Count frequency by normalised name (lower-case first token)
        name_freq: Counter = Counter()
        first_seen: dict = {}   # normalised_name → first start_char
        for start, end, name in person_spans:
            key = name.strip().lower().split()[0]
            name_freq[key] += 1
            if key not in first_seen:
                first_seen[key] = start

        # Seed name_freq with email-derived names not already found by NER
        for first, last, full in email_names:
            key = first.lower()
            if key not in name_freq:
                # Count how often this first name appears in the full text
                name_freq[key] = len(re.findall(rf"\b{re.escape(first)}\b", anonymized, re.IGNORECASE))
                first_seen[key] = anonymized.lower().find(key)

        if name_freq:
            # Most-frequent name is the candidate
            candidate_key = name_freq.most_common(1)[0][0]

            # Remaining names ordered by first appearance
            other_keys = sorted(
                [k for k in name_freq if k != candidate_key],
                key=lambda k: first_seen.get(k, 9999),
            )
            label_map = {candidate_key: "[CANDIDATE]"}
            for i, k in enumerate(other_keys, start=2):
                label_map[k] = f"[PERSON_{i}]"

            # Extend person spans: if the token immediately after a PER span
            # is a capitalised word not in common vocab, treat it as a surname
            # e.g. NER finds "Marcus" but misses "Obi" → extend to "Marcus Obi"
            extended_spans = []
            for start, end, name in person_spans:
                key = name.strip().lower().split()[0]
                label = label_map.get(key, "[PERSON]")
                # Check if next token after span is a capitalised word (surname candidate)
                rest = anonymized[end:]
                surname_match = re.match(r"^\s+([A-Z][A-Za-z]{1,20})\b", rest)
                if surname_match:
                    candidate_surname = surname_match.group(1)
                    # Only extend if it's not a common non-name word
                    NON_NAMES = {"The", "This", "That", "Their", "They", "He", "She",
                                 "His", "Her", "During", "In", "At", "For", "And",
                                 "But", "With", "From", "To", "Of", "On", "By"}
                    if candidate_surname not in NON_NAMES:
                        new_end = end + len(surname_match.group(0))
                        new_name = anonymized[start:new_end].strip()
                        extended_spans.append((start, new_end, new_name, label))
                        # Also register the surname token in full_name_label later
                        continue
                extended_spans.append((start, end, name, label))

            # Replace NER spans in reverse order to preserve char offsets
            for start, end, name, label in reversed(extended_spans):
                anonymized = anonymized[:start] + label + anonymized[end:]

            # Second pass: regex sweep for any remaining occurrences of known names
            # (catches names spaCy missed because they were next to org/title context).
            # Sort longest-first to avoid partial replacements.
            full_name_label: dict = {}
            for start, end, name, label in extended_spans:
                full_name_label[name.strip()] = label
                # Also register individual tokens (first name, last name separately)
                for token in name.strip().split():
                    if len(token) >= 4:
                        full_name_label.setdefault(token, label)
            # Also add email-derived names that NER missed
            for first, last, full in email_names:
                key = first.lower()
                if key not in label_map:
                    continue
                # Add full name, first name, and last name so all forms are caught
                for variant in [full, first, last]:
                    if variant not in full_name_label:
                        full_name_label[variant] = label_map[key]

            # Build token_label AFTER full_name_label is complete (including email names)
            # so last names like "Obi" or "Reeves" from email are included
            token_label: dict = {}
            for full_name, label in full_name_label.items():
                for token in full_name.split():
                    if len(token) >= 4 and token not in token_label:
                        token_label[token] = label

            # Replace full names first (longest first), then individual tokens
            for full_name, label in sorted(full_name_label.items(), key=lambda x: -len(x[0])):
                anonymized = re.sub(re.escape(full_name), label, anonymized)
            for token, label in sorted(token_label.items(), key=lambda x: -len(x[0])):
                anonymized = re.sub(r"\b" + re.escape(token) + r"\b", label, anonymized)

        # --- Step C: Pronouns, titles, gendered nouns ---
        replacements = [
            # Pronoun + verb agreement (she/he → they)
            (r"\b[Ss]he has\b",            "They have"),
            (r"\b[Ss]he is\b",             "They are"),
            (r"\b[Ss]he was\b",            "They were"),
            (r"\b[Ss]he works\b",          "They work"),
            (r"\b[Ss]he often\b",          "They often"),
            (r"\b[Ss]he also\b",           "They also"),
            (r"\b[Ss]he always\b",         "They always"),
            (r"\b[Ss]he then\b",           "They then"),
            (r"\b[Ss]he quickly\b",        "They quickly"),
            (r"\b[Ss]he approached\b",     "They approached"),
            (r"\b[Ss]he independently\b",  "They independently"),
            (r"\b[Hh]e has\b",             "They have"),
            (r"\b[Hh]e is\b",             "They are"),
            (r"\b[Hh]e was\b",             "They were"),
            (r"\b[Hh]e works\b",           "They work"),
            (r"\b[Hh]e often\b",           "They often"),
            (r"\b[Hh]e also\b",            "They also"),
            (r"\b[Hh]e always\b",          "They always"),
            (r"\b[Hh]e then\b",            "They then"),
            (r"\b[Hh]e quickly\b",         "They quickly"),
            # Object pronoun: verb + her/him → verb + them
            (r"\benable [Hh]er\b",  "enable them"),
            (r"\benable [Hh]im\b",  "enable them"),
            (r"\bmade [Hh]er\b",    "made them"),
            (r"\bmade [Hh]im\b",    "made them"),
            (r"\bmake [Hh]er\b",    "make them"),
            (r"\bmake [Hh]im\b",    "make them"),
            (r"\bhelped [Hh]er\b",  "helped them"),
            (r"\bhelped [Hh]im\b",  "helped them"),
            (r"\ballow [Hh]er\b",   "allow them"),
            (r"\ballow [Hh]im\b",   "allow them"),
            (r"\bgave [Hh]er\b",    "gave them"),
            (r"\bgave [Hh]im\b",    "gave them"),
            (r"\btold [Hh]er\b",    "told them"),
            (r"\btold [Hh]im\b",    "told them"),
            (r"\basked [Hh]er\b",   "asked them"),
            (r"\basked [Hh]im\b",   "asked them"),
            (r"\bshowed [Hh]er\b",  "showed them"),
            (r"\bshowed [Hh]im\b",  "showed them"),
            (r"\btaught [Hh]er\b",  "taught them"),
            (r"\btaught [Hh]im\b",  "taught them"),
            (r"\boffered [Hh]er\b", "offered them"),
            (r"\boffered [Hh]im\b", "offered them"),
            (r"\bsent [Hh]er\b",    "sent them"),
            (r"\bsent [Hh]im\b",    "sent them"),
            (r"\bserve [Hh]er\b",   "serve them"),
            (r"\bserve [Hh]im\b",   "serve them"),
            # Possessive before nouns
            (r"\b[Hh]er(?=\s+\w)",  "their"),
            (r"\b[Hh]is(?=\s+\w)",  "their"),
            (r"(?<=[.!?]\s)their\b",  "Their"),
            # Standalone pronouns
            (r"\bShe\b",       "They"),
            (r"\bshe\b",       "they"),
            (r"\bHe\b",        "They"),
            (r"\bhe\b",        "they"),
            (r"\b[Hh]im\b",   "them"),
            (r"\b[Hh]erself\b",  "themselves"),
            (r"\b[Hh]imself\b",  "themselves"),

            # Subject-verb agreement: fix third-person singular verbs after "they"
            # These arise when "he/she + verb-s" is replaced with "they + verb-s".
            (r"\bThey pursues\b",      "They pursue"),
            (r"\bthey pursues\b",      "they pursue"),
            (r"\bThey works\b",        "They work"),
            (r"\bthey works\b",        "they work"),
            (r"\bThey leads\b",        "They lead"),
            (r"\bthey leads\b",        "they lead"),
            (r"\bThey manages\b",      "They manage"),
            (r"\bthey manages\b",      "they manage"),
            (r"\bThey brings\b",       "They bring"),
            (r"\bthey brings\b",       "they bring"),
            (r"\bThey demonstrates\b", "They demonstrate"),
            (r"\bthey demonstrates\b", "they demonstrate"),
            (r"\bThey contributes\b",  "They contribute"),
            (r"\bthey contributes\b",  "they contribute"),
            (r"\bThey shows\b",        "They show"),
            (r"\bthey shows\b",        "they show"),
            (r"\bThey excels\b",       "They excel"),
            (r"\bthey excels\b",       "they excel"),
            (r"\bThey delivers\b",     "They deliver"),
            (r"\bthey delivers\b",     "they deliver"),
            (r"\bThey handles\b",      "They handle"),
            (r"\bthey handles\b",      "they handle"),
            (r"\bThey drives\b",       "They drive"),
            (r"\bthey drives\b",       "they drive"),
            (r"\bThey seeks\b",        "They seek"),
            (r"\bthey seeks\b",        "they seek"),
            (r"\bThey holds\b",        "They hold"),
            (r"\bthey holds\b",        "they hold"),
            (r"\bThey oversees\b",     "They oversee"),
            (r"\bthey oversees\b",     "they oversee"),
            (r"\bThey possesses\b",    "They possess"),
            (r"\bthey possesses\b",    "they possess"),
            (r"\bThey believes\b",     "They believe"),
            (r"\bthey believes\b",     "they believe"),
            (r"\bThey strives\b",      "They strive"),
            (r"\bthey strives\b",      "they strive"),
            (r"\bThey pursues\b",      "They pursue"),
            (r"\bthey pursues\b",      "they pursue"),
            (r"\bThey approaches\b",   "They approach"),
            (r"\bthey approaches\b",   "they approach"),
            (r"\bThey applies\b",      "They apply"),
            (r"\bthey applies\b",      "they apply"),
            (r"\bThey takes\b",        "They take"),
            (r"\bthey takes\b",        "they take"),
            (r"\bThey makes\b",        "They make"),
            (r"\bthey makes\b",        "they make"),
            (r"\bThey meets\b",        "They meet"),
            (r"\bthey meets\b",        "they meet"),
            (r"\bThey comes\b",        "They come"),
            (r"\bthey comes\b",        "they come"),
            (r"\bThey goes\b",         "They go"),
            (r"\bthey goes\b",         "they go"),
            (r"\bThey gets\b",         "They get"),
            (r"\bthey gets\b",         "they get"),
            (r"\bThey gives\b",        "They give"),
            (r"\bthey gives\b",        "they give"),
            (r"\bThey says\b",         "They say"),
            (r"\bthey says\b",         "they say"),
            (r"\bThey knows\b",        "They know"),
            (r"\bthey knows\b",        "they know"),
            (r"\bThey thinks\b",       "They think"),
            (r"\bthey thinks\b",       "they think"),
            (r"\bThey sees\b",         "They see"),
            (r"\bthey sees\b",         "they see"),
            (r"\bThey uses\b",         "They use"),
            (r"\bthey uses\b",         "they use"),
            (r"\bThey builds\b",       "They build"),
            (r"\bthey builds\b",       "they build"),
            (r"\bThey creates\b",      "They create"),
            (r"\bthey creates\b",      "they create"),
            (r"\bThey develops\b",     "They develop"),
            (r"\bthey develops\b",     "they develop"),
            (r"\bThey supports\b",     "They support"),
            (r"\bthey supports\b",     "they support"),
            (r"\bThey helps\b",        "They help"),
            (r"\bthey helps\b",        "they help"),
            (r"\bThey joins\b",        "They join"),
            (r"\bthey joins\b",        "they join"),
            (r"\bThey plays\b",        "They play"),
            (r"\bthey plays\b",        "they play"),
            (r"\bThey runs\b",         "They run"),
            (r"\bthey runs\b",         "they run"),
            (r"\bThey sets\b",         "They set"),
            (r"\bthey sets\b",         "they set"),
            (r"\bThey starts\b",       "They start"),
            (r"\bthey starts\b",       "they start"),
            (r"\bThey stops\b",        "They stop"),
            (r"\bthey stops\b",        "they stop"),
            (r"\bThey writes\b",       "They write"),
            (r"\bthey writes\b",       "they write"),
            (r"\bThey reads\b",        "They read"),
            (r"\bthey reads\b",        "they read"),
            (r"\bThey speaks\b",       "They speak"),
            (r"\bthey speaks\b",       "they speak"),
            (r"\bThey listens\b",      "They listen"),
            (r"\bthey listens\b",      "they listen"),
            (r"\bThey learns\b",       "They learn"),
            (r"\bthey learns\b",       "they learn"),
            (r"\bThey teaches\b",      "They teach"),
            (r"\bthey teaches\b",      "they teach"),
            (r"\bThey represents\b",   "They represent"),
            (r"\bthey represents\b",   "they represent"),
            (r"\bThey reports\b",      "They report"),
            (r"\bthey reports\b",      "they report"),
            (r"\bThey serves\b",       "They serve"),
            (r"\bthey serves\b",       "they serve"),
            # Titles (remove)
            (r"\bMrs?\.\s*", ""),
            (r"\bMs\.\s*",   ""),
            (r"\bMiss\s+",   ""),
            # Gendered nouns
            (r"\b[Hh]usband\b",    "spouse"),
            (r"\b[Ww]ife\b",       "spouse"),
            (r"\b[Mm]other\b",     "parent"),
            (r"\b[Ff]ather\b",     "parent"),
            (r"\b[Ss]on\b",        "child"),
            (r"\b[Dd]aughter\b",   "child"),
            (r"\b[Bb]rother\b",    "sibling"),
            (r"\b[Ss]ister\b",     "sibling"),
            (r"\b[Bb]oyfriend\b",  "partner"),
            (r"\b[Gg]irlfriend\b", "partner"),
            (r"\b[Ss]pokesman\b",  "spokesperson"),
            (r"\b[Ss]pokeswoman\b","spokesperson"),
            (r"\b[Cc]hairman\b",   "chairperson"),
            (r"\b[Cc]hairwoman\b", "chairperson"),
            (r"\b[Mm]anpower\b",   "workforce"),
            (r"\b[Gg]irl\b",       "person"),
            (r"\b[Bb]oy\b",        "person"),
            (r"\b[Ww]oman\b",      "person"),
            (r"\b[Ww]omen\b",      "people"),
            (r"\b[Mm]an\b",        "person"),
            (r"\b[Mm]en\b",        "people"),
            (r"\b[Ll]ady\b",       "person"),
            (r"\b[Ll]adies\b",     "people"),
            (r"\b[Gg]entleman\b",  "person"),
            (r"\b[Gg]entlemen\b",  "people"),
            (r"\b[Ff]emale\b",     "person"),
            (r"\b[Ff]emales\b",    "people"),
            (r"\b[Mm]ale\b",       "person"),
            (r"\b[Mm]ales\b",      "people"),
        ]

        for pattern, replacement in replacements:
            anonymized = re.sub(pattern, replacement, anonymized)

        # Clean up artefacts
        anonymized = re.sub(r"\s{2,}", " ", anonymized)
        anonymized = re.sub(r"\.\s*\.", ".", anonymized)
        return anonymized.strip()

    def anonymize_document(self, text: str) -> dict:
        """
        Full anonymization pipeline:
          Step A: Email regex → [EMAIL]
          Step B: spaCy NER → frequency-based [CANDIDATE] / [PERSON_N] labels
          Step C: Rule-based → replace pronouns, titles, gendered nouns
          Step D: Lexicon-based → replace gendered style words
        Note: LLM pass removed — small models hallucinate on CV-length text,
        and the deterministic pipeline handles anonymization reliably on its own.
        Sustainability covers the full pipeline.

        Returns dict with: original, surface_anonymized,
        fully_anonymized, sustainability.
        """
        tracker = EmissionsTracker(
            project_name="nubias_cv_anonymize",
            log_level="error",
            save_to_file=False,
        )
        tracker.start()

        # Steps A–C: surface anonymization (emails, names, pronouns, titles)
        surface_result = self._surface_anonymize(text)

        # Step D: Style word neutralization + subject-verb agreement fix.
        # Grammar is corrected deterministically — no LLM needed.
        fully_anonymized = self._replace_style_words(surface_result, self.style_replacements)

        sustainability = _build_sustainability(tracker)

        return {
            "original":           text,
            "surface_anonymized": surface_result,
            "fully_anonymized":   fully_anonymized,
            "sustainability":     sustainability,
        }


# =============================================
# Quick test (run this file directly to verify)
# =============================================
if __name__ == "__main__":
    bd = BiasDetector()

    print("=" * 50)
    print("TEST 1: Job Posting Bias Detection")
    print("=" * 50)
    test_posting = "We need an aggressive go-getter who can dominate the competition."
    result = bd.analyze_job_posting(test_posting)
    print(f"Label: {result['overall_label']}")
    print(f"Score: {result['bias_score']}")
    print(f"Flagged: {[f['word'] for f in result['flagged_words']]}")
    print(f"Sustainability: {result['sustainability']}")

    print("\n" + "=" * 50)
    print("TEST 2: Job Posting Rewrite")
    print("=" * 50)
    result = bd.rewrite_job_posting(test_posting)
    print(f"Original:  {result['original']}")
    print(f"Lexicon:   {result['lexicon_fixed']}")
    print(f"Rewritten: {result['fully_rewritten']}")
    print(f"Sustainability: {result['sustainability']}")

    print("\n" + "=" * 50)
    print("TEST 3: CV Anonymization")
    print("=" * 50)
    test_cv = """Dr. Sarah Johnson (sarah.johnson@email.com) is an exceptionally warm and nurturing leader.
She has always been deeply compassionate and sympathetic toward her colleagues.
Her husband mentioned she is also a devoted mother who balances work and family gracefully.
I, Prof. Michael Davies, am delighted to recommend her for this position."""
    result = bd.anonymize_document(test_cv)
    print(f"SURFACE:\n{result['surface_anonymized']}\n")
    print(f"FULLY ANONYMIZED:\n{result['fully_anonymized']}")
    print(f"Sustainability: {result['sustainability']}")