# ============================================ # Nubias: Gender Bias Detector & CV Anonymizer # ============================================ # Usage: # from bias_detector import BiasDetector # bd = BiasDetector() # bias_result = bd.analyze_job_posting("We need an aggressive go-getter.") # rewrite_result = bd.rewrite_job_posting("We need an aggressive go-getter.") # anon_result = bd.anonymize_document("Sarah is a warm and nurturing leader.") # # Each public method returns a dict that includes a "sustainability" key: # { # "energy_kwh": float, # estimated kWh consumed by this request # "co2_grams": float, # gCO2e for this request # "water_liters": float, # estimated water used for cooling # } # ============================================ import re import torch import spacy from collections import Counter from transformers import ( pipeline, AutoTokenizer, AutoModelForCausalLM, ) from codecarbon import EmissionsTracker # --------------------------------------------------------------------------- # Water Usage Effectiveness constant. # Industry average for hyperscale data centres ≈ 1.6 L / kWh of IT load. # Source: Mytton (2021), "Hiding greenhouse gas emissions in the cloud". # --------------------------------------------------------------------------- _WUE_LITERS_PER_KWH = 1.6 def _build_sustainability(tracker: EmissionsTracker) -> dict: """ Stop the tracker and convert its output into the three metrics we report. CodeCarbon gives energy in kWh and emissions in kg CO2e. """ tracker.stop() energy_kwh = tracker.final_emissions_data.energy_consumed # kWh co2_kg = tracker.final_emissions_data.emissions # kg CO2e co2_grams = co2_kg * 1000 water_liters = energy_kwh * _WUE_LITERS_PER_KWH return { "energy_kwh": round(energy_kwh, 6), "co2_grams": round(co2_grams, 4), "water_liters": round(water_liters, 6), } class BiasDetector: def __init__(self, llm_model_name: str = "Qwen/Qwen2.5-1.5B-Instruct"): """ Initialize all models. Call once at startup, reuse for every request. Args: llm_model_name: HuggingFace instruction-tuned model for rewriting and CV anonymization. Defaults to Qwen2.5-7B-Instruct loaded in 4-bit to fit on CPU-only Spaces (~4.5 GB RAM). """ print("Loading bias detection model...") self.classifier = pipeline( "text-classification", model="valurank/distilroberta-bias", device=0 if torch.cuda.is_available() else -1, ) print("Loading spaCy NER model...") self.nlp = spacy.load("en_core_web_sm") print(f"Loading {llm_model_name}...") self.tokenizer = AutoTokenizer.from_pretrained(llm_model_name) self.llm = AutoModelForCausalLM.from_pretrained( llm_model_name, torch_dtype=torch.float16, # float16 halves memory to ~14 GB vs ~28 GB for float32 device_map="cpu", low_cpu_mem_usage=True, # load shards one at a time to avoid RAM spike during loading ) print("All models loaded successfully!") # ----- Detection Lexicon (stem-based) ----- self.detection_lexicon = { "masculine": [ "active", "adventurous", "aggress", "ambitio", "analy", "assert", "autonom", "challeng", "compet", "confident", "courag", "decisive", "determin", "dominant", "force", "independen", "individual", "intellect", "lead", "logic", "objective", "outspoken", "persist", "self-confiden", "self-relian", "self-sufficien", "superior", "rockstar", "ninja", "manpower", "chairman", "he/him", "fearless", "strong-willed", "ruthless", "hustle", "crush", "conquer", "warrior", "battle-tested", "go-getter", ], "feminine": [ "affectionate", "cheer", "commit", "communal", "compassion", "connect", "considerate", "cooperat", "depend", "empath", "gentle", "honest", "interpersonal", "interdependen", "kind", "loyal", "nurtur", "pleasant", "polite", "respon", "sensitiv", "support", "sympath", "tender", "trust", "understand", "warm", "yield", "collaborative", "caring", "welcoming", "friendly", "helpful", "people-oriented", "team-oriented", "encouraging", "agreeable", "devoted", "gracious", "modest", ], } # ----- Word-level replacements for job posting rewriting ----- self.word_replacements = { "aggressive": "proactive", "aggressively": "proactively", "ambitious": "motivated", "dominant": "experienced", "dominate": "excel in", "manpower": "workforce", "chairman": "chairperson", "ninja": "specialist", "rockstar": "top performer", "fearless": "confident", "ruthless": "results-driven", "crush": "achieve", "conquer": "succeed in", "warrior": "professional", "hustle": "dedication", "go-getter": "self-starter", "nurturing": "developing", # verb form: "nurturing their growth" → "developing their growth" "warm": "approachable", "sympathetic": "understanding", "gentle": "thoughtful", "tender": "considerate", "yielding": "flexible", "caring": "attentive", "agreeable": "cooperative", "devoted": "dedicated", "modest": "professional", } # ----- Gendered style word replacements for CV anonymization ----- self.style_replacements = { # Feminine-coded → neutral "warm": "effective", "nurturing": "developing", # verb form: "nurturing their growth" → "developing their growth" "nurture": "develop", "nurtured": "developed", "gentle": "measured", "caring": "attentive", "compassionate": "considerate", "compassion": "consideration", "sympathetic": "understanding", "sympathy": "understanding", "affectionate": "personable", "tender": "thoughtful", "devoted": "dedicated", "gracious": "professional", "motherly": "mentoring", "sweet": "pleasant", "yielding": "flexible", "submissive": "cooperative", "emotional": "perceptive", "cheerful": "positive", "pleasant": "professional", "polite": "courteous", "agreeable": "cooperative", "modest": "understated", "friendly": "collegial", "welcoming": "inclusive", # Masculine-coded → neutral "aggressive": "proactive", "aggressively": "proactively", "dominant": "strong", "dominated": "excelled in", "forceful": "effective", "forcefully": "effectively", "ambitious": "motivated", "fierce": "determined", "fiercely": "with determination", "ruthless": "results-oriented", "bold": "decisive", "boldly": "decisively", "fearless": "confident", "fearlessly": "confidently", "commanding": "authoritative", "headstrong": "resolute", } # ----- System prompts ----- self.JOB_REWRITE_PROMPT = """You are a job posting editor. Your ONLY job is to rewrite job postings to be gender-neutral. Rules: 1. Replace masculine-coded words (aggressive, dominant, fearless, ninja, rockstar) with neutral alternatives 2. Replace feminine-coded words (nurturing, warm, sympathetic) with neutral alternatives 3. Keep ALL job requirements, qualifications, and responsibilities intact 4. Keep the same professional tone and structure 5. Do NOT add or remove job requirements 6. Output ONLY the rewritten job posting, nothing else Example: Input: "We need an aggressive go-getter who can crush the competition and dominate the market." Output: "We need a motivated professional who can deliver strong results and excel in the market." """ self.CV_GRAMMAR_PROMPT = """You are a grammar correction assistant. A document has had all gendered pronouns replaced with singular "they/their/them/themselves". This sometimes breaks subject-verb agreement (e.g. "they pursues" instead of "they pursue", "they works" instead of "they work"). Your ONLY job is to fix subject-verb agreement where "they" is the subject followed by a third-person singular verb. Rules: 1. Fix ONLY verb agreement errors caused by "they" replacing "he/she" (e.g. "they pursues" → "they pursue", "they works" → "they work", "they has" → "they have") 2. Do NOT change [CANDIDATE], [PERSON_2], [PERSON_3], [EMAIL] or any other placeholder 3. Do NOT rephrase, rewrite, summarize, or change anything else 4. Do NOT add or remove any content 5. Output ONLY the corrected text, nothing else Example: Input: "They pursues excellence and they works hard. They has delivered results." Output: "They pursue excellence and they work hard. They have delivered results." """ # ============================================= # HELPER: LLM generation # ============================================= def _generate(self, system_prompt: str, user_message: str, max_extra_tokens: int = 300) -> str: """Run a single LLM inference call. Used by both rewrite and anonymize.""" messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_message}, ] input_text = self.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) inputs = self.tokenizer(input_text, return_tensors="pt").to(self.llm.device) with torch.no_grad(): outputs = self.llm.generate( **inputs, max_new_tokens=max_extra_tokens, temperature=0.3, do_sample=True, top_p=0.9, repetition_penalty=1.2, ) generated = outputs[0][inputs["input_ids"].shape[1]:] result = self.tokenizer.decode(generated, skip_special_tokens=True).strip() # Strip LLM commentary that sometimes follows the output cutoff_markers = [ "\nI made", "\nI changed", "\nI replaced", "\nI also", "\nNote:", "\nChanges:", "\nExplanation:", "\nHere's", "\nThe revised", "\nThis version", "\nIn this", "\nBy replacing", ] for marker in cutoff_markers: if marker in result: result = result[:result.index(marker)].strip() # Strip wrapping quotes if result.startswith('"') and result.endswith('"'): result = result[1:-1].strip() return result # ============================================= # HELPER: Lexicon-based style word replacement # ============================================= def _replace_style_words(self, text: str, replacements_dict: dict) -> str: """Replace gendered style words using a dictionary. Preserves capitalization.""" result = text for old_word, new_word in replacements_dict.items(): pattern = re.compile(r"\b" + re.escape(old_word) + r"\b", re.IGNORECASE) def replace_keep_case(match, replacement=new_word): original = match.group(0) if original[0].isupper(): return replacement[0].upper() + replacement[1:] return replacement result = pattern.sub(replace_keep_case, result) return result # ============================================= # PHASE 1A: BIAS DETECTION IN JOB POSTINGS # ============================================= def _run_bias_analysis(self, text: str) -> dict: """ Core bias detection logic (no tracker). Called internally by both analyze_job_posting() and rewrite_job_posting() to avoid nested trackers. """ # 1. ML model score ml_result = self.classifier(text)[0] # 2. Lexicon scan using stem matching flagged_words = [] words_in_text = text.lower().split() for category, stems in self.detection_lexicon.items(): for stem in stems: for word in words_in_text: if word.startswith(stem) or stem in word: flagged_words.append({ "word": word.strip(".,;:!?()\"'"), "stem": stem, "category": category + "-coded", }) # Remove duplicates seen = set() unique_flags = [] for f in flagged_words: if f["word"] not in seen: seen.add(f["word"]) unique_flags.append(f) # 3. Combined bias score ml_bias_score = ( 1 - ml_result["score"] if ml_result["label"] == "NEUTRAL" else ml_result["score"] ) lexicon_boost = min(len(unique_flags) * 0.15, 0.5) combined_score = min(ml_bias_score + lexicon_boost, 1.0) # 4. Overall label overall_label = "BIASED" if combined_score >= 0.5 else "NEUTRAL" return { "overall_label": overall_label, "bias_score": round(combined_score, 3), "ml_model_result": ml_result, "flagged_words": unique_flags, "masculine_count": sum(1 for f in unique_flags if f["category"] == "masculine-coded"), "feminine_count": sum(1 for f in unique_flags if f["category"] == "feminine-coded"), } def analyze_job_posting(self, text: str) -> dict: """ Hybrid bias detector: ML model + lexicon scan. Tracks energy, CO2, and water consumption for the full request. Returns dict with: overall_label, bias_score, ml_model_result, flagged_words, masculine_count, feminine_count, sustainability. """ tracker = EmissionsTracker( project_name="nubias_bias_detection", log_level="error", save_to_file=False, ) tracker.start() result = self._run_bias_analysis(text) sustainability = _build_sustainability(tracker) return {**result, "sustainability": sustainability} # ============================================= # PHASE 1B: JOB POSTING REWRITER # ============================================= def rewrite_job_posting(self, text: str) -> dict: """ Rewrites a job posting to remove gendered language. Combines lexicon-based replacements with Qwen2.5-7B rewriting. Sustainability covers the full pipeline (classifier + LLM). Returns dict with: original, analysis, lexicon_fixed, fully_rewritten, sustainability. """ tracker = EmissionsTracker( project_name="nubias_job_rewrite", log_level="error", save_to_file=False, ) tracker.start() # 1. Bias detection (use private method to avoid nested trackers) analysis = self._run_bias_analysis(text) # 2. Lexicon-based quick fix lexicon_fixed = self._replace_style_words(text, self.word_replacements) # 3. LLM rewrite for deeper neutralization rewritten = self._generate( self.JOB_REWRITE_PROMPT, f"Rewrite this job posting to be gender-neutral. Output ONLY the rewritten text:\n\n{lexicon_fixed}", max_extra_tokens=300, ) # Fallback if LLM output looks broken if len(rewritten) < 10 or len(rewritten) > len(text) * 3: rewritten = lexicon_fixed sustainability = _build_sustainability(tracker) return { "original": text, "analysis": analysis, "lexicon_fixed": lexicon_fixed, "fully_rewritten": rewritten, "sustainability": sustainability, } # ============================================= # PHASE 2: CV / LETTER GENDER ANONYMIZER # ============================================= def _surface_anonymize(self, text: str) -> str: """ Step A: Replace emails with [EMAIL]. Step B: spaCy NER → frequency-based person labelling. Most-mentioned person → [CANDIDATE]. Others → [PERSON_2], [PERSON_3], … in order of first appearance. Step C: Rule-based pronoun / title / gendered-noun replacement. """ # --- Step A: Extract name from email BEFORE masking it --- # e.g. "vanessa.collard@ses.com" → Vanessa Collard # Must run on original text before email is replaced with [EMAIL]. email_names = [] header_text_orig = "\n".join(text.splitlines()[:3]) email_match = re.search(r"([a-zA-Z]+)[._]([a-zA-Z]+)@", header_text_orig) if email_match: first = email_match.group(1).capitalize() last = email_match.group(2).capitalize() email_names.append((first, last, f"{first} {last}")) # --- Step A: Email addresses --- anonymized = re.sub( r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}", "[EMAIL]", text, ) # --- Step B: Person names via spaCy NER --- doc = self.nlp(anonymized) # Collect all PERSON spans and count mention frequency per canonical name # (use the first token as a rough canonical key to handle "Sarah" vs "Sarah Johnson") # Academic degree keywords — spaCy sometimes mislabels degree names as PERSON # e.g. "MSc Machine Learning", "BSc Mathematics", "PhD Computer Science" DEGREE_KEYWORDS = { "msc", "bsc", "ba", "ma", "mba", "phd", "llb", "llm", "beng", "meng", "doctorate", "bachelor", "master", "masters", "graduate", "postgraduate", } person_spans = [ (ent.start_char, ent.end_char, ent.text) for ent in doc.ents if ent.label_ == "PERSON" and not any(token.lower() in DEGREE_KEYWORDS for token in ent.text.split()) ] # Count frequency by normalised name (lower-case first token) name_freq: Counter = Counter() first_seen: dict = {} # normalised_name → first start_char for start, end, name in person_spans: key = name.strip().lower().split()[0] name_freq[key] += 1 if key not in first_seen: first_seen[key] = start # Seed name_freq with email-derived names not already found by spaCy for first, last, full in email_names: key = first.lower() if key not in name_freq: # Count how often this first name appears in the full text name_freq[key] = len(re.findall(rf"\b{re.escape(first)}\b", anonymized, re.IGNORECASE)) first_seen[key] = anonymized.lower().find(key) if name_freq: # Most-frequent name is the candidate candidate_key = name_freq.most_common(1)[0][0] # Remaining names ordered by first appearance other_keys = sorted( [k for k in name_freq if k != candidate_key], key=lambda k: first_seen.get(k, 9999), ) label_map = {candidate_key: "[CANDIDATE]"} for i, k in enumerate(other_keys, start=2): label_map[k] = f"[PERSON_{i}]" # Replace NER spans in reverse order to preserve char offsets for start, end, name in reversed(person_spans): key = name.strip().lower().split()[0] label = label_map.get(key, "[PERSON]") anonymized = anonymized[:start] + label + anonymized[end:] # Second pass: regex sweep for any remaining occurrences of known names # (catches names spaCy missed because they were next to org/title context). # Sort longest-first to avoid partial replacements. full_name_label: dict = {} for start, end, name in person_spans: key = name.strip().lower().split()[0] label = label_map.get(key, "[PERSON]") full_name_label[name.strip()] = label # Also add email-derived names that spaCy missed for first, last, full in email_names: key = first.lower() if key not in label_map: continue # Add both full name and individual tokens so all forms are caught for variant in [full, first, last]: if variant not in full_name_label: full_name_label[variant] = label_map[key] # Also add individual tokens (first name, last name) for each known person # so partial matches like "[CANDIDATE] Obi" get cleaned up token_label: dict = {} for full_name, label in full_name_label.items(): for token in full_name.split(): if len(token) > 2 and token not in token_label: token_label[token] = label # Replace full names first (longest first), then individual tokens for full_name, label in sorted(full_name_label.items(), key=lambda x: -len(x[0])): anonymized = re.sub(re.escape(full_name), label, anonymized) for token, label in sorted(token_label.items(), key=lambda x: -len(x[0])): anonymized = re.sub(r"\b" + re.escape(token) + r"\b", label, anonymized) # --- Step C: Pronouns, titles, gendered nouns --- replacements = [ # Pronoun + verb agreement (she/he → they) (r"\b[Ss]he has\b", "They have"), (r"\b[Ss]he is\b", "They are"), (r"\b[Ss]he was\b", "They were"), (r"\b[Ss]he works\b", "They work"), (r"\b[Ss]he often\b", "They often"), (r"\b[Ss]he also\b", "They also"), (r"\b[Ss]he always\b", "They always"), (r"\b[Ss]he then\b", "They then"), (r"\b[Ss]he quickly\b", "They quickly"), (r"\b[Ss]he approached\b", "They approached"), (r"\b[Ss]he independently\b", "They independently"), (r"\b[Hh]e has\b", "They have"), (r"\b[Hh]e is\b", "They are"), (r"\b[Hh]e was\b", "They were"), (r"\b[Hh]e works\b", "They work"), (r"\b[Hh]e often\b", "They often"), (r"\b[Hh]e also\b", "They also"), (r"\b[Hh]e always\b", "They always"), (r"\b[Hh]e then\b", "They then"), (r"\b[Hh]e quickly\b", "They quickly"), # Object pronoun: verb + her/him → verb + them (r"\benable [Hh]er\b", "enable them"), (r"\benable [Hh]im\b", "enable them"), (r"\bmade [Hh]er\b", "made them"), (r"\bmade [Hh]im\b", "made them"), (r"\bmake [Hh]er\b", "make them"), (r"\bmake [Hh]im\b", "make them"), (r"\bhelped [Hh]er\b", "helped them"), (r"\bhelped [Hh]im\b", "helped them"), (r"\ballow [Hh]er\b", "allow them"), (r"\ballow [Hh]im\b", "allow them"), (r"\bgave [Hh]er\b", "gave them"), (r"\bgave [Hh]im\b", "gave them"), (r"\btold [Hh]er\b", "told them"), (r"\btold [Hh]im\b", "told them"), (r"\basked [Hh]er\b", "asked them"), (r"\basked [Hh]im\b", "asked them"), (r"\bshowed [Hh]er\b", "showed them"), (r"\bshowed [Hh]im\b", "showed them"), (r"\btaught [Hh]er\b", "taught them"), (r"\btaught [Hh]im\b", "taught them"), (r"\boffered [Hh]er\b", "offered them"), (r"\boffered [Hh]im\b", "offered them"), (r"\bsent [Hh]er\b", "sent them"), (r"\bsent [Hh]im\b", "sent them"), (r"\bserve [Hh]er\b", "serve them"), (r"\bserve [Hh]im\b", "serve them"), # Possessive before nouns (r"\b[Hh]er(?=\s+\w)", "their"), (r"\b[Hh]is(?=\s+\w)", "their"), (r"(?<=[.!?]\s)their\b", "Their"), # Standalone pronouns (r"\bShe\b", "They"), (r"\bshe\b", "they"), (r"\bHe\b", "They"), (r"\bhe\b", "they"), (r"\b[Hh]im\b", "them"), (r"\b[Hh]erself\b", "themselves"), (r"\b[Hh]imself\b", "themselves"), # Subject-verb agreement: fix third-person singular verbs after "they" # These arise when "he/she + verb-s" is replaced with "they + verb-s". (r"\bThey pursues\b", "They pursue"), (r"\bthey pursues\b", "they pursue"), (r"\bThey works\b", "They work"), (r"\bthey works\b", "they work"), (r"\bThey leads\b", "They lead"), (r"\bthey leads\b", "they lead"), (r"\bThey manages\b", "They manage"), (r"\bthey manages\b", "they manage"), (r"\bThey brings\b", "They bring"), (r"\bthey brings\b", "they bring"), (r"\bThey demonstrates\b", "They demonstrate"), (r"\bthey demonstrates\b", "they demonstrate"), (r"\bThey contributes\b", "They contribute"), (r"\bthey contributes\b", "they contribute"), (r"\bThey shows\b", "They show"), (r"\bthey shows\b", "they show"), (r"\bThey excels\b", "They excel"), (r"\bthey excels\b", "they excel"), (r"\bThey delivers\b", "They deliver"), (r"\bthey delivers\b", "they deliver"), (r"\bThey handles\b", "They handle"), (r"\bthey handles\b", "they handle"), (r"\bThey drives\b", "They drive"), (r"\bthey drives\b", "they drive"), (r"\bThey seeks\b", "They seek"), (r"\bthey seeks\b", "they seek"), (r"\bThey holds\b", "They hold"), (r"\bthey holds\b", "they hold"), (r"\bThey oversees\b", "They oversee"), (r"\bthey oversees\b", "they oversee"), (r"\bThey possesses\b", "They possess"), (r"\bthey possesses\b", "they possess"), (r"\bThey believes\b", "They believe"), (r"\bthey believes\b", "they believe"), (r"\bThey strives\b", "They strive"), (r"\bthey strives\b", "they strive"), (r"\bThey pursues\b", "They pursue"), (r"\bthey pursues\b", "they pursue"), (r"\bThey approaches\b", "They approach"), (r"\bthey approaches\b", "they approach"), (r"\bThey applies\b", "They apply"), (r"\bthey applies\b", "they apply"), (r"\bThey takes\b", "They take"), (r"\bthey takes\b", "they take"), (r"\bThey makes\b", "They make"), (r"\bthey makes\b", "they make"), (r"\bThey meets\b", "They meet"), (r"\bthey meets\b", "they meet"), (r"\bThey comes\b", "They come"), (r"\bthey comes\b", "they come"), (r"\bThey goes\b", "They go"), (r"\bthey goes\b", "they go"), (r"\bThey gets\b", "They get"), (r"\bthey gets\b", "they get"), (r"\bThey gives\b", "They give"), (r"\bthey gives\b", "they give"), (r"\bThey says\b", "They say"), (r"\bthey says\b", "they say"), (r"\bThey knows\b", "They know"), (r"\bthey knows\b", "they know"), (r"\bThey thinks\b", "They think"), (r"\bthey thinks\b", "they think"), (r"\bThey sees\b", "They see"), (r"\bthey sees\b", "they see"), (r"\bThey uses\b", "They use"), (r"\bthey uses\b", "they use"), (r"\bThey builds\b", "They build"), (r"\bthey builds\b", "they build"), (r"\bThey creates\b", "They create"), (r"\bthey creates\b", "they create"), (r"\bThey develops\b", "They develop"), (r"\bthey develops\b", "they develop"), (r"\bThey supports\b", "They support"), (r"\bthey supports\b", "they support"), (r"\bThey helps\b", "They help"), (r"\bthey helps\b", "they help"), (r"\bThey joins\b", "They join"), (r"\bthey joins\b", "they join"), (r"\bThey plays\b", "They play"), (r"\bthey plays\b", "they play"), (r"\bThey runs\b", "They run"), (r"\bthey runs\b", "they run"), (r"\bThey sets\b", "They set"), (r"\bthey sets\b", "they set"), (r"\bThey starts\b", "They start"), (r"\bthey starts\b", "they start"), (r"\bThey stops\b", "They stop"), (r"\bthey stops\b", "they stop"), (r"\bThey writes\b", "They write"), (r"\bthey writes\b", "they write"), (r"\bThey reads\b", "They read"), (r"\bthey reads\b", "they read"), (r"\bThey speaks\b", "They speak"), (r"\bthey speaks\b", "they speak"), (r"\bThey listens\b", "They listen"), (r"\bthey listens\b", "they listen"), (r"\bThey learns\b", "They learn"), (r"\bthey learns\b", "they learn"), (r"\bThey teaches\b", "They teach"), (r"\bthey teaches\b", "they teach"), (r"\bThey represents\b", "They represent"), (r"\bthey represents\b", "they represent"), (r"\bThey reports\b", "They report"), (r"\bthey reports\b", "they report"), (r"\bThey serves\b", "They serve"), (r"\bthey serves\b", "they serve"), # Titles (remove) (r"\bMrs?\.\s*", ""), (r"\bMs\.\s*", ""), (r"\bMiss\s+", ""), # Gendered nouns (r"\b[Hh]usband\b", "spouse"), (r"\b[Ww]ife\b", "spouse"), (r"\b[Mm]other\b", "parent"), (r"\b[Ff]ather\b", "parent"), (r"\b[Ss]on\b", "child"), (r"\b[Dd]aughter\b", "child"), (r"\b[Bb]rother\b", "sibling"), (r"\b[Ss]ister\b", "sibling"), (r"\b[Bb]oyfriend\b", "partner"), (r"\b[Gg]irlfriend\b", "partner"), (r"\b[Ss]pokesman\b", "spokesperson"), (r"\b[Ss]pokeswoman\b","spokesperson"), (r"\b[Cc]hairman\b", "chairperson"), (r"\b[Cc]hairwoman\b", "chairperson"), (r"\b[Mm]anpower\b", "workforce"), (r"\b[Gg]irl\b", "person"), (r"\b[Bb]oy\b", "person"), (r"\b[Ww]oman\b", "person"), (r"\b[Ww]omen\b", "people"), (r"\b[Mm]an\b", "person"), (r"\b[Mm]en\b", "people"), (r"\b[Ll]ady\b", "person"), (r"\b[Ll]adies\b", "people"), (r"\b[Gg]entleman\b", "person"), (r"\b[Gg]entlemen\b", "people"), (r"\b[Ff]emale\b", "person"), (r"\b[Ff]emales\b", "people"), (r"\b[Mm]ale\b", "person"), (r"\b[Mm]ales\b", "people"), ] for pattern, replacement in replacements: anonymized = re.sub(pattern, replacement, anonymized) # Clean up artefacts anonymized = re.sub(r"\s{2,}", " ", anonymized) anonymized = re.sub(r"\.\s*\.", ".", anonymized) return anonymized.strip() def anonymize_document(self, text: str) -> dict: """ Full anonymization pipeline: Step A: Email regex → [EMAIL] Step B: spaCy NER → frequency-based [CANDIDATE] / [PERSON_N] labels Step C: Rule-based → replace pronouns, titles, gendered nouns Step D: Lexicon-based → replace gendered style words Note: LLM pass removed — small models hallucinate on CV-length text, and the deterministic pipeline handles anonymization reliably on its own. Sustainability covers the full pipeline. Returns dict with: original, surface_anonymized, fully_anonymized, sustainability. """ tracker = EmissionsTracker( project_name="nubias_cv_anonymize", log_level="error", save_to_file=False, ) tracker.start() # Steps A–C: surface anonymization (emails, names, pronouns, titles) surface_result = self._surface_anonymize(text) # Step D: Style word neutralization + subject-verb agreement fix. # Grammar is corrected deterministically — no LLM needed. fully_anonymized = self._replace_style_words(surface_result, self.style_replacements) sustainability = _build_sustainability(tracker) return { "original": text, "surface_anonymized": surface_result, "fully_anonymized": fully_anonymized, "sustainability": sustainability, } # ============================================= # Quick test (run this file directly to verify) # ============================================= if __name__ == "__main__": bd = BiasDetector() print("=" * 50) print("TEST 1: Job Posting Bias Detection") print("=" * 50) test_posting = "We need an aggressive go-getter who can dominate the competition." result = bd.analyze_job_posting(test_posting) print(f"Label: {result['overall_label']}") print(f"Score: {result['bias_score']}") print(f"Flagged: {[f['word'] for f in result['flagged_words']]}") print(f"Sustainability: {result['sustainability']}") print("\n" + "=" * 50) print("TEST 2: Job Posting Rewrite") print("=" * 50) result = bd.rewrite_job_posting(test_posting) print(f"Original: {result['original']}") print(f"Lexicon: {result['lexicon_fixed']}") print(f"Rewritten: {result['fully_rewritten']}") print(f"Sustainability: {result['sustainability']}") print("\n" + "=" * 50) print("TEST 3: CV Anonymization") print("=" * 50) test_cv = """Dr. Sarah Johnson (sarah.johnson@email.com) is an exceptionally warm and nurturing leader. She has always been deeply compassionate and sympathetic toward her colleagues. Her husband mentioned she is also a devoted mother who balances work and family gracefully. I, Prof. Michael Davies, am delighted to recommend her for this position.""" result = bd.anonymize_document(test_cv) print(f"SURFACE:\n{result['surface_anonymized']}\n") print(f"FULLY ANONYMIZED:\n{result['fully_anonymized']}") print(f"Sustainability: {result['sustainability']}")