Fix: TerminalPunctuationGuard for short phrases and comprehensive Nasb/Jazm grammar rules fixes

apply_locks.py

@@ -0,0 +1,77 @@
+import os
+
+def apply_lock_to_file(filepath, var_name, engine_name, func_name):
+    with open(filepath, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+        
+    out_lines = []
+    in_imports = False
+    added_threading = False
+    in_globals = False
+    added_lock_var = False
+    in_func = False
+    
+    for line in lines:
+        if line.startswith('import ') and not added_threading:
+            out_lines.append(line)
+            out_lines.append("import threading\n")
+            added_threading = True
+            continue
+            
+        if line.startswith(f'_{var_name} = None') and not added_lock_var:
+            out_lines.append(line)
+            out_lines.append(f"_load_lock = threading.Lock()\n")
+            added_lock_var = True
+            continue
+            
+        if line.startswith(f'def {func_name}('):
+            in_func = True
+            out_lines.append(line)
+            continue
+            
+        if in_func:
+            if line.startswith(f'    global '):
+                out_lines.append(line.replace('\n', f', _load_lock\n'))
+                continue
+                
+            if line.startswith(f'    try:'):
+                # The start of the old try block. We wrap everything from here.
+                out_lines.append(f'    with _load_lock:\n')
+                out_lines.append(f'        if _{var_name} is not None:\n')
+                out_lines.append(f'            return _{var_name}\n\n')
+                out_lines.append(f'        try:\n')
+                continue
+                
+            # If we are inside the function and past the global declaration,
+            # and it's indented with at least 4 spaces, we need to add 4 more spaces
+            # for the lines that were inside the old `try:` and `except:`
+            # EXCEPT for `if _xxx is not None: return _xxx` which comes before the try
+            if line.startswith('    if _') or line.startswith('        return _'):
+                # This is the old `if checker is not None:` logic before try. Leave it alone.
+                out_lines.append(line)
+                continue
+                
+            if line.startswith('    '):
+                # Shift everything that was inside try/except right by 4 spaces
+                if line.strip() == '':
+                    out_lines.append('\n')
+                else:
+                    out_lines.append('    ' + line)
+                
+                if line.startswith('    return _') or line.startswith('    raise RuntimeError'):
+                    # End of function
+                    in_func = False
+                continue
+
+        out_lines.append(line)
+
+    with open(filepath, 'w', encoding='utf-8') as f:
+        f.writelines(out_lines)
+
+
+apply_lock_to_file(r'src/nlp/spelling/araspell_service.py', 'spell_checker', 'AraSpell', 'get_spelling_model')
+apply_lock_to_file(r'src/nlp/punctuation/punctuation_service.py', 'punctuation_checker', 'PuncAra', 'get_punctuation_model')
+apply_lock_to_file(r'src/nlp/grammar/grammar_service.py', 'grammar_checker', 'Grammar', 'get_grammar_model')
+apply_lock_to_file(r'src/nlp/autocomplete/autocomplete_service.py', 'autocomplete_engine', 'Autocomplete', 'get_autocomplete_model')
+
+print("Locks applied perfectly with correct indentation!")

debug_pipeline.py

@@ -0,0 +1,24 @@
+import sys
+from pathlib import Path
+
+# Add src to python path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+from app import app
+import json
+
+client = app.test_client()
+
+def test(text):
+    print(f"\n--- Testing: {text} ---")
+    resp = client.post('/api/analyze', json={'text': text})
+    data = resp.get_json()
+    if 'suggestions' in data:
+        for s in data['suggestions']:
+            print(f"[{s['type'].upper()}] '{s['original']}' -> '{s['correction']}'")
+    else:
+        print("Error:", data)
+
+test("ذهبت المهندسون الي العمل")
+test("ذهبت المهندسون")
+test("الي العمل")

debug_punctuation.py

@@ -0,0 +1,45 @@
+import sys
+import io
+from pathlib import Path
+
+# Force UTF-8 encoding for standard output
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+
+# Add src to python path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+from app import app
+
+client = app.test_client()
+
+def test(text):
+    print(f"\n--- Testing: {text} ---")
+    resp = client.post('/api/analyze', json={'text': text})
+    data = resp.get_json()
+    
+    if data and 'corrected' in data:
+        print(f"Corrected: {data['corrected']}")
+        
+    if data and 'suggestions' in data:
+        for s in data['suggestions']:
+            print(f"[{s['type'].upper()}] '{s['original']}' -> '{s['correction']}'")
+    elif data and 'error' in data:
+        print("Error:", data['error'])
+    else:
+        print("Raw Data:", data)
+
+if __name__ == "__main__":
+    # Test 1: Single Entity (Should not have punctuation added)
+    test("شركة أبل")
+    
+    # Test 2: Short phrase (Should not have punctuation added)
+    test("مرحبا بكم في التطبيق")
+    
+    # Test 3: Grammar error that gets fixed but then corrupted by punctuation
+    # "الى" is spelled wrong (needs hamza on alif below if it's إِلى or just remains الى depending on rules, 
+    # but let's see what grammar does). Actually "يذهبون المهندسون" is a grammar error in Arabic 
+    # (should be يذهب المهندسون).
+    test("يذهبون المهندسون الى الشركة")
+    
+    # Test 4: Verify Grammar Model preserves punctuation
+    test("يذهبون المهندسون الى الشركة، أليس كذلك؟")

extract_grammar_fails.py

@@ -0,0 +1,23 @@
+import json
+import sys
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+try:
+    with open('tests/phase10/reports/phase10_results.json', 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    with open('grammar_fails_output.md', 'w', encoding='utf-8') as out_f:
+        out_f.write("=== GRAMMAR FALSE NEGATIVES ===\n")
+        for r in data.get('results', []):
+            if r.get('dataset') == 'grammar' and r.get('pipeline_verdict') == 'FN':
+                out_f.write(f"[{r.get('id')}] - {r.get('category')}\n")
+                out_f.write(f"  IN:       {r.get('input')}\n")
+                out_f.write(f"  EXP:      {r.get('expected')}\n")
+                out_f.write(f"  RAW_GRAM: {r.get('grammar_raw_output')}\n")
+                out_f.write(f"  FINAL:    {r.get('pipeline_output')}\n")
+                out_f.write("-" * 50 + "\n")
+
+
+except Exception as e:
+    print(f"Error: {e}")

grammar_fails_output.md

@@ -0,0 +1,37 @@
+=== GRAMMAR FALSE NEGATIVES ===
+[G006] - sv_agree
+  IN:       الأولاد لعب في الحديقة
+  EXP:      لعبوا
+  RAW_GRAM: الأولاد لعبوَ في الحديقة
+  FINAL:    الأولاد لعب في الحديقة.
+--------------------------------------------------
+[G009] - sv_agree
+  IN:       العمال بنى المبنى
+  EXP:      بنوا
+  RAW_GRAM: العمال بنى المبنى
+  FINAL:    العمال بنى المبنى.
+--------------------------------------------------
+[G013] - gender
+  IN:       الطالبة متفوق في دراسته
+  EXP:      متفوقة/دراستها
+  RAW_GRAM: الطالب متفوق في دراسته
+  FINAL:    الطالب متفوق في دراسته.
+--------------------------------------------------
+[G022] - five_nouns
+  IN:       رأيت أخوك في المسجد
+  EXP:      أخاك
+  RAW_GRAM: رأيت أخوك في المسجد
+  FINAL:    رأيت أخوك في المسجد
+--------------------------------------------------
+[G026] - dual
+  IN:       هاتان الطالبان مجتهدان
+  EXP:      هذان
+  RAW_GRAM: هذان الطالبان مجتهدان
+  FINAL:    هاتان الطالبات مجتهدان.
+--------------------------------------------------
+[G028] - nasb
+  IN:       لم يفعلون الواجب بعد
+  EXP:      يفعلوا
+  RAW_GRAM: لم يفعلوَ الواجب بعد
+  FINAL:    لم يفعلون الواجب بعد
+--------------------------------------------------

reports/Phase10_Post_IVtoOOV_Audit.md

@@ -0,0 +1,72 @@
+# Phase 10 Benchmark Audit — Post IVtoOOV Removal
+
+> **Date**: 2026-06-24
+> **Action**: Removed `IVtoOOV` filter and added advanced `camel-tools` POS tagging for dual/plural noun-adjective agreement.
+
+## 1. Top-Level Aggregate Metrics
+
+| Metric | Score | Notes |
+|---|---|---|
+| **Overall Pass Rate** | **56.2%** | Massive improvement (previously ~25%) |
+| Total Tests | 320 | |
+| True Positives (TP) | 95 | Successfully fixed real errors |
+| True Negatives (TN) | 85 | Successfully ignored correct text |
+| False Positives (FP) | 79 | Hallucinations or overcorrections |
+| False Negatives (FN) | 61 | Failed to fix real errors |
+
+### Root Cause Analysis (61 FN + 79 FP = 140 Failures)
+- **Punctuation Model (`MODEL:punctuation`)**: 64 failures
+- **Integration/Collisions (`PIPELINE:integration`)**: 32 failures
+- **Spelling Model (`MODEL:spelling`)**: 21 failures
+- **Grammar Model (`MODEL:grammar`)**: 18 failures
+
+---
+
+## 2. Per-Dataset Breakdown
+
+### Grammar Dataset
+* **Pass Rate:** 57.8% (up from 26.7%)
+* **Recall:** 80.0% (up from 40.0%)
+* **Analysis:** Removing `IVtoOOV` successfully unblocked valid grammatical structural changes. The recall doubled.
+* **Remaining Issue:** High False Positive Rate on the `correct...` category. The model hallucinates changes on already perfect text.
+
+### Spelling Dataset
+* **Pass Rate:** 63.7% (up from 42.5%)
+* **Recall:** 79.4%
+* **Remaining Issue:** Still missing some Hamza errors and complex word splits (`عندالباب` -> `عند الباب`).
+
+### Structured Content & Religious Datasets
+* **Structured Pass Rate:** 82.9% (up from 5.7%)
+* **Religious Pass Rate:** 90.0% (up from 10.0%)
+* **Analysis:** The `DigitGuard` and punctuation bypass rules are working incredibly well to protect specialized text.
+
+### Pipeline Collision Dataset
+* **Pass Rate:** 16.0% (Terrible)
+* **False Negative Rate:** 84.0%
+* **Analysis:** When a spelling error is adjacent to a grammar error, `StageLocker` is locking the word and preventing the grammar model from seeing or fixing the grammatical context.
+
+### Entities Dataset
+* **Pass Rate:** 13.3%
+* **Analysis:** The models (especially punctuation and spelling) are aggressively modifying named entities (people, places).
+
+---
+
+## 3. Strategic Action Plan for Enhancements
+
+To push the pass rate from **56.2%** to **>80%**, we must address the following critical areas:
+
+### A. Tame the "StageLocker" (Fix Pipeline Collisions)
+The `StageLocker` in `app.py` enforces a rigid "Spelling locks word X, Grammar cannot touch word X" rule. This breaks multi-stage corrections.
+**Solution:** Relax the `StageLocker`. Allow the grammar model to operate on tokens that were modified by spelling, provided the grammatical change doesn't completely revert the spelling correction (e.g., checking Jaccard distance or allowing suffix-only changes to locked words).
+
+### B. Stop Punctuation Hallucinations
+The punctuation model causes **64 failures**, mostly by adding periods `.` or question marks `؟` to the end of short sentences or entities where they don't belong.
+**Solution:** Implement a strict `TerminalPunctuationGuard`. If the original text is < 5 words and doesn't end in punctuation, automatically strip any trailing punctuation added by the model.
+
+### C. Implement Named Entity Recognition (NER) Bypass
+Entities (Person names, Cities) are failing at an 86% rate.
+**Solution:** Integrate `camel-tools` NER (Named Entity Recognition). Scan the input text for `LOC`, `PERS`, and `ORG`. If a word is an entity, add it to a dynamic whitelist so the Spelling and Grammar models skip it entirely.
+
+### D. Tame Grammar Hallucinations on Correct Text
+The grammar model hallucinates on perfectly correct text.
+**Solution:** Use a POS-based confidence score. If the grammar model attempts to change a noun into a verb, or completely alters the POS structure of an already valid sentence, reject the change. Alternatively, enforce stricter `Jaccard_05` checks for non-structural changes.

src/app.py

@@ -1101,6 +1101,15 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
         # Exception: if diff is just adding/removing ا at start (hamza)
         if abs(len(orig_word) - len(corr_word)) > 1:
             return 0.0
+
+    # ── FIX: Block Grammar Changes masked as Spelling Typos (Dual → Plural) ──
+    if orig_word.endswith('ان') and corr_word.endswith('ات') and orig_word[:-2] == corr_word[:-2]:
+        logger.info(
+            f"[SPELLING] Blocked grammatical change (Dual→Plural): "
+            f"'{orig_word}'→'{corr_word}'"
+        )
+        return 0.0
+
     # ── Phase 12 (A1): Keyboard-neighbor and phonetic acceptance ──
     # Check each differing character: ortho → full accept, keyboard/phonetic → dampened
     _has_keyboard_or_phonetic = False

src/nlp/grammar/grammar_rules.py

@@ -118,7 +118,8 @@ class ArabicGrammarGuard:
                 if prev_word in jazm_particles or word.startswith('ل') or word.startswith('ول'):
                     is_jazm_context = True
 
-            if pos_tag == 'verb' and (is_nasb_context or is_jazm_context):
+            is_present_tense = word.startswith('ي') or word.startswith('ت') or word.startswith('ن') or word.startswith('أ')
+            if (pos_tag == 'verb' or is_present_tense) and (is_nasb_context or is_jazm_context):
                 if word.endswith('ون'):
                     word = word[:-2] + 'وا'
                 elif word.endswith('ان'):
@@ -126,12 +127,30 @@ class ArabicGrammarGuard:
                 elif word.endswith('ين'):
                     word = word[:-2] + 'ي'
                 elif is_jazm_context:
-                    if word.endswith('و') and len(word) > 3:
+                    if word.endswith('و') and len(word) >= 3:
                         word = word[:-1] + 'ُ'
-                    elif (word.endswith('i') or word.endswith('ي')) and len(word) > 3:
-                        if word.endswith('ي'): word = word[:-1] + 'ِ'
-                    elif (word.endswith('ى') or word.endswith('ا')) and len(word) > 3:
-                        word = word[:-1] + 'َ'
+                    elif word.endswith('ي') or word.endswith('i'):
+                        stem = word[:-1]
+                        fatha_stems = {'يسع', 'تسع', 'أسع', 'نسع', 
+                                       'يخش', 'تخش', 'أخش', 'نخش',
+                                       'يرض', 'ترض', 'أرض', 'نرض',
+                                       'ينس', 'تنس', 'أنس', 'ننس',
+                                       'يبق', 'تبق', 'أبق', 'نبق',
+                                       'ير', 'تر', 'أر', 'نر',
+                                       'يلق', 'تلق', 'ألق', 'نلق',
+                                       'ينه', 'تنه', 'أنه', 'ننه'}
+                        if stem in fatha_stems:
+                            word = stem + 'َ'
+                        elif len(word) > 3:
+                            word = stem + 'ِ'
+                    elif (word.endswith('ى') or word.endswith('ا')) and len(word) >= 3:
+                        if not word.endswith('وا'):
+                            word = word[:-1] + 'َ'
+                elif is_nasb_context:
+                    if word.endswith('و') and len(word) > 3:
+                        word = word + 'َ'
+                    elif word.endswith('ي') and len(word) > 3:
+                        word = word + 'َ'
 
             corrected_tokens.append(word)
         return " ".join(corrected_tokens)
@@ -325,15 +344,23 @@ class ArabicGrammarGuard:
                 elif is_plural_masc:
                     if (not verb_word.endswith('ون') and not verb_word.endswith('وا')
                             and not verb_word.endswith('ين')):
+                        if verb_word.endswith('وَ'):
+                            verb_word = verb_word[:-1]
                         corrected_tokens[i+1] = verb_word + 'ون'
             else:
                 # Past tense: ذهب→ذهبوا (masc) / ذهبن (fem)
                 if is_plural_fem:
                     if not verb_word.endswith('ن') and not verb_word.endswith('نَ'):
+                        if verb_word.endswith('ى') or verb_word.endswith('ا'):
+                            verb_word = verb_word[:-1]
                         corrected_tokens[i+1] = verb_word + 'ن'
                 elif is_plural_masc:
                     if (not verb_word.endswith('وا') and not verb_word.endswith('ون')
                             and not verb_word.endswith('ين')):
+                        if verb_word.endswith('وَ'):
+                            verb_word = verb_word[:-1]
+                        elif verb_word.endswith('ى') or verb_word.endswith('ا'):
+                            verb_word = verb_word[:-1]
                         corrected_tokens[i+1] = verb_word + 'وا'
 
         return " ".join(corrected_tokens)
@@ -343,6 +370,10 @@ class ArabicGrammarGuard:
         text = re.sub(r'\b(إن|أن|كأن|لكن|لعل|ليت)\s+(أبوك|أخوك|ذو|فوك)\b',
                       lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ا')}", text)
 
+        # الأفعال المتعدية (Object position)
+        text = re.sub(r'\b(رأيت|شاهدت|قابلت|زرت|سمعت|عرفت|وجدت|أحب|أكرمت|صادفت)\s+(أبوك|أخوك|ذو|فوك)\b',
+                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ا')}", text)
+
         # حروف الجر المنفصلة بمسافة (في أخوك -> في أخيك)
         text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن))\s+(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
                       lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
@@ -509,6 +540,19 @@ class ArabicGrammarGuard:
         """Apply all grammar rules to model output."""
         text = self.preserve_numbers(original_text, generated_text)
         
+        # ── Fix Hallucinated Subject Gender ──
+        # If model incorrectly changes female subject to male, restore it.
+        orig_words = original_text.split()
+        corr_words = text.split()
+        if len(orig_words) == len(corr_words):
+            for i, (o, c) in enumerate(zip(orig_words, corr_words)):
+                o_clean = o.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                c_clean = c.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                # If model dropped 'ة' from a word of length >= 4
+                if o_clean.endswith('ة') and not c_clean.endswith('ة') and o_clean[:-1] == c_clean:
+                    corr_words[i] = o
+            text = " ".join(corr_words)
+
         # Each rule is wrapped in try/except so that if camel-tools
         # functions fail, the regex-based rules still execute.
         for rule_name, rule_fn in [
@@ -527,8 +571,7 @@ class ArabicGrammarGuard:
                 text = rule_fn(text)
             except Exception as e:
                 logger.warning(f"[GRAMMAR-RULES] {rule_name} failed: {e}")
-                # Continue with remaining rules
-        
+
         text = re.sub(r'\s+', ' ', text).strip()
         return text
 

src/nlp/punctuation/punctuation_rules.py

@@ -149,7 +149,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
                     # Also check for ellipsis (... at end)
                     _full_has_ellipsis = full_text.rstrip().endswith('...') if full_text else False
 
-                    if _full_word_count >= 3 and not _full_already_has_terminal and not _full_has_ellipsis:
+                    if _full_word_count >= 5 and not _full_already_has_terminal and not _full_has_ellipsis:
                         # ── FIX-29: Exclamation mark guard ──
                         # PuncAra sometimes adds ! to declarative sentences.
                         # Only allow ! if text contains exclamatory cues.
@@ -161,7 +161,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
                             _has_cue = any(w in _EXCL_CUES for w in full_text.split())
                             if not _has_cue:
                                 logger.info(
-                                    f"[PUNC-SAFETY] Blocked !/?  on declarative sentence: "
+                                    f"[PUNC-SAFETY] Blocked !/? on declarative sentence: "
                                     f"'{original}' → '{correction}'"
                                 )
                                 return False
@@ -175,7 +175,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
                     else:
                         # Short fragment OR already has terminal punct → REJECT
                         logger.info(
-                            f"[PUNC-SAFETY] Rejected terminal punct injection: "
+                            f"[PUNC-SAFETY] TerminalPunctuationGuard triggered: removing trailing punctuation "
                             f"'{original}' → '{correction}'"
                         )
                         return False

src/nlp/spelling/araspell_service.py

@@ -76,8 +76,8 @@ def get_spelling_model():
 
         epoch = checkpoint.get('epoch', 'N/A')
         logger.info(f"Spelling model loaded on {device}, epoch: {epoch}")
-
         # 6. Initialize the spell checker pipeline (contextual=True for MLM-based refinement)
+
         from nlp.spelling.araspell_rules import ArabicSpellChecker
         _spell_checker = ArabicSpellChecker(
             model, tokenizer, device, use_contextual=True

test_grammar_fixes.py

@@ -0,0 +1,18 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+from src.app import init_pipeline
+
+p = init_pipeline()
+tests = [
+    'الأولاد لعب في الحديقة',
+    'العمال بنى المبنى',
+    'الطالبة متفوق في دراسته',
+    'رأيت أخوك في المسجد',
+    'هاتان الطالبان مجتهدان',
+    'لم يفعلون الواجب بعد'
+]
+
+with open('test_grammar_output.md', 'w', encoding='utf-8') as f:
+    for t in tests:
+        res = p.analyze(t)['corrected']
+        f.write(f"IN: {t}\nOUT: {res}\n---\n")

test_models.py

@@ -0,0 +1,24 @@
+import sys
+import os
+from pathlib import Path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+from src.nlp.spelling.araspell_service import get_spelling_model
+from src.nlp.grammar.grammar_service import get_grammar_model
+
+print("Loading Spelling...")
+spell = get_spelling_model()
+print("Loading Grammar...")
+grammar = get_grammar_model()
+
+text1 = "لم ينمو الاقتصاد كالمعتاد"
+text2 = "ذهبت المهندسون"
+
+print(f"\n--- Text 1: {text1} ---")
+print("AraSpell output:", spell.correct(text1))
+print("Grammar output:", grammar.correct(text1))
+
+print(f"\n--- Text 2: {text2} ---")
+print("AraSpell output:", spell.correct(text2))
+print("Grammar output:", grammar.correct(text2))
+

test_punctuation.py

@@ -0,0 +1,17 @@
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+
+from src.app import init_pipeline
+
+p = init_pipeline()
+
+tests = [
+    "الخطة السنوية للشركة",         # 3 words -> should block trailing .
+    "هذا هو تقرير المبيعات",      # 4 words -> should block trailing .
+    "محمد ذهب إلى المدرسة اليوم", # 5 words -> allowed!
+]
+
+with open('test_punct_output.md', 'w', encoding='utf-8') as f:
+    for t in tests:
+        res = p.analyze(t)['corrected']
+        f.write(f"IN: {t}\nOUT: {res}\n---\n")

tests/phase10/benchmark_runner.py

@@ -9,6 +9,7 @@ Usage:
     python tests/phase10/benchmark_runner.py [--url URL] [--dataset NAMES] [--out DIR]
 """
 import argparse, json, time, re, os, sys
+sys.stdout.reconfigure(encoding='utf-8')
 from pathlib import Path
 from dataclasses import dataclass, field, asdict
 from typing import List, Dict, Optional, Any