Add debugging scripts and fix grammar/punctuation logic

camel_test_out.json

@@ -0,0 +1,112 @@
+[
+  {
+    "word": "يسعى",
+    "diac": "يَسْعَى",
+    "lex": "سَعَى",
+    "pos": "verb",
+    "per": "3",
+    "num": "s",
+    "gen": "m",
+    "root": "س.ع.#",
+    "stem": "سْعَى"
+  },
+  {
+    "word": "يسعي",
+    "diac": "يَسْعَى",
+    "lex": "سَعَى",
+    "pos": "verb",
+    "per": "3",
+    "num": "s",
+    "gen": "m",
+    "root": "س.ع.#",
+    "stem": "سْعَى"
+  },
+  {
+    "word": "يدعو",
+    "diac": "يَدْعُو",
+    "lex": "دَعا",
+    "pos": "verb",
+    "per": "3",
+    "num": "s",
+    "gen": "m",
+    "root": "د.ع.#",
+    "stem": "دْعُو"
+  },
+  {
+    "word": "يمشي",
+    "diac": "يَمْشَى",
+    "lex": "مَشَى",
+    "pos": "verb",
+    "per": "3",
+    "num": "s",
+    "gen": "m",
+    "root": "م.ش.#",
+    "stem": "مْشَى"
+  },
+  {
+    "word": "يتقي",
+    "diac": "يَتَّقِي",
+    "lex": "ٱِتَّقَى",
+    "pos": "verb",
+    "per": "3",
+    "num": "s",
+    "gen": "m",
+    "root": "#.ق.#",
+    "stem": "تَّقِي"
+  },
+  {
+    "word": "يسعون",
+    "diac": "يَسْعَُونَ",
+    "lex": "سَعَى",
+    "pos": "verb",
+    "per": "3",
+    "num": "p",
+    "gen": "m",
+    "root": "س.ع.#",
+    "stem": "سْعَ"
+  },
+  {
+    "word": "يذهبون",
+    "diac": "يَذْهَبُونَ",
+    "lex": "ذَهَب",
+    "pos": "verb",
+    "per": "3",
+    "num": "p",
+    "gen": "m",
+    "root": "ذ.ه.ب",
+    "stem": "ذْهَب"
+  },
+  {
+    "word": "تذهبين",
+    "diac": "تَذْهَبِينَ",
+    "lex": "ذَهَب",
+    "pos": "verb",
+    "per": "2",
+    "num": "s",
+    "gen": "f",
+    "root": "ذ.ه.ب",
+    "stem": "ذْهَب"
+  },
+  {
+    "word": "تذهبان",
+    "diac": "تَذْهَبانِ",
+    "lex": "ذَهَب",
+    "pos": "verb",
+    "per": "2",
+    "num": "d",
+    "gen": "m",
+    "root": "ذ.ه.ب",
+    "stem": "ذْهَب"
+  },
+  {
+    "word": "متي",
+    "diac": "مَتَى",
+    "lex": "مَتَى",
+    "pos": "conj",
+    "per": "na",
+    "num": "na",
+    "gen": "na",
+    "root": "م.ت.#",
+    "stem": "مَتَى"
+  }
+]

debug_pipeline.py

@@ -1,6 +1,7 @@
 import sys
+import io
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
 from pathlib import Path
-
 # Add src to python path
 sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
 

src/nlp/grammar/grammar_rules.py

@@ -53,18 +53,6 @@ class ArabicGrammarGuard:
                     if w1_word.endswith('ون'): corrected_tokens[i] = w1_word[:-2]
                     elif w1_word.endswith('وا'): corrected_tokens[i] = w1_word[:-2]
 
-            elif w1_pos == 'noun' and w2_pos == 'verb':
-                if w1_word.endswith('ون') and not (w2_word.endswith('ون') or w2_word.endswith('وا') or w2_word.endswith('ين')):
-                    if w2_info.analysis.get('num') == 's':
-                        corrected_tokens[i+1] = w2_word + 'ون'
-
-            # Match adjectives (adj) only; skip words starting with ب or ending with alef tanween
-            elif w1_pos == 'noun' and w2_pos == 'adj':
-                if w1_word.endswith('ون') and not w2_word.endswith('ون'):
-                    if w2_info.analysis.get('num') == 's' and w2_info.analysis.get('gen') == 'm':
-                        if len(w2_word) > 2 and not w2_word.endswith('ا') and not w2_word.startswith('ب'):
-                            corrected_tokens[i+1] = w2_word + 'ون'
-
         return " ".join(corrected_tokens)
 
     def smart_asmaa_khamsa_fix(self, text):
@@ -369,9 +357,8 @@ class ArabicGrammarGuard:
             verb_num = verb_info.analysis.get('num', 's')
 
             # Skip if verb is already plural
-            if verb_num != 's':
-                continue
-
+            # Removed singular verb check to allow fixing gender mismatch on already plural verbs (e.g. البنات يذهبون -> يذهبن)
+            
             # Only trigger on CONFIRMED plurals:
             # 1. Known broken plural nouns (hardcoded list)
             # 2. Sound masculine plural ending in ون/ين
@@ -429,9 +416,13 @@ class ArabicGrammarGuard:
                         is_plural_fem = True
                     else:
                         is_plural_masc = True
-
+            
+            is_singular_fem = False
             if not is_plural_masc and not is_plural_fem:
-                continue
+                if noun_gen == 'f' or noun_word.endswith('ة') or noun_word in KNOWN_FEMININE_NOUNS:
+                    is_singular_fem = True
+                else:
+                    continue
 
             # Fix the verb to agree with the plural subject
             # Detect if verb is present tense (starts with ي/ت/ن/أ)
@@ -453,12 +444,15 @@ class ArabicGrammarGuard:
                         if verb_word.endswith('وَ'):
                             verb_word = verb_word[:-1]
                         corrected_tokens[i+1] = verb_word + 'ون'
+                elif is_singular_fem:
+                    if verb_word.startswith('ي'):
+                        corrected_tokens[i+1] = 'ت' + verb_word[1:]
             else:
                 # Past tense: ذهب→ذهبوا (masc) / ذهبن (fem)
                 if is_plural_fem:
                     if verb_word.endswith('وا') or verb_word.endswith('ون'):
                         verb_word = verb_word[:-2]
-                    elif verb_word.endswith('ت') or verb_word.endswith('تْ') or verb_word.endswith('تَ'):
+                    elif verb_word.endswith('ت') or verb_word.endswith('تْ') or verb_word.endswith('تَ') or verb_word.endswith('و'):
                         verb_word = verb_word[:-1]
                     if not verb_word.endswith('ن') and not verb_word.endswith('نَ'):
                         if verb_word.endswith('ى') or verb_word.endswith('ا'):
@@ -476,21 +470,31 @@ class ArabicGrammarGuard:
                         elif verb_word.endswith('ى') or verb_word.endswith('ا'):
                             verb_word = verb_word[:-1]
                         corrected_tokens[i+1] = verb_word + 'وا'
+                elif is_singular_fem:
+                    if not verb_word.endswith('ت') and not verb_word.endswith('تْ') and not verb_word.endswith('تَ'):
+                        if verb_word.endswith('ى'):
+                            verb_word = verb_word[:-1] + 'ا'
+                        corrected_tokens[i+1] = verb_word + 'ت'
 
         return " ".join(corrected_tokens)
 
     def regex_rules_fallback(self, text):
+        def _add_hamza(word):
+            if word.startswith('ا') and not word.startswith('ال'):
+                return 'أ' + word[1:]
+            return word
+
         # إن وأخواتها
         text = re.sub(r'\b(إن|أن|كأن|لكن|لعل|ليت|ان|كان)\s+(أبوك|ابوك|أخوك|اخوك|ذو|فوك)\b',
-                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ا')}", text)
+                      lambda m: f"{m.group(1)} {_add_hamza(m.group(2)).replace('و', 'ا')}", text)
 
         # الأفعال المتعدية (Object position)
         text = re.sub(r'\b(رأيت|شاهدت|قابلت|زرت|سمعت|عرفت|وجدت|أحب|أكرمت|صادفت)\s+(أبوك|ابوك|أخوك|اخوك|ذو|فوك)\b',
-                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ا')}", text)
+                      lambda m: f"{m.group(1)} {_add_hamza(m.group(2)).replace('و', 'ا')}", text)
 
         # حروف الجر المنفصلة بمسافة (في أخوك -> في أخيك)
         text = re.sub(r'\b([وف]?(?:في|من|إلى|الي|على|علي|عن))\s+(أبوك|ابوك|أباك|اباك|أخوك|اخوك|أخاك|اخاك|ذو|ذا)\b',
-                      lambda m: f"{m.group(1)} {m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
+                      lambda m: f"{m.group(1)} {_add_hamza(m.group(2)).replace('و', 'ي').replace('ا', 'ي')}", text)
 
         # حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
         text = re.sub(r'\b([وف]?[بل])(أبوك|ابوك|أباك|اباك|أخوك|اخوك|أخاك|اخاك|ذو|ذا)\b',
@@ -500,6 +504,11 @@ class ArabicGrammarGuard:
         # were REMOVED because they caused massive overcorrection on correct text.
         # These patterns are handled by CamelTools-based rules (fix_prepositions_advanced,
         # fix_verbs_nasb_and_jazm) which have POS-tag awareness.
+        
+        # FIX-PC010: Add targeted safe regex for Nasb/Jazm particles + verb
+        # Only match clear present tense verbs starting with ي/ت/ن/أ and ending in ون
+        text = re.sub(r'\b(أن|ان|لن|كي|حتى|لم|لما)\s+([يتا][\u0600-\u06FF]{2,})ون\b',
+                      r'\1 \2وا', text)
 
         return text
 

test_failures.py

@@ -0,0 +1,9 @@
+import json
+import io
+import sys
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+with open('tests/phase10/reports/collision_benchmark_results.json', 'r', encoding='utf-8') as f:
+    data = json.load(f)
+for r in data['results']:
+    if r.get('pipeline_verdict') != 'PASS':
+        print(f"{r['id']} | {r['category']} | Expected: {r['expected']} | Actual: {r.get('pipeline_output', '')}")

test_grammar_logic.py

@@ -0,0 +1,27 @@
+import sys
+import codecs
+from pathlib import Path
+
+sys.stdout.reconfigure(encoding='utf-8')
+
+# Add src to python path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+from nlp.grammar.grammar_rules import ArabicGrammarGuard
+
+guard = ArabicGrammarGuard()
+
+tests = [
+    "المهندسون صممت المشروع",  # PC001 spelling output
+    "البنات يذهبون إلى المدرسة", # PC003 spelling output
+    "أن ابوك رجل طيب", # PC007 spelling output
+    "العمال بنى المبنى الجديد", # PC020 spelling output
+    "الامهات طبخ الطعام والاطفال لعب", # PC043 spelling output
+    "المديره وافق علي المشروع والموظفات وافق أيضا" # PC046
+]
+
+for t in tests:
+    print(f"\n[IN]  {t}")
+    out = guard.apply_rules(t)
+    print(f"[OUT] {out}")
+

test_pc.py

@@ -0,0 +1,20 @@
+import io
+import sys
+import unittest
+from pathlib import Path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+import logging
+from app import app as flask_app
+
+class TestPC004(unittest.TestCase):
+    def test_pc004(self):
+        text = "الرجال يعملون في المصنعو"
+        client = flask_app.test_client()
+        response = client.post('/api/analyze', json={'text': text})
+        res = response.get_json()
+        print("Final:", res.get('corrected_text', res))
+
+if __name__ == '__main__':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    unittest.main()

test_sv.py

@@ -0,0 +1,45 @@
+import io
+import sys
+import unittest
+from pathlib import Path
+sys.path.insert(0, str(Path(r'c:\Users\dell\PycharmProjects\JupyterProject1\PythonProject\BAYAN\src')))
+
+from app import apply_patches
+
+class TestPC003(unittest.TestCase):
+    def test_pc003(self):
+        orig_text = "البنات يذهبون الي المدرسه"
+        spelling_corr = "البنات يذهبون إلى المدرسة"
+        grammar_corr = "البنات يذهبن إلى المدرسة"
+        
+        from src.nlp.stage_locker import StageLocker
+        class MockCtx:
+            def __init__(self):
+                self.stage_locker = StageLocker()
+                self.current_text = orig_text
+                from nlp.offset_mapper import OffsetMapper
+                self._mapper = OffsetMapper()
+                self._mapper.init(orig_text)
+            
+            def add_patch(self, stage, start, end, corr, **kwargs):
+                self.stage_locker.add_patch(stage, start, end)
+                orig_start, orig_end = self._mapper.map_range(start, end)
+                print(f"Adding patch: {self.current_text[start:end]} -> {corr} (mapped to {orig_start}:{orig_end})")
+            
+            def mutate_text(self, new_text, mapper_cls):
+                self.current_text = new_text
+                self._mapper.init(new_text)
+
+        ctx = MockCtx()
+        
+        # Apply spelling
+        apply_patches(orig_text, orig_text, spelling_corr, "spelling", ctx)
+        ctx.mutate_text(spelling_corr, None)
+        
+        # Apply grammar
+        final = apply_patches(orig_text, spelling_corr, grammar_corr, "grammar", ctx)
+        print("Final:", final)
+
+if __name__ == '__main__':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    unittest.main()