Bayan AI commited on
Commit
dd8a53f
·
1 Parent(s): d8125c8

Refactor: Remove obsolete IVtoOOV filter to prevent False Negatives

Browse files
Files changed (1) hide show
  1. src/app.py +0 -26
src/app.py CHANGED
@@ -2127,32 +2127,6 @@ def analyze_text():
2127
  if _gram_dir_blocked:
2128
  continue
2129
 
2130
- if not _is_grammar_pattern:
2131
- if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
2132
- try:
2133
- from nlp.spelling.araspell_service import get_spelling_model
2134
- _vm = get_spelling_model().vocab_manager
2135
- if _vm:
2136
- # ── Phase 12 (B3): Strip diacritics before IV/OOV check ──
2137
- # Grammar model sometimes outputs correct words with
2138
- # diacritics (e.g. يفعلوَ) which fail OOV check.
2139
- # Strip diacritics for vocabulary check only.
2140
- _DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
2141
- _corr_clean = _DIACRITICS_RE.sub('', corr_text)
2142
- _orig_clean = _DIACRITICS_RE.sub('', orig_text)
2143
- if _vm.is_iv(_orig_clean) and _vm.is_oov(_corr_clean):
2144
- logger.info(
2145
- f"[GRAMMAR] Rejected corruption: '{orig_text}'→'{corr_text}' "
2146
- f"(valid word → non-word)"
2147
- )
2148
- logger.info(f'[FILTER-TEL] {_tel_json.dumps({"event":"filter_reject","filter":"IVtoOOV","original":orig_text[:80],"correction":corr_text[:80]})}')
2149
- _tel_events.append({"event":"filter_reject","filter":"IVtoOOV","original":orig_text[:80],"correction":corr_text[:80]})
2150
- continue
2151
- # Also strip diacritics from correction for cleaner output
2152
- if _corr_clean != corr_text and _vm.is_iv(_corr_clean):
2153
- corr_text = _corr_clean
2154
- except Exception:
2155
- pass
2156
 
2157
  # FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
2158
  _TANWEEN_CHARS = set('ًٌٍ')
 
2127
  if _gram_dir_blocked:
2128
  continue
2129
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2130
 
2131
  # FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
2132
  _TANWEEN_CHARS = set('ًٌٍ')