Bayan AI commited on
Commit ·
dd8a53f
1
Parent(s): d8125c8
Refactor: Remove obsolete IVtoOOV filter to prevent False Negatives
Browse files- src/app.py +0 -26
src/app.py
CHANGED
|
@@ -2127,32 +2127,6 @@ def analyze_text():
|
|
| 2127 |
if _gram_dir_blocked:
|
| 2128 |
continue
|
| 2129 |
|
| 2130 |
-
if not _is_grammar_pattern:
|
| 2131 |
-
if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
|
| 2132 |
-
try:
|
| 2133 |
-
from nlp.spelling.araspell_service import get_spelling_model
|
| 2134 |
-
_vm = get_spelling_model().vocab_manager
|
| 2135 |
-
if _vm:
|
| 2136 |
-
# ── Phase 12 (B3): Strip diacritics before IV/OOV check ──
|
| 2137 |
-
# Grammar model sometimes outputs correct words with
|
| 2138 |
-
# diacritics (e.g. يفعلوَ) which fail OOV check.
|
| 2139 |
-
# Strip diacritics for vocabulary check only.
|
| 2140 |
-
_DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
|
| 2141 |
-
_corr_clean = _DIACRITICS_RE.sub('', corr_text)
|
| 2142 |
-
_orig_clean = _DIACRITICS_RE.sub('', orig_text)
|
| 2143 |
-
if _vm.is_iv(_orig_clean) and _vm.is_oov(_corr_clean):
|
| 2144 |
-
logger.info(
|
| 2145 |
-
f"[GRAMMAR] Rejected corruption: '{orig_text}'→'{corr_text}' "
|
| 2146 |
-
f"(valid word → non-word)"
|
| 2147 |
-
)
|
| 2148 |
-
logger.info(f'[FILTER-TEL] {_tel_json.dumps({"event":"filter_reject","filter":"IVtoOOV","original":orig_text[:80],"correction":corr_text[:80]})}')
|
| 2149 |
-
_tel_events.append({"event":"filter_reject","filter":"IVtoOOV","original":orig_text[:80],"correction":corr_text[:80]})
|
| 2150 |
-
continue
|
| 2151 |
-
# Also strip diacritics from correction for cleaner output
|
| 2152 |
-
if _corr_clean != corr_text and _vm.is_iv(_corr_clean):
|
| 2153 |
-
corr_text = _corr_clean
|
| 2154 |
-
except Exception:
|
| 2155 |
-
pass
|
| 2156 |
|
| 2157 |
# FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
|
| 2158 |
_TANWEEN_CHARS = set('ًٌٍ')
|
|
|
|
| 2127 |
if _gram_dir_blocked:
|
| 2128 |
continue
|
| 2129 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2130 |
|
| 2131 |
# FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
|
| 2132 |
_TANWEEN_CHARS = set('ًٌٍ')
|