| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import gradio as gr |
| import os, csv, re, base64, unicodedata, gzip |
| import torch |
| from transformers import AutoProcessor, VitsModel |
| import numpy as np |
| from html import escape |
|
|
| |
| os.environ['TRANSFORMERS_CACHE'] = os.environ.get('TRANSFORMERS_CACHE', '/tmp/cache') |
| os.environ['HF_HOME'] = os.environ.get('HF_HOME', '/tmp/hf') |
|
|
| DEBUG_MODE = False |
| def debug_print(msg): |
| if DEBUG_MODE: print(f"[DEBUG] {msg}") |
|
|
| |
| def _open_maybe_gzip(path): |
| if str(path).endswith(".gz"): |
| |
| return gzip.open(path, "rt", encoding="utf-8", newline="") |
| return open(path, "r", encoding="utf-8", newline="") |
|
|
| def norm(x): return (str(x).strip()) if x is not None else "" |
| def lower(x): return norm(x).lower() |
| def fold(s:str)->str: |
| return ''.join(c for c in unicodedata.normalize('NFD', s or "") if unicodedata.category(c)!="Mn") |
|
|
| |
| def _cand(*names): |
| for n in names: |
| if os.path.exists(n): return n |
| p = os.path.join("salida", n) |
| if os.path.exists(p): return p |
| return names[0] |
|
|
| |
| CSV_BI = _cand( |
| "LEXICON_v86_IBERIAN.csv.gz", |
| "LEXICON_v84_IBERIAN.csv.gz", |
| "LEXICON_v83_IBERIAN.csv.gz", |
| "LEXICON_v82_IBERIAN.csv.gz", |
| "LEXICON_v81_IBERIAN.csv.gz", |
| "LEXICON_v80_IBERIAN.csv.gz", |
| "LEXICON_v79_IBERIAN.csv.gz", |
| "LEXICON_v78_IBERIAN.csv.gz", |
| "LEXICON_v77_IBERIAN.csv.gz", |
| "LEXICON_v76_IBERIAN.csv.gz", |
| "LEXICON_v75_IBERIAN.csv.gz", |
| "LEXICON_v74_IBERIAN.csv.gz", |
| "LEXICON_v73_IBERIAN.csv.gz", |
| "LEXICON_v72_IBERIAN.csv.gz", |
| "LEXICON_v71_IBERIAN.csv.gz", |
| "LEXICON_v70_IBERIAN.csv.gz", |
| "LEXICON_v68_IBERIAN.csv.gz", |
| "LEXICON_v67_IBERIAN.csv.gz", |
| "LEXICON_v66_IBERIAN.csv.gz", |
| "LEXICON_v65_IBERIAN.csv.gz", |
| "LEXICON_v64_IBERIAN.csv.gz", |
| "LEXICON_v63_IBERIAN.csv.gz", |
| "LEXICON_v60_FINAL.csv.gz", |
| "LEXICON_v59_PATCHED.csv.gz", |
| "LEXICON_UNICO_1a1_v43_all_verbs.csv.gz", |
| "LEXICON_UNICO_1a1_v42_verbs_fix.csv.gz", |
| "LEXICON_UNICO_1a1_v41_family_fix.csv.gz", |
| "LEXICON_UNICO_1a1_v40_accent_fix.csv.gz", |
| "MASTER_SURFACE_READY.csv.gz", |
| "MASTER_REEXTENDED.csv.gz", |
| "BI_SURFACE_READY.csv.gz", |
| "HF_Pairs_BI_REEXTENDED.csv.gz", |
| "HF_Pairs_BI_EXPANDED1_EXTENDED_FILLED.csv.gz", |
| "HF_Pairs_BI_EXPANDED1.csv.gz" |
| ) |
|
|
| |
| |
| ES2NI = {} |
| NI2ES = {} |
| ES2NI_VERB = {} |
| ES2NI_POS = {} |
| ES2NI_MORPH = {} |
|
|
| |
| ESPHRASE2NI = {} |
| NIPHRASE2ES = {} |
| MAX_NGRAM = 3 |
|
|
| |
| ES_FOLD = {} |
| NI_FOLD = {} |
|
|
| |
| VISIBLE_PUNCT = set(list(",.;:!?¡¿…()[]{}\"'«»—–“”‘’")) |
| _num_re = re.compile(r"^\d+([.,]\d+)?$") |
| def is_number(tok:str)->bool: return bool(_num_re.fullmatch(tok or "")) |
|
|
| |
| _NI_UNITS = {0:'',1:'ban',2:'bi',3:'irur',4:'laur',5:'borste', |
| 6:'sei',7:'sisbi',8:'sorse',9:'bedar',10:'abar'} |
| _NI_TWENTIES = {1:'orkei',2:'binorkei',3:'irurokei',4:'laurokei'} |
|
|
| def digit_to_ni(tok:str)->str: |
| """Convierte un número entero (str de dígitos) a numeral neoíbero.""" |
| try: |
| n = int(tok) |
| except (ValueError, TypeError): |
| return tok |
| if n <= 0: return tok |
| if n <= 10: return _NI_UNITS[n] |
| if n <= 19: return f"abar-ke-{_NI_UNITS[n-10]}" |
| if n == 20: return "orkei" |
| if n < 100: |
| twenties = n // 20 |
| remainder = n % 20 |
| base = _NI_TWENTIES.get(twenties, tok) |
| if remainder == 0: return base |
| elif remainder == 10: return f"{base}-abar" |
| elif remainder > 10: return f"{base}-abar-ke-{_NI_UNITS[remainder-10]}" |
| else: return f"{base}-ke-{_NI_UNITS[remainder]}" |
| if n == 100: return "atun" |
| if n <= 999: |
| hundreds = n // 100 |
| remainder = n % 100 |
| h = "atun" if hundreds == 1 else f"{_NI_UNITS[hundreds]}-atun" |
| if remainder == 0: return h |
| r = digit_to_ni(str(remainder)) |
| return f"{h}-ke-{r}" |
| return tok |
|
|
| |
| CLAUSE_BREAKS = {",", ";", "—", "–", ":"} |
| PLACEHOLDER_RE = re.compile(r"^\[[^\]]+\]$") |
| def is_placeholder(tok: str) -> bool: |
| return bool(PLACEHOLDER_RE.match(tok or "")) |
|
|
| def _restore_brk(tok, protected): |
| m = re.fullmatch(r"__BRK(\d+)__(?:-(na|ba))?", tok or "") |
| if not m: return tok |
| idx = int(m.group(1)) |
| suf = m.group(2) |
| base = protected[idx] if 0 <= idx < len(protected) else tok |
| return base + (f"-{suf}" if suf else "") |
|
|
| def simple_tokenize(text:str): |
| """Tokenización mínima, sin romper [ ... ] ni [ ... ]-na/-ba.""" |
| if not text: |
| return [] |
| protected = [] |
| def _repl(m): |
| key = f"__BRK{len(protected)}__" |
| protected.append(m.group(0)) |
| return key |
| t = re.sub(r"\[[^\]]*\]", _repl, (text or "").strip()) |
| t = re.sub(r"\s+"," ", t) |
| t = re.sub(r"([,.;:!?¡¿…()\[\]{}\"'«»—–“”‘’])", r" \1 ", t) |
| toks = [tok for tok in t.split() if tok] |
| for i, tok in enumerate(toks): |
| if tok.startswith("__BRK") and "__" in tok: |
| toks[i] = _restore_brk(tok, protected) |
| return toks |
|
|
| |
| |
| _ENCLITICS = ('los','las','les','nos','me','te','lo','la','le','se','os') |
| _ACCENTED_VOWELS = str.maketrans('áéíóú', 'aeiou') |
|
|
| def _strip_accents(s): |
| return s.translate(_ACCENTED_VOWELS) |
|
|
| def expand_enclitics(toks): |
| """ |
| Separa formas verbo+pronombre enclítico en dos tokens cuando ES2NI |
| no contiene la forma combinada pero sí el verbo base. |
| |
| Ejemplos: |
| 'ayudarme' → ['ayudar', 'me'] |
| 'tocarlo' → ['tocar', 'lo'] |
| 'ayúdame' → ['ayuda', 'me'] |
| 'cuídate' → ['cuida', 'te'] (cuida es ADJ pero cuidar es V) |
| 'recordadla'→ ['recordad', 'la'] |
| 'ayudándome'→ ['ayudando', 'me'] |
| 'dárselo' → ['darse', 'lo'] |
| |
| Reglas estrictas para evitar falsos positivos: |
| 1. Si la palabra completa YA está en ES2NI, no se toca (ej. 'interesante', |
| 'gigante', 'adelante', 'mediante' — todos terminan en formas que |
| parecen clíticos pero son entradas legítimas del léxico). |
| 2. La raíz tras quitar el clítico debe estar en ES2NI como verbo (V), |
| O bien su infinitivo correspondiente debe estar como V en el lex. |
| Esto último es necesario porque el lex registra muchas formas como |
| ADJ/N (cuida=ADJ "cuidadoso", cuenta=ADJ "cuento") aunque también |
| son IMP-2S informales válidos en español. |
| 3. Si la raíz tiene tilde (caso típico de imperativos: 'ayúda+me'), |
| se prueba también la versión sin tilde. |
| """ |
| if not toks: |
| return toks |
| if not ES2NI or not ES2NI_POS: |
| return toks |
|
|
| def _stem_is_verb_or_has_infinitive(stem): |
| """¿Es esta raíz un verbo conocido, o tiene un infinitivo en el lex? |
| |
| Considera también la des-diptongación común del español: |
| cuenta→contar, vuelve→volver, duerme→dormir, siente→sentir, etc. |
| """ |
| if stem not in ES2NI: |
| |
| |
| pass |
| elif ES2NI_POS.get(stem, "") == "V": |
| return True |
| |
| for suf in ("r", "er", "ir"): |
| inf_candidate = stem + suf |
| if inf_candidate in ES2NI and ES2NI_POS.get(inf_candidate, "") == "V": |
| return True |
| |
| |
| |
| |
| |
| |
| for diph, base in (("ue", "o"), ("ie", "e")): |
| idx = stem.rfind(diph) |
| if idx < 0: |
| continue |
| stem_undiph = stem[:idx] + base + stem[idx+2:] |
| |
| stem_root = stem_undiph |
| if stem_root and stem_root[-1] in "ae": |
| stem_root = stem_root[:-1] |
| for suf in ("ar", "er", "ir"): |
| inf_candidate = stem_root + suf |
| if inf_candidate in ES2NI and ES2NI_POS.get(inf_candidate, "") == "V": |
| return True |
| return False |
|
|
| out = [] |
| for tok in toks: |
| tok_l = tok.lower() |
| if not tok_l.isalpha() or tok_l in ES2NI: |
| out.append(tok) |
| continue |
| tok_noacc = _strip_accents(tok_l) |
| if tok_noacc != tok_l and tok_noacc in ES2NI: |
| out.append(tok) |
| continue |
|
|
| split = None |
| for clit in _ENCLITICS: |
| if not tok_l.endswith(clit): |
| continue |
| stem = tok_l[:-len(clit)] |
| |
| |
| |
| if len(stem) < 2: |
| continue |
| if len(stem) == 2 and stem not in ES2NI: |
| continue |
| |
| if _stem_is_verb_or_has_infinitive(stem): |
| split = (stem, clit, None) |
| break |
| |
| stem_noacc = _strip_accents(stem) |
| if stem_noacc != stem and _stem_is_verb_or_has_infinitive(stem_noacc): |
| split = (stem_noacc, clit, None) |
| break |
| |
| |
| for clit2 in _ENCLITICS: |
| if not stem.endswith(clit2): |
| continue |
| stem2 = stem[:-len(clit2)] |
| if len(stem2) < 2: |
| continue |
| if len(stem2) == 2 and stem2 not in ES2NI: |
| continue |
| if _stem_is_verb_or_has_infinitive(stem2): |
| split = (stem2, clit2, clit) |
| break |
| stem2_noacc = _strip_accents(stem2) |
| if stem2_noacc != stem2 and _stem_is_verb_or_has_infinitive(stem2_noacc): |
| split = (stem2_noacc, clit2, clit) |
| break |
| if split: |
| break |
|
|
| if split: |
| stem, clit1, clit2 = split |
| if tok[0].isupper(): |
| stem = stem[0].upper() + stem[1:] |
| out.append(stem) |
| out.append(clit1) |
| if clit2: |
| out.append(clit2) |
| else: |
| out.append(tok) |
| return out |
|
|
| def detokenize(tokens): |
| s = " ".join(tokens) |
| s = re.sub(r"\s+([,.;:!?])", r"\1", s) |
| s = re.sub(r"([¿¡])\s+", r"\1", s) |
| s = re.sub(r"\(\s+", "(", s) |
| s = re.sub(r"\s+\)", ")", s) |
| s = re.sub(r"\s{2,}", " ", s).strip() |
| return s |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| _SAFE_CLITICS_RE = r"(me|te|nos|os|se)" |
| _AMBIG_CLITICS_RE = r"(lo|la|le|los|las|les)" |
| |
| |
| _SAFE_AFTER_AMBIG = ( |
| r"(?:\s*[.,;:!?)\]»\"”]" |
| r"|\s+(?:y|o|pero|sino|aunque|mientras|porque|si|cuando|donde|que|" |
| r"también|tampoco|ya|no|todavía|después|antes|ahora|luego|aquí|allí|" |
| r"ahí|así|sólo|solo|nunca|jamás|siempre|" |
| r"bien|mal|mucho|poco|muy|más|menos|tan|todo|todos|nada|algo|" |
| r"hoy|ayer|mañana|pronto|tarde|" |
| r"fuerte|fuertemente|suavemente|fijamente|atentamente)\b" |
| |
| |
| r"|\s+otra\s+vez\b" |
| r"|\s+otras\s+veces\b" |
| r"|\s+(?:cada|todos\s+los|todas\s+las)\s+(?:día|días|mañana|mañanas|tarde|tardes|noche|noches|mes|meses|año|años|semana|semanas|hora|horas|momento|momentos|vez|veces)\b" |
| r"|\s*$|\s*\n)" |
| ) |
| _FUSE_INF_SAFE_RE = re.compile( |
| r"\b([a-záéíóúñü]+(?:ar|er|ir))\s+" + _SAFE_CLITICS_RE + r"\b", |
| re.IGNORECASE) |
| _FUSE_INF_AMBIG_RE = re.compile( |
| r"\b([a-záéíóúñü]+(?:ar|er|ir))\s+" + _AMBIG_CLITICS_RE + r"(?=" + _SAFE_AFTER_AMBIG + r")", |
| re.IGNORECASE) |
| _FUSE_GER_SAFE_RE = re.compile( |
| r"\b([a-záéíóúñü]+(?:ando|iendo|yendo))\s+" + _SAFE_CLITICS_RE + r"\b", |
| re.IGNORECASE) |
| _FUSE_GER_AMBIG_RE = re.compile( |
| r"\b([a-záéíóúñü]+(?:ando|iendo|yendo))\s+" + _AMBIG_CLITICS_RE + r"(?=" + _SAFE_AFTER_AMBIG + r")", |
| re.IGNORECASE) |
| |
| |
| _FUSE_VERB_SAFE_RE = re.compile( |
| r"\b([a-záéíóúñü]+)\s+" + _SAFE_CLITICS_RE + r"\b", |
| re.IGNORECASE) |
| _FUSE_VERB_AMBIG_RE = re.compile( |
| r"\b([a-záéíóúñü]+)\s+" + _AMBIG_CLITICS_RE + r"(?=" + _SAFE_AFTER_AMBIG + r")", |
| re.IGNORECASE) |
| _GER_ACCENT_MAP = (("ando","ándo"), ("iendo","iéndo"), ("yendo","yéndo")) |
| |
| |
| |
| |
| _MORPH_ADMITS_ENCLITIC = {"PRS", "IMP"} |
|
|
| def _verb_admits_enclitic(verb): |
| """¿El verbo tiene morfología compatible con enclítico (PRS o IMP)?""" |
| v = (verb or "").lower() |
| |
| |
| if v in {"me","te","se","nos","os","lo","la","le","los","las","les"}: |
| return False |
| |
| |
| |
| if v in {"como", "mientras", "para", "sobre", "luego", "casi", "según", |
| "salvo", "bajo", "sin", "pasada", "vista", "puesto", "dada", |
| "siendo", "habiendo"}: |
| return False |
| if v not in ES2NI: |
| return False |
| if ES2NI_POS.get(v, "") != "V": |
| return False |
| morph = ES2NI_MORPH.get(v, "") |
| return morph in _MORPH_ADMITS_ENCLITIC |
|
|
| def _fuse_imp_with_accent(verb, clit): |
| """Fusiona verbo + clítico, añadiendo tilde si la fusión hace que la |
| tónica del verbo caiga en posición proparoxítona (3+ sílabas con |
| tónica en antepenúltima).""" |
| VOWELS = "aeiouáéíóú" |
| fused = verb + clit |
| positions = [i for i,c in enumerate(fused) if c.lower() in VOWELS] |
| if len(positions) < 3: |
| return fused |
| verb_positions = [i for i,c in enumerate(verb) if c.lower() in VOWELS] |
| if not verb_positions: |
| return fused |
| |
| if len(verb_positions) == 1: |
| tonic_idx = verb_positions[0] |
| else: |
| tonic_idx = verb_positions[-2] |
| if fused[tonic_idx] in 'áéíóú': |
| return fused |
| vowels_after = sum(1 for p in positions if p > tonic_idx) |
| if vowels_after >= 2: |
| accent_map = {'a':'á','e':'é','i':'í','o':'ó','u':'ú'} |
| ch = fused[tonic_idx].lower() |
| if ch in accent_map: |
| new_ch = accent_map[ch] |
| if fused[tonic_idx].isupper(): |
| new_ch = new_ch.upper() |
| return fused[:tonic_idx] + new_ch + fused[tonic_idx+1:] |
| return fused |
|
|
| def _is_real_infinitive(word): |
| """¿Es 'word' realmente un infinitivo en el lex (no un sustantivo |
| como 'mar', 'sur', 'par')?""" |
| w = (word or "").lower() |
| if w not in ES2NI: |
| return False |
| pos = ES2NI_POS.get(w, "") |
| return pos == "V" |
|
|
| def fuse_enclitics_es(es_text): |
| """Rejunta verbo + clítico siguiendo dos reglas: |
| - Clíticos no ambiguos (me, te, nos, os, se): se fusionan siempre tras |
| INF o GER del lex. |
| - Clíticos ambiguos (lo, la, le, los, las, les): se fusionan tras INF |
| o GER solo si el contexto siguiente garantiza que el clítico no es |
| artículo (puntuación, conjunción, adverbio breve). |
| |
| NOTA: NO fusionamos formas flexionadas (PRS, IMP, FUT, etc.) + clítico |
| porque sin acceso a la morfología precisa no podemos distinguir entre |
| "escucha + me" (= escúchame, debe fusionar) y "enseñaré + lo" |
| (= "le enseñaré lo que..." = no debe fusionar). Aceptamos como |
| limitación que los imperativos + clítico ambiguo aparezcan separados. |
| """ |
| if not es_text: |
| return es_text |
|
|
| def _inf_safe(m): |
| verb, clit = m.group(1), m.group(2) |
| if _is_real_infinitive(verb): |
| return verb + clit |
| return m.group(0) |
| def _inf_ambig(m): |
| verb, clit = m.group(1), m.group(2) |
| if _is_real_infinitive(verb): |
| return verb + clit |
| return m.group(0) |
|
|
| def _ger_accent(verb, clit): |
| for plain, accented in _GER_ACCENT_MAP: |
| if verb.lower().endswith(plain): |
| return verb[:-len(plain)] + accented + clit |
| return verb + " " + clit |
| def _ger_safe(m): |
| verb, clit = m.group(1), m.group(2) |
| if not _is_real_infinitive(verb): |
| return m.group(0) |
| return _ger_accent(verb, clit) |
| def _ger_ambig(m): |
| verb, clit = m.group(1), m.group(2) |
| if not _is_real_infinitive(verb): |
| return m.group(0) |
| return _ger_accent(verb, clit) |
|
|
| es_text = _FUSE_INF_SAFE_RE.sub(_inf_safe, es_text) |
| es_text = _FUSE_INF_AMBIG_RE.sub(_inf_ambig, es_text) |
| es_text = _FUSE_GER_SAFE_RE.sub(_ger_safe, es_text) |
| es_text = _FUSE_GER_AMBIG_RE.sub(_ger_ambig, es_text) |
|
|
| |
| |
| |
| |
| def _verb_safe(m): |
| verb, clit = m.group(1), m.group(2) |
| v = verb.lower() |
| if v.endswith(("ar","er","ir","ando","iendo","yendo")): |
| return m.group(0) |
| if not _verb_admits_enclitic(v): |
| return m.group(0) |
| return _fuse_imp_with_accent(verb, clit) |
| def _verb_ambig(m): |
| verb, clit = m.group(1), m.group(2) |
| v = verb.lower() |
| if v.endswith(("ar","er","ir","ando","iendo","yendo")): |
| return m.group(0) |
| if not _verb_admits_enclitic(v): |
| return m.group(0) |
| return _fuse_imp_with_accent(verb, clit) |
|
|
| es_text = _FUSE_VERB_SAFE_RE.sub(_verb_safe, es_text) |
| es_text = _FUSE_VERB_AMBIG_RE.sub(_verb_ambig, es_text) |
|
|
| |
| |
| |
| |
| |
| _DOUBLE_CLIT_RE = re.compile( |
| r"\b([a-záéíóúñü]+(?:me|te|se|nos|os))\s+" + _AMBIG_CLITICS_RE + |
| r"(?=" + _SAFE_AFTER_AMBIG + r")", |
| re.IGNORECASE) |
| def _double_clit(m): |
| word, clit2 = m.group(1), m.group(2) |
| |
| |
| |
| clit1_endings = ("me","te","se","nos","os") |
| verb_orig = None |
| for end in clit1_endings: |
| if word.lower().endswith(end): |
| cand = word[:-len(end)] |
| |
| cand_noacc = (cand.replace('á','a').replace('é','e') |
| .replace('í','i').replace('ó','o').replace('ú','u')) |
| if (cand.lower() in ES2NI or cand_noacc.lower() in ES2NI): |
| verb_orig = cand_noacc |
| break |
| if not verb_orig: |
| return m.group(0) |
| |
| if not _verb_admits_enclitic(verb_orig): |
| return m.group(0) |
| |
| return _fuse_imp_with_accent(word, clit2) |
|
|
| es_text = _DOUBLE_CLIT_RE.sub(_double_clit, es_text) |
| return es_text |
|
|
| |
| MODAL_SUFFIX_ENABLE = True |
| MODAL_ONLY_ON_FINITE = True |
| MODAL_STRIP_QE_IN_NI = True |
|
|
| SENT_END = {".", "!", "?", "…"} |
| OPEN_FOR = {"?": "¿", "!": "¡"} |
| WRAP_PREFIX = set(list("«“‘([{\"'")) |
| PERS_ENDINGS = ("-n","-śe","-ek","-śek","-k") |
| TAM_FINITE = ("-ke","-bo","-ta","-ni","-ir") |
|
|
| def looks_like_finite_ni(tok:str)->bool: |
| t = (tok or "").lower() |
| if not t or t.startswith("["): return False |
| base = re.sub(r"-(na|ba)$","", t) |
| for tam in TAM_FINITE: |
| if base.endswith(tam) or any(base.endswith(tam+pe) for pe in PERS_ENDINGS): |
| return True |
| return False |
|
|
| def last_content_index(tokens, start, end_exclusive): |
| i = end_exclusive - 1 |
| while i >= start and tokens[i] in VISIBLE_PUNCT: |
| i -= 1 |
| return i if i >= start else -1 |
|
|
| def strip_qe_punct(tokens): |
| return [t for t in tokens if t not in ("¿","?","¡","!")] |
|
|
| |
| def _is_numeric_comma(tokens, i): |
| return (0 < i < len(tokens)-1 and tokens[i] == "," and |
| is_number(tokens[i-1]) and is_number(tokens[i+1])) |
|
|
| def _is_time_colon(tokens, i): |
| return (0 < i < len(tokens)-1 and tokens[i] == ":" and |
| is_number(tokens[i-1]) and is_number(tokens[i+1])) |
|
|
| def _is_true_clause_break(tokens, i): |
| if tokens[i] not in CLAUSE_BREAKS: return False |
| if _is_numeric_comma(tokens, i): return False |
| if _is_time_colon(tokens, i): return False |
| return True |
|
|
| def add_modal_suffixes_es2ni(tokens): |
| """Añade -na/-ba al último verbo finito (o último constituyente) por oración.""" |
| if not MODAL_SUFFIX_ENABLE: |
| return tokens |
| out = tokens[:] |
| n = len(out) |
| i = 0 |
| sent_start = 0 |
| while i < n: |
| if out[i] in ("?", "!"): |
| closer = out[i] |
| |
| |
| |
| target = last_content_index(out, sent_start, i) |
| if target != -1: |
| suf = "na" if closer == "?" else "ba" |
| if not re.search(rf"-(?:{suf})$", out[target].lower()): |
| out[target] = out[target] + "-" + suf |
| sent_start = i + 1 |
| elif out[i] in SENT_END: |
| sent_start = i + 1 |
| i += 1 |
| if MODAL_STRIP_QE_IN_NI: |
| out = strip_qe_punct(out) |
| return out |
|
|
| def strip_modal_suffixes_ni(tokens): |
| """ |
| Interpreta -na/-ba como modalidad de pregunta/exclamación; cerramos solo |
| al final de oración (no en comas/“:”, salvo que ya haya ?/! explícitos). |
| |
| Excepción importante: si el token completo (con -na o -ba) es una entrada |
| válida en NI2ES, NO se pela el sufijo. Esto protege formas verbales del |
| léxico que terminan en -na/-ba (p. ej. aŕen-na = "sea", forma SBJ 3S del |
| verbo "ser") frente a falsa lectura como pregunta "¿aŕen?" = "¿ser?". |
| """ |
| if not MODAL_SUFFIX_ENABLE: |
| return tokens |
|
|
| out = [] |
| buf = [] |
| pending_end = None |
| mode = None |
|
|
| def _emit(end_override=None, also_append=None): |
| nonlocal buf, mode, pending_end, out |
| local = [t for t in buf if t not in ("¿","?","¡","!")] |
| if local: |
| end_tok = end_override or ("?" if mode == "?" else "!" if mode == "!" else pending_end or ".") |
| out.extend(local) |
| out.append(end_tok) |
| buf.clear(); mode = None; pending_end = None |
| if also_append: |
| out.append(also_append) |
|
|
| toks = tokens + ["."] |
| for i, t in enumerate(toks): |
| if t in ("¿", "¡"): |
| _emit(); mode = "?" if t == "¿" else "!" |
| continue |
| if t in ("?", "!"): |
| pending_end = t; _emit(); continue |
| if t in SENT_END: |
| pending_end = t; _emit(); continue |
|
|
| |
| if t in CLAUSE_BREAKS and mode in ("?","!"): |
| buf.append(t) |
| continue |
|
|
| m = re.search(r"-(na|ba)$", (t or "").lower()) |
| if m: |
| |
| |
| |
| if (t or "").lower() in NI2ES: |
| buf.append(t) |
| continue |
| t = t[:-len(m.group(0))] |
| if t: buf.append(t) |
| mode = "?" if m.group(1) == "na" else "!" |
| _emit() |
| continue |
|
|
| if t: |
| buf.append(t) |
|
|
| if len(out) >= 2 and out[-1] == "." and out[-2] == ".": out.pop() |
| return out |
|
|
| def add_inverted_openers(tokens): |
| """Inserta ¿/¡ al inicio de cada tramo que acaba en ?/!, ignorando comas/“:” numéricos. |
| |
| CASO ESPECIAL: cuando una sola exclamación/interrogación abarca varias |
| cláusulas separadas por comas (ej. "¡Cuántas cosas he visto, cuántas cosas |
| he aprendido, cuántas personas he amado!"), todas las cláusulas son parte |
| del mismo tramo exclamativo. La apertura va al INICIO ABSOLUTO, no después |
| de la última coma. Esto se detecta porque hay palabras exclamativas |
| (qué, cuán, cuánto, cuántas, ...) o interrogativas (qué, quién, cuándo, |
| dónde, cómo, cuál) en cláusulas anteriores dentro del mismo tramo. |
| """ |
| out = tokens[:] |
| START_BREAKS = SENT_END | CLAUSE_BREAKS |
|
|
| EXCL_WORDS = {'qué','que','cuán','cuan','cuánto','cuanto','cuánta','cuanta', |
| 'cuántos','cuantos','cuántas','cuantas','cómo','como'} |
| INTERR_WORDS = {'qué','que','quién','quien','quiénes','quienes','cuándo','cuando', |
| 'dónde','donde','cómo','como','cuál','cual','cuáles','cuales', |
| 'por','cuán','cuan','cuánto','cuanto','cuánta','cuanta', |
| 'cuántos','cuantos','cuántas','cuantas'} |
|
|
| def _is_true_start_break(idx): |
| if out[idx] in SENT_END: return True |
| if out[idx] in CLAUSE_BREAKS: return _is_true_clause_break(out, idx) |
| return False |
|
|
| def _has_qword_before_clause_break(start_idx, end_idx, word_set): |
| """¿Hay alguna palabra del set entre start_idx (excluyente) y end_idx?""" |
| for k in range(start_idx+1, end_idx): |
| if out[k].lower() in word_set: |
| return True |
| return False |
|
|
| i = 0 |
| while i < len(out): |
| if out[i] in ("?", "!"): |
| closer = out[i]; opener = OPEN_FOR[closer] |
| word_set = EXCL_WORDS if closer == "!" else INTERR_WORDS |
|
|
| |
| |
| |
| j = i - 1 |
| while j >= 0 and not _is_true_start_break(j): |
| j -= 1 |
| |
| |
| while j >= 0 and out[j] in CLAUSE_BREAKS: |
| |
| k_prev = j - 1 |
| while k_prev >= 0 and not _is_true_start_break(k_prev): |
| k_prev -= 1 |
| |
| if _has_qword_before_clause_break(k_prev, j, word_set): |
| j = k_prev |
| else: |
| break |
|
|
| start = j + 1 |
| k = start |
| while k < i and out[k] in WRAP_PREFIX: |
| k += 1 |
| if not (k < len(out) and out[k] == opener): |
| out.insert(k, opener); i += 1 |
| i += 1 |
| return out |
|
|
| |
| EXPANSION_ENABLE = True |
| FLAG_COLNAMES = ("flags","FLAGS","expand","EXPAND","tags","TAGS","morph","MORPH") |
| FLAG_PLURAL = ("S",) |
| FLAG_3PL = ("3","V3") |
|
|
| VOWELS = "aeiouáéíóúüAEIOUÁÉÍÓÚÜ" |
|
|
| def _has_flag(cell:str, wanted:tuple)->bool: |
| c = (cell or "") |
| return any(w in c for w in wanted) |
|
|
| def _pluralize_es_form(s: str) -> str: |
| if not s: return s |
| sl = s.lower() |
| if sl.endswith("z"): |
| return s[:-1] + ("ces" if s[-1].islower() else "CES") |
| if s[-1] not in VOWELS: |
| return s + ("es" if s[-1].islower() else "ES") |
| return s + ("s" if s[-1].islower() else "S") |
|
|
| def _present_3pl_from_3sg(s: str) -> str: |
| if not s: return s |
| return s + ("n" if s[-1].islower() else "N") |
|
|
| |
| print("Cargando modelo de voz (opcional)…") |
| device = "cuda" if torch.cuda.is_available() else "cpu" |
| processor = model = None |
| try: |
| processor = AutoProcessor.from_pretrained("facebook/mms-tts-spa") |
| model = VitsModel.from_pretrained("facebook/mms-tts-spa").to(device) |
| print("Modelo de voz cargado.") |
| except Exception as e: |
| print(f"AVISO TTS: {e}") |
|
|
| def add_reading_pauses(text: str, level:int=3) -> str: |
| if level <= 1: return text |
| t = re.sub(r",\s*", ", , ", text) |
| t = re.sub(r"\.\s*", ". . ", text) |
| return re.sub(r'\s+',' ',t).strip() |
|
|
| def hispanize_for_tts(ni_text: str) -> str: |
| text=unicodedata.normalize('NFC', (ni_text or "").lower()) |
| text=text.replace('ŕ','rr').replace('ś','s').replace('eś','es').replace('-', ' ') |
| text=re.sub(r'\[.*?\]','',text); text=re.sub(r'\s+',' ',text).strip() |
| return add_reading_pauses(text, 3) |
|
|
| def synthesize_speech(text): |
| if not text or not text.strip() or model is None or processor is None: return None |
| try: |
| inputs = processor(text=hispanize_for_tts(text), return_tensors="pt").to(device) |
| with torch.no_grad(): output = model(**inputs).waveform |
| speech_np = output.cpu().numpy().squeeze() |
| mx = max(abs(speech_np.min()), abs(speech_np.max())) |
| if mx>0: speech_np = speech_np/mx*0.9 |
| return (16000, speech_np.astype(np.float32)) |
| except Exception as e: |
| print(f"Error TTS: {e}"); return None |
|
|
| |
| V = "aeiou" |
| SYL_FOR = { |
| "b":["‹BA›","‹BE›","‹BI›","‹BO›","‹BU›"], |
| "d":["‹DA›","‹DE›","‹DI›","‹DO›","‹DU›"], |
| "t":["‹TA›","‹TE›","‹TI›","‹TO›","‹TU›"], |
| "g":["‹GA›","‹GE›","‹GI›","‹GO›","‹GU›"], |
| "k":["‹KA›","‹KE›","‹KI›","‹KO›","‹KU›"] |
| } |
| ALPHA_FOR={"a":"‹A›","e":"‹E›","i":"‹I›","o":"‹O›","u":"‹U›","s":"‹S›","ś":"‹Ś›", |
| "l":"‹L›","r":"‹R›","ŕ":"‹Ŕ›","n":"‹N›","m":"‹M›"} |
| CODA_FOR={"":"","n":"‹N›","s":"‹S›","ś":"‹Ś›","r":"‹R›","ŕ":"‹Ŕ›","l":"‹L›","m":"‹M›","k":"‹K›","t":"‹T›"} |
|
|
| def tokens_from_latin(ni:str)->str: |
| out=[]; i=0; ni=unicodedata.normalize('NFC', (ni or "").lower()) |
| while i<len(ni): |
| c=ni[i] |
| if c=="p": c="b" |
| if c=="-": out.append("—"); i+=1; continue |
| if c in V: |
| out.append(ALPHA_FOR.get(c, c.upper())); i+=1; continue |
| if c in SYL_FOR and i+1<len(ni) and ni[i+1] in V: |
| idx=V.index(ni[i+1]); tok=SYL_FOR[c][idx] |
| coda=ni[i+2] if i+2<len(ni) else "" |
| if coda in CODA_FOR and coda!="": tok+=CODA_FOR[coda]; i+=3 |
| else: i+=2 |
| out.append(tok); continue |
| out.append(ALPHA_FOR.get(c, c.upper())); i+=1 |
| return "".join(out) |
|
|
| KEYS_MODE = "full" |
| KEYS_OVERRIDE = {} |
|
|
| def georgeos_keys(token_str:str, ni_plain:str)->str: |
| low=unicodedata.normalize('NFC', (ni_plain or "").lower()) |
| if low in KEYS_OVERRIDE: return KEYS_OVERRIDE[low] |
| m=re.findall(r"‹(.*?)›", token_str) |
| out=[] |
| for t in m: |
| if KEYS_MODE == "compact": |
| if len(t)==2 and t[0] in "BDTGK": out.append(t[0]) |
| elif t in ("A","E","I","O","U"): out.append(t) |
| elif t=="Ś": out.append("X") |
| elif t=="Ŕ": out.append("r") |
| else: out.append(t[0].upper()) |
| else: |
| if len(t)==2 and t[0] in "BDTGK": out.append(t) |
| elif t=="Ś": out.append("X") |
| elif t=="Ŕ": out.append("r") |
| else: out.append(t) |
| return "".join(out) |
|
|
| TRIDOT = "|" |
| def render_ib_with_tridots(ib_toks): |
| res=[]; prev_word=False |
| for tk in ib_toks: |
| is_punct = tk in VISIBLE_PUNCT |
| if is_punct: |
| res.append(" "+tk+" "); prev_word=False |
| else: |
| if prev_word: res.append(" "+TRIDOT+" ") |
| res.append(tk); prev_word=True |
| return "".join(res).strip() |
|
|
| |
|
|
| |
| STRICT_BI_ENFORCE = True |
| AMBIG_NI = {} |
| BI_DIAG_HTML = "<em>Sin CSV cargado.</em>" |
|
|
| def load_bi_strict_and_diagnose(): |
| """Carga el CSV, llena ES2NI/NI2ES y prepara un HTML de diagnóstico.""" |
| global BI_DIAG_HTML |
| |
| ES2NI.clear(); NI2ES.clear(); ESPHRASE2NI.clear(); NIPHRASE2ES.clear() |
| AMBIG_NI.clear(); ES2NI_VERB.clear() |
| ES2NI_POS.clear() |
| ES2NI_MORPH.clear() |
| NI2ES_LEMMA = {} |
|
|
| if not os.path.exists(CSV_BI): |
| msg=f"[ERROR] No se encontró el CSV bilingüe: {CSV_BI}" |
| print(msg); BI_DIAG_HTML=f"<b>Error:</b> {escape(msg)}" |
| return False |
|
|
| rows=0; dup_es=0; dup_ni=0; empty_pid=0 |
| mismatch_backmap = 0 |
| mismatch_samples = [] |
| pid_seen=set() |
|
|
| print(f"Detectado CSV bilingüe: {CSV_BI}") |
| try: |
| with _open_maybe_gzip(CSV_BI) as f: |
| rd = csv.DictReader(f) |
| flds=set(rd.fieldnames or []) |
| ES_COL = "source_es" if "source_es" in flds else "es_surface" if "es_surface" in flds else "es" |
| NI_COL = "target_ni" if "target_ni" in flds else "ni_surface" if "ni_surface" in flds else "ni" |
| IDCOL = "pair_id" if "pair_id" in flds else "id" if "id" in flds else None |
| FLAGCOL = None |
| for cand in FLAG_COLNAMES: |
| if cand in flds: |
| FLAGCOL = cand; break |
| POS_COL = "pos_es" if "pos_es" in flds else "pos" if "pos" in flds else None |
| LEMMA_COL = "es_lemma" if "es_lemma" in flds else "lemma" if "lemma" in flds else None |
| MORPH_COL = "es_morph" if "es_morph" in flds else "morph" if "morph" in flds else None |
|
|
| base_rows = [] |
| for r in rd: |
| es_orig = (r.get(ES_COL) or "").strip() |
| ni_orig = (r.get(NI_COL) or "").strip() |
| if not (es_orig and ni_orig): continue |
| pid = (r.get(IDCOL) or "").strip() if IDCOL else "" |
| if not pid: empty_pid += 1 |
| else: pid_seen.add(pid) |
| flags = (r.get(FLAGCOL) or "") if FLAGCOL else "" |
|
|
| es = lower(es_orig) |
| ni = lower(ni_orig) |
|
|
| |
| if " " in es: |
| if es not in ESPHRASE2NI: |
| ESPHRASE2NI[es] = (ni_orig, pid) |
| if " " in ni: |
| if ni not in NIPHRASE2ES: |
| NIPHRASE2ES[ni] = (es_orig, pid) |
|
|
| |
| pos = (r.get(POS_COL) or "").strip() if POS_COL else "" |
| morph = (r.get(MORPH_COL) or "").strip() if MORPH_COL else "" |
| _MORPH_PRIO = {"PRS":10,"PST":9,"IPFV":8,"FUT":7,"COND":6, |
| "INF":5,"GER":4,"PART":3,"SBJ":2,"SBJ_IPFV":1,"IMP":0} |
| _POS_PRIO = {"ADJ":3, "N":2, "V":1} |
| if es in ES2NI: |
| dup_es += 1 |
| old_pos = ES2NI_POS.get(es, "") |
| old_morph = ES2NI_MORPH.get(es, "") |
| replace = False |
| new_p = _POS_PRIO.get(pos, 0) |
| old_p = _POS_PRIO.get(old_pos, 0) |
| if new_p > old_p: |
| |
| if old_pos == "V": |
| ES2NI_VERB[es] = ES2NI[es] |
| replace = True |
| elif pos == "V" and old_pos == "V": |
| new_m = _MORPH_PRIO.get(morph, -1) |
| old_m = _MORPH_PRIO.get(old_morph, -1) |
| if new_m > old_m: |
| ES2NI_VERB[es] = ES2NI[es] |
| replace = True |
| elif pos == "V" and old_pos in ("N", "ADJ"): |
| ES2NI_VERB[es] = (ni_orig, pid) |
| if replace: |
| ES2NI[es] = (ni_orig, pid) |
| ES2NI_POS[es] = pos |
| ES2NI_MORPH[es] = morph |
| else: |
| ES2NI[es] = (ni_orig, pid) |
| ES2NI_POS[es] = pos |
| ES2NI_MORPH[es] = morph |
|
|
| |
| lemma = (r.get(LEMMA_COL) or "").strip().lower() if LEMMA_COL else "" |
| if ni in NI2ES: |
| dup_ni += 1 |
| old_lemma = NI2ES_LEMMA.get(ni, "") |
| |
| |
| if lemma and old_lemma and lemma == old_lemma: |
| NI2ES[ni] = (es_orig, pid) |
| else: |
| |
| s = AMBIG_NI.get(ni, set()) |
| s.add(NI2ES[ni][0]); s.add(es_orig) |
| AMBIG_NI[ni] = s |
| if STRICT_BI_ENFORCE: |
| NI2ES.pop(ni, None) |
| else: |
| if STRICT_BI_ENFORCE and ni in AMBIG_NI: |
| pass |
| else: |
| NI2ES[ni] = (es_orig, pid) |
| NI2ES_LEMMA[ni] = lemma |
|
|
| base_rows.append((es_orig, ni_orig, pid, flags)) |
| rows += 1 |
|
|
| |
| if EXPANSION_ENABLE: |
| for es_orig, ni_orig, pid, flags in base_rows: |
| if not flags: continue |
| if _has_flag(flags, FLAG_PLURAL): |
| pl = _pluralize_es_form(es_orig) |
| pl_key = lower(pl) |
| if pl_key not in ES2NI: |
| ES2NI[pl_key] = (ni_orig, pid) |
| if _has_flag(flags, FLAG_3PL): |
| p3 = _present_3pl_from_3sg(es_orig) |
| p3_key = lower(p3) |
| if p3_key not in ES2NI: |
| ES2NI[p3_key] = (ni_orig, pid) |
|
|
| |
| for es_low, (ni_surf, _) in ES2NI.items(): |
| ni_low = lower(ni_surf) |
| back = NI2ES.get(ni_low) |
| if back and lower(back[0]) != es_low: |
| mismatch_backmap += 1 |
| if len(mismatch_samples) < 10: |
| mismatch_samples.append((es_low, ni_low, lower(back[0]))) |
|
|
| except Exception as e: |
| msg=f"[ERROR] Al leer {CSV_BI}: {e}" |
| print(msg); BI_DIAG_HTML=f"<b>Error:</b> {escape(msg)}" |
| return False |
|
|
| |
| ES_FOLD.clear(); NI_FOLD.clear() |
| for es_key in ES2NI: |
| fk = fold(es_key) |
| if fk != es_key and fk not in ES_FOLD: |
| ES_FOLD[fk] = es_key |
| for ni_key in NI2ES: |
| fk = fold(ni_key) |
| if fk != ni_key and fk not in NI_FOLD: |
| NI_FOLD[fk] = ni_key |
| debug_print(f"Fold maps: ES_FOLD={len(ES_FOLD)}, NI_FOLD={len(NI_FOLD)}") |
|
|
| es_unique = len(ES2NI) |
| ni_unique = len(NI2ES) |
| pid_unique = len(pid_seen) |
|
|
| print(f"✓ BI-ONLY ESTRICTO cargado: {rows:,} filas.") |
| if dup_es: print(f"[AVISO] {dup_es:,} duplicados ES (se usó la primera).") |
| if dup_ni: print(f"[AVISO] {dup_ni:,} duplicados NI (bloqueados en modo estricto).") |
| if empty_pid: print(f"[AVISO] {empty_pid:,} filas sin pair_id.") |
| if mismatch_backmap: |
| print(f"[ALERTA] {mismatch_backmap:,} asimetrías ES↔NI (misma NI apunta a otro ES).") |
|
|
| sam_html = "" |
| if mismatch_samples: |
| sam_rows = "".join( |
| f"<li><code>{escape(es)}</code> → <code>{escape(ni)}</code> → <code>{escape(es2)}</code></li>" |
| for es,ni,es2 in mismatch_samples |
| ) |
| sam_html = f"<details><summary>Muestras</summary><ul>{sam_rows}</ul></details>" |
|
|
| ambN = sum(len(v) > 1 for v in AMBIG_NI.values()) |
| ambList = ", ".join(f"{k}→{sorted(list(v))[:3]}" for k,v in list(AMBIG_NI.items())[:5]) |
|
|
| BI_DIAG_HTML = f""" |
| <div style="font-family:Georgia,serif"> |
| <b>Diagnóstico del CSV BI</b><br> |
| Archivo: <b>{escape(CSV_BI)}</b><br> |
| Filas base (CSV): <b>{rows:,}</b><br> |
| ES únicas (tras expansiones): <b>{es_unique:,}</b> | NI únicas: <b>{ni_unique:,}</b> | pair_id únicos: <b>{pid_unique:,}</b><br> |
| Duplicados ES: <b>{dup_es:,}</b> | Duplicados NI: <b>{dup_ni:,}</b> (bloqueados en estricto) | Sin pair_id: <b>{empty_pid:,}</b><br> |
| Asimetrías ES↔NI: <b>{mismatch_backmap:,}</b> |
| {sam_html} |
| <hr style="border:0;border-top:1px solid #caa"> |
| <small>NI ambiguas bloqueadas: <b>{ambN:,}</b>{(' · ej.: ' + escape(ambList)) if ambN else ''}</small><br> |
| <small>Regla: el motor usa <b>sólo</b> tablas 1:1; NI duplicadas se bloquean y se muestran como <code>[AMB-NI:...]</code>.</small> |
| </div> |
| """ |
| return rows > 0 |
|
|
| print("Cargando léxico/pares (BI-estricto)…") |
| load_bi_strict_and_diagnose() |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| import glob, re as _re_patches |
|
|
| LEX_PATCH_PATTERN = _re_patches.compile(r"^\d{3}_.+\.csv$") |
| LEX_PATCH_LOG = [] |
| LEX_DEPRECATED_DIR = "deprecated" |
| STRICT_PATCHES = False |
|
|
| |
| |
| |
| |
| _KNOWN_LEGACY_ORPHANS = set() |
| _LEGACY_ORPHANS_FILE = os.path.join(LEX_DEPRECATED_DIR, "_known_legacy_orphans.csv") |
|
|
| def _load_known_legacy_orphans(): |
| if not os.path.exists(_LEGACY_ORPHANS_FILE): |
| return |
| try: |
| with open(_LEGACY_ORPHANS_FILE, "r", encoding="utf-8", newline="") as f: |
| for row in csv.DictReader(f): |
| es = (row.get("source_es") or "").strip().lower() |
| ni = (row.get("target_ni") or "").strip().lower() |
| if es and ni: |
| _KNOWN_LEGACY_ORPHANS.add((es, ni)) |
| debug_print(f"[PATCH] Legacy orphans aceptados: {len(_KNOWN_LEGACY_ORPHANS)}") |
| except Exception as e: |
| debug_print(f"[PATCH] No se pudo leer {_LEGACY_ORPHANS_FILE}: {e}") |
|
|
| def _append_to_graveyard(patch_name, row_data): |
| """Añade una fila al cementerio para el parche dado. |
| row_data: dict con source_es, target_ni, pos_es, es_morph, pair_id, reason.""" |
| try: |
| os.makedirs(LEX_DEPRECATED_DIR, exist_ok=True) |
| path = os.path.join(LEX_DEPRECATED_DIR, patch_name) |
| new_file = not os.path.exists(path) |
| with open(path, "a", encoding="utf-8", newline="") as f: |
| w = csv.DictWriter(f, fieldnames=[ |
| "source_es","target_ni","pos_es","es_morph","pair_id","reason" |
| ]) |
| if new_file: |
| w.writeheader() |
| w.writerow({k: row_data.get(k, "") for k in |
| ["source_es","target_ni","pos_es","es_morph","pair_id","reason"]}) |
| except Exception as e: |
| debug_print(f"[PATCH] No se pudo escribir cementerio: {e}") |
|
|
| def _patch_pre_lint(rows, patch_name, future_targets=None, future_es=None): |
| """Devuelve (errors, warnings) detectados antes de aplicar el parche. |
| Cada elemento: (kind, source_es, target_ni, message). |
| |
| future_targets / future_es: NIs y ES que aparecen como add/alias/replace |
| en parches POSTERIORES. Si un delete genera un huérfano cuyo NI o ES |
| aparece allí, se trata como rescate diferido (warning, no error).""" |
| errors = [] |
| warns = [] |
| future_targets = future_targets or set() |
| future_es = future_es or set() |
|
|
| |
| es_added = {} |
| ni_targets = {} |
| for r in rows: |
| op = (r.get("op") or "").strip().lower() |
| es = (r.get("source_es") or "").strip().lower() |
| ni = (r.get("target_ni") or "").strip().lower() |
| if op in ("add", "alias", "replace") and es and ni: |
| es_added[es] = ni |
| ni_targets.setdefault(ni, set()).add(es) |
|
|
| for r in rows: |
| op = (r.get("op") or "").strip().lower() |
| es = (r.get("source_es") or "").strip().lower() |
| ni = (r.get("target_ni") or "").strip().lower() |
| reason = (r.get("reason") or "").strip() |
|
|
| if op == "delete": |
| |
| if es in ES2NI: |
| old_ni = ES2NI[es][0].lower() if isinstance(ES2NI[es], tuple) else ES2NI[es] |
| ni_rescued = (old_ni in ni_targets) or (es in es_added) |
| if not ni_rescued: |
| pair = (es, old_ni) |
| if pair in _KNOWN_LEGACY_ORPHANS: |
| warns.append(("delete-orphan-known", es, old_ni, |
| "huérfano legacy aceptado")) |
| elif old_ni in future_targets or es in future_es: |
| warns.append(("delete-orphan-deferred-rescue", es, old_ni, |
| "rescatado en parche posterior")) |
| else: |
| errors.append(("delete-orphan-ni", es, old_ni, |
| f"NI {old_ni!r} quedaría huérfano sin reemplazo")) |
| elif op == "retire": |
| if not reason: |
| errors.append(("retire-no-reason", es, ni, |
| "retire requiere campo reason")) |
| elif op == "replace": |
| if not (es and ni): |
| errors.append(("replace-incomplete", es, ni, |
| "replace requiere ES y NI")) |
|
|
| return errors, warns |
|
|
| def apply_lex_patches(): |
| """Aplica todos los archivos de parche en orden numérico, NO DESTRUCTIVO. |
| Operaciones: add, override, alias, delete, replace, retire. |
| Antes de aplicar, ejecuta pre-lint para detectar errores destructivos. |
| """ |
| _load_known_legacy_orphans() |
|
|
| try: |
| all_files = os.listdir(".") |
| except Exception as e: |
| debug_print(f"[PATCH] No se pudo listar el directorio: {e}") |
| return |
|
|
| patch_files = sorted([f for f in all_files if LEX_PATCH_PATTERN.match(f)]) |
| if not patch_files: |
| debug_print("[PATCH] No se encontraron archivos NNN_*.csv") |
| return |
|
|
| print(f"[PATCH] Aplicando {len(patch_files)} archivo(s) de parche...") |
| totals = {"add":0,"override":0,"alias":0,"delete":0,"replace":0,"retire":0,"skipped":0} |
|
|
| |
| |
| parsed_patches = [] |
| for pf in patch_files: |
| try: |
| with open(pf, "r", encoding="utf-8", newline="") as f: |
| parsed_patches.append((pf, list(csv.DictReader(f)))) |
| except Exception: |
| parsed_patches.append((pf, None)) |
|
|
| for idx, (patch_path, rows) in enumerate(parsed_patches): |
| patch_name = patch_path |
| if rows is None: |
| print(f"[PATCH] Error leyendo {patch_name}") |
| continue |
| |
| future_ni = set() |
| future_es = set() |
| for fp, frows in parsed_patches[idx+1:]: |
| if frows is None: continue |
| for r in frows: |
| op = (r.get("op") or "").strip().lower() |
| es = (r.get("source_es") or "").strip().lower() |
| ni = (r.get("target_ni") or "").strip().lower() |
| if op in ("add", "alias", "replace") and es and ni: |
| future_ni.add(ni) |
| future_es.add(es) |
|
|
| ops = {"add":0,"override":0,"alias":0,"delete":0,"replace":0,"retire":0,"skipped":0} |
|
|
| |
| errors, warns = _patch_pre_lint(rows, patch_name, |
| future_targets=future_ni, |
| future_es=future_es) |
| if errors: |
| print(f"[PATCH] {patch_name}: {len(errors)} error(es) destructivo(s) detectado(s):") |
| for kind, es, ni, msg in errors[:5]: |
| print(f" ✗ [{kind}] {es!r}: {msg}") |
| if len(errors) > 5: |
| print(f" ... y {len(errors)-5} más") |
| if STRICT_PATCHES: |
| raise RuntimeError( |
| f"Parche {patch_name} no pasa el linter (STRICT_PATCHES=True). " |
| "Documenta en deprecated/_known_legacy_orphans.csv o usa op=replace.") |
| for kind, es, ni, msg in errors: |
| LEX_PATCH_LOG.append((patch_name, kind, es, ni, "lint-error", msg)) |
| if warns: |
| for kind, es, ni, msg in warns: |
| LEX_PATCH_LOG.append((patch_name, kind, es, ni, "lint-warn", msg)) |
|
|
| for row in rows: |
| op = (row.get("op") or "").strip().lower() |
| es = (row.get("source_es") or "").strip() |
| ni = (row.get("target_ni") or "").strip() |
| pos = (row.get("pos_es") or "").strip() |
| morph = (row.get("es_morph") or "").strip() |
| pid = (row.get("pair_id") or "").strip() or f"patch::{patch_name}" |
| reason = (row.get("reason") or "").strip() |
|
|
| es_l = es.lower() |
| ni_l = ni.lower() |
|
|
| if op == "add": |
| if es_l in ES2NI: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", "ES ya existe")) |
| continue |
| if ni_l in NI2ES: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", "NI ya existe (ambigüedad)")) |
| continue |
| ES2NI[es_l] = (ni, pid) |
| NI2ES[ni_l] = (es, pid) |
| if pos: ES2NI_POS[es_l] = pos |
| if morph: ES2NI_MORPH[es_l] = morph |
| ops["add"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", "")) |
|
|
| elif op == "override": |
| prev = ES2NI.get(es_l) |
| ES2NI[es_l] = (ni, pid) |
| NI2ES[ni_l] = (es, pid) |
| if pos: ES2NI_POS[es_l] = pos |
| if morph: ES2NI_MORPH[es_l] = morph |
| ops["override"] += 1 |
| prev_str = f"era {prev[0]}" if prev else "no existía" |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", prev_str)) |
|
|
| elif op == "alias": |
| if es_l in ES2NI: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", "ES ya existe")) |
| continue |
| ES2NI[es_l] = (ni, pid) |
| if pos: ES2NI_POS[es_l] = pos |
| if morph: ES2NI_MORPH[es_l] = morph |
| |
| ops["alias"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", "alias ortográfico")) |
|
|
| elif op == "delete": |
| if es_l not in ES2NI: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", "no existía")) |
| continue |
| old_ni_surf, old_pid = ES2NI[es_l] |
| old_pos = ES2NI_POS.get(es_l, "") |
| old_morph = ES2NI_MORPH.get(es_l, "") |
| |
| _append_to_graveyard(patch_name, { |
| "source_es": es, "target_ni": old_ni_surf, |
| "pos_es": old_pos, "es_morph": old_morph, |
| "pair_id": old_pid, |
| "reason": reason or "delete sin reason (legacy)", |
| }) |
| del ES2NI[es_l] |
| ES2NI_POS.pop(es_l, None) |
| ES2NI_MORPH.pop(es_l, None) |
| |
| if old_ni_surf.lower() in NI2ES and \ |
| NI2ES[old_ni_surf.lower()][0].lower() == es_l: |
| del NI2ES[old_ni_surf.lower()] |
| ops["delete"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", "movido a deprecated/")) |
|
|
| elif op == "replace": |
| if not (es and ni): |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", |
| "replace requiere ES y NI")) |
| continue |
| |
| if ni_l in NI2ES and NI2ES[ni_l][0].lower() != es_l: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", |
| f"NI nuevo ya pertenece a {NI2ES[ni_l][0]!r}")) |
| continue |
| |
| if es_l in ES2NI: |
| old_ni_surf, old_pid = ES2NI[es_l] |
| old_pos = ES2NI_POS.get(es_l, "") |
| old_morph = ES2NI_MORPH.get(es_l, "") |
| _append_to_graveyard(patch_name, { |
| "source_es": es, "target_ni": old_ni_surf, |
| "pos_es": old_pos, "es_morph": old_morph, |
| "pair_id": old_pid, |
| "reason": reason or f"replaced by {ni}", |
| }) |
| if old_ni_surf.lower() in NI2ES and \ |
| NI2ES[old_ni_surf.lower()][0].lower() == es_l: |
| del NI2ES[old_ni_surf.lower()] |
| |
| ES2NI[es_l] = (ni, pid) |
| NI2ES[ni_l] = (es, pid) |
| if pos: ES2NI_POS[es_l] = pos |
| if morph: ES2NI_MORPH[es_l] = morph |
| ops["replace"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", reason or "")) |
|
|
| elif op == "retire": |
| if not reason: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", |
| "retire requiere reason")) |
| continue |
| if es_l not in ES2NI: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "skip", |
| "no existía")) |
| continue |
| old_ni_surf, old_pid = ES2NI[es_l] |
| _append_to_graveyard(patch_name, { |
| "source_es": es, "target_ni": old_ni_surf, |
| "pos_es": ES2NI_POS.get(es_l,""), |
| "es_morph": ES2NI_MORPH.get(es_l,""), |
| "pair_id": old_pid, "reason": reason, |
| }) |
| del ES2NI[es_l] |
| ES2NI_POS.pop(es_l, None) |
| ES2NI_MORPH.pop(es_l, None) |
| if old_ni_surf.lower() in NI2ES and \ |
| NI2ES[old_ni_surf.lower()][0].lower() == es_l: |
| del NI2ES[old_ni_surf.lower()] |
| ops["retire"] += 1 |
| LEX_PATCH_LOG.append((patch_name, op, es, ni, "ok", reason)) |
|
|
| else: |
| ops["skipped"] += 1 |
| LEX_PATCH_LOG.append((patch_name, "?", es, ni, "skip", |
| f"op desconocida: {op!r}")) |
|
|
| |
| summary = ", ".join(f"{k}={v}" for k,v in ops.items() if v) |
| print(f"[PATCH] {patch_name}: {summary or 'sin cambios'}") |
| for k in totals: |
| totals[k] += ops[k] |
|
|
| print(f"[PATCH] Total: " + ", ".join(f"{k}={v}" for k,v in totals.items() if v)) |
|
|
| |
| if any(totals[k] for k in ("add","override","replace","delete","retire")): |
| ES_FOLD.clear(); NI_FOLD.clear() |
| for es_key in ES2NI: |
| fk = fold(es_key) |
| if fk != es_key and fk not in ES_FOLD: |
| ES_FOLD[fk] = es_key |
| for ni_key in NI2ES: |
| fk = fold(ni_key) |
| if fk != ni_key and fk not in NI_FOLD: |
| NI_FOLD[fk] = ni_key |
|
|
| apply_lex_patches() |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| def _register_ipfv_3s_reverse(): |
| """Registra formas 3S inversas para tiempos donde 1S=3S en español. |
| |
| Patrones cubiertos: |
| -ska-n (IPFV 1S) → -ska (IPFV 3S) |
| -tei-n (COND 1S) → -tei (COND 3S) |
| -na-n (SBJ 1S) → -na (SBJ 3S) |
| -nabo-n (SBJ_IPFV 1S) → -nabo (SBJ_IPFV 3S) |
| """ |
| suffixes_1s_to_3s = ['-ska-n', '-tei-n', '-na-n', '-nabo-n'] |
| added = 0 |
| skipped = 0 |
| new_entries = [] |
| for ni_key, val in list(NI2ES.items()): |
| for suf in suffixes_1s_to_3s: |
| if ni_key.endswith(suf): |
| ni_3s = ni_key[:-2] |
| if ni_3s in NI2ES: |
| skipped += 1 |
| break |
| new_entries.append((ni_3s, val)) |
| break |
| for ni_3s, val in new_entries: |
| NI2ES[ni_3s] = val |
| added += 1 |
| debug_print(f"3S reverse map (IPFV/COND/SBJ/SBJ_IPFV): {added} formas registradas, {skipped} ya existían") |
|
|
| _register_ipfv_3s_reverse() |
|
|
| |
| |
| |
| |
| VERSION_MARKER = "v87_2026_04_30" |
| try: |
| print(f"[Neoíbero translator] versión cargada: {VERSION_MARKER}", flush=True) |
| print(f"[Neoíbero translator] léxico activo: {CSV_BI}", flush=True) |
| except Exception: |
| pass |
|
|
| |
| def _longest_match(tokens, i, phrase_map): |
| """Devuelve (span, surface) si hay frase que comience en i.""" |
| if not phrase_map: return (0, None) |
| max_span = 0; surface = None |
| for span in range(1, MAX_NGRAM+1): |
| if i+span > len(tokens): break |
| cand = " ".join(lower(t) for t in tokens[i:i+span]) |
| if cand in phrase_map: |
| max_span = span |
| surface = phrase_map[cand][0] |
| else: |
| fcand = " ".join(fold(lower(t)) for t in tokens[i:i+span]) |
| if fcand != cand and fcand in phrase_map: |
| max_span = span |
| surface = phrase_map[fcand][0] |
| return (max_span, surface) |
|
|
| |
| def sentence_case_spanish(s: str) -> str: |
| out = [] |
| start = True |
| in_br = False |
| WRAPS = "¿¡\"'«(“‘[" |
|
|
| for ch in s: |
| if ch == '[': |
| in_br = True |
|
|
| if not in_br and start: |
| if ch.isspace(): |
| out.append(ch) |
| elif ch in WRAPS: |
| out.append(ch) |
| elif ch.isalpha(): |
| out.append(ch.upper()); start = False |
| else: |
| out.append(ch) |
| start = ch in "¿¡" |
| else: |
| out.append(ch) |
| if not in_br and ch in ".?!…": |
| start = True |
| elif not in_br and ch in "¿¡": |
| start = True |
|
|
| if ch == ']': |
| in_br = False |
|
|
| return "".join(out) |
|
|
| |
| def postprocess_spanish(s: str) -> str: |
| |
| s = re.sub(r"(\d)\s*:\s*(\d)", r"\1:\2", s) |
| s = re.sub(r"(\d)\s*([.,])\s*(\d)", r"\1\2\3", s) |
| |
| s = re.sub(r"\s+([,.;:!?])", r"\1", s) |
| |
| s = re.sub(r"([?.!;])(?!\s|$)([^\s])", r"\1 \2", s) |
| |
| s = re.sub(r"([¿¡])\s+", r"\1", s) |
| |
| s = re.sub(r"\s{2,}", " ", s).strip() |
| |
| return sentence_case_spanish(s) |
|
|
| |
| def translate_es_to_ni_bi(text:str): |
| |
| lines = (text or "").split("\n") |
| if len(lines) > 1: |
| results = [translate_es_to_ni_bi(line) for line in lines] |
| lat = "\n".join(r[0] for r in results) |
| ib = "\n".join(r[1] for r in results) |
| return lat, ib |
|
|
| toks = simple_tokenize(text) |
| toks = expand_enclitics(toks) |
|
|
| |
| _NOUN_CTX = frozenset({'el','la','los','las','un','una','unos','unas','al','del', |
| 'de','en','con','por','para','a','sin','sobre','entre', |
| 'hacia','hasta','desde','contra','según','ante','bajo','tras', |
| 'mi','tu','su','mis','tus','sus','nuestro','nuestra', |
| 'nuestros','nuestras','vuestro','vuestra','vuestros','vuestras', |
| 'este','esta','estos','estas','ese','esa','esos','esas', |
| 'aquel','aquella','aquellos','aquellas','cada','otro','otra', |
| 'mucho','mucha','muchos','muchas','poco','poca','pocos','pocas', |
| 'todo','toda','todos','todas','algún','alguna','ningún','ninguna', |
| 'buen','mal','gran','primer','tercer','qué','cuánto','cuánta'}) |
| _VERB_CTX = frozenset({'yo','tú','él','ella','nosotros','nosotras','vosotros','vosotras', |
| 'ellos','ellas','usted','ustedes', |
| 'se','me','te','nos','os','le','les','lo', |
| 'no','ya','también','tampoco','nunca','siempre','aún','todavía', |
| 'que','quien','quienes','donde','cuando','como','si'}) |
| _INFINITIVE_ENDINGS = ('ar','er','ir') |
| _INFINITIVE_CTX = frozenset({'de','sin','para','por','al','antes','tras','hasta'}) |
| _VERB_ALWAYS = frozenset({'son','es','ha','he','era','fue','fui','van', |
| 'dan','das','den','des','hay','doy','soy','voy', |
| 'iba','di','haya'}) |
|
|
| |
| |
| _TIME_WORDS = frozenset({'año','años','día','días','mes','meses', |
| 'semana','semanas','hora','horas', |
| 'minuto','minutos','segundo','segundos', |
| 'tiempo','rato','siglo','siglos', |
| 'década','décadas','momento','momentos', |
| 'instante','instantes','jornada','jornadas', |
| 'noche','noches','tarde','tardes','mañana','mañanas', |
| 'milenio','milenios'}) |
| _TIME_QUANTIFIERS = frozenset({'mucho','muchos','mucha','muchas', |
| 'poco','pocos','poca','pocas', |
| 'tanto','tantos','tanta','tantas', |
| 'algunos','algunas','varios','varias', |
| 'demasiado','demasiados','demasiada','demasiadas', |
| 'un','una','unos','unas', |
| 'dos','tres','cuatro','cinco','seis','siete', |
| 'ocho','nueve','diez','once','doce','trece', |
| 'catorce','quince','veinte','treinta','cuarenta', |
| 'cincuenta','cien','mil','medio','media'}) |
|
|
| |
| |
| |
| _FIRST_PERSON_SUBJECTS = frozenset({'yo'}) |
| _THIRD_SG_PRON_SUBJECTS = frozenset({'él','ella','ello','usted','esto','eso','aquello'}) |
| _SINGULAR_NP_DETS = frozenset({'el','la','un','una','este','esta','ese','esa','aquel','aquella', |
| 'mi','tu','su','nuestro','nuestra','vuestro','vuestra'}) |
|
|
| |
| _PREP_CTX = frozenset({'a','ante','con','contra','de','desde','en','entre', |
| 'hacia','hasta','para','por','según','sin','sobre','tras'}) |
| _TILDE_MAP = {'mi': 'mí', 'el': 'él', 'si': 'sí', 'tu': 'tú'} |
| _PHRASE_BREAK = frozenset({'y','o','e','u','ni','que','pero','sino','como', |
| 'porque','cuando','donde','aunque','pues'}) |
|
|
| def _accented_lookup(key): |
| if key in _TILDE_MAP and _TILDE_MAP[key] in ES2NI: |
| return ES2NI[_TILDE_MAP[key]][0] |
| return None |
|
|
| def _resolve_forms(raw_key:str): |
| """Devuelve (canon_key, ni_nom, ni_verb) con soporte fold.""" |
| key = lower(raw_key) |
| if key in ES2NI: |
| return key, ES2NI[key][0], ES2NI_VERB.get(key, (None, None))[0] |
| fkey = fold(key) |
| if fkey in ES_FOLD: |
| actual = ES_FOLD[fkey] |
| return actual, ES2NI[actual][0], ES2NI_VERB.get(actual, (None, None))[0] |
| return key, None, None |
|
|
| def _choose_es_to_ni(current_tok:str, prev_key:str, next_key:str, next_next_key:str, sent_start:bool): |
| actual_key, ni_nom, ni_verb = _resolve_forms(current_tok) |
| if ni_nom is None and ni_verb is None: |
| return None |
|
|
| key = lower(actual_key) |
|
|
| |
| |
| |
| |
| |
| if key == 'hace': |
| is_time_context = False |
| if next_key in _TIME_WORDS: |
| is_time_context = True |
| elif (next_key in _TIME_QUANTIFIERS or is_number(next_key)) and next_next_key in _TIME_WORDS: |
| is_time_context = True |
| if is_time_context: |
| |
| if 'atrás' in ES2NI: |
| return ES2NI['atrás'][0] |
|
|
| |
| |
| if key.endswith(_INFINITIVE_ENDINGS): |
| if key in ES2NI: |
| return ES2NI[key][0] |
| fkey = fold(key) |
| if fkey in ES_FOLD: |
| return ES2NI[ES_FOLD[fkey]][0] |
|
|
| |
| if key in _VERB_ALWAYS and ni_verb: |
| return ni_verb |
|
|
| |
| if key in _TILDE_MAP: |
| accented_ni = _accented_lookup(key) |
| if accented_ni: |
| if prev_key in _PREP_CTX: |
| if not next_key or next_key in VISIBLE_PUNCT or next_key in _PHRASE_BREAK: |
| return accented_ni |
|
|
| |
| if prev_key in _NOUN_CTX and ni_nom: |
| return ni_nom |
|
|
| |
| if prev_key in _VERB_CTX and ni_verb: |
| return ni_verb |
|
|
| |
| if prev_key in _INFINITIVE_CTX: |
| inf_key = lower(current_tok) |
| if inf_key.endswith(_INFINITIVE_ENDINGS): |
| if inf_key in ES2NI: |
| return ES2NI[inf_key][0] |
| ff = fold(inf_key) |
| if ff in ES_FOLD: |
| return ES2NI[ES_FOLD[ff]][0] |
| if ni_verb: |
| return ni_verb |
|
|
| |
| |
| if sent_start: |
| if key.endswith(_INFINITIVE_ENDINGS): |
| if key in ES2NI: |
| return ES2NI[key][0] |
| ff = fold(key) |
| if ff in ES_FOLD: |
| return ES2NI[ES_FOLD[ff]][0] |
| if ni_verb and not ni_nom: |
| return ni_verb |
|
|
| |
| |
| |
| |
| |
| if ni_verb and next_key in _NOUN_CTX: |
| pos_nom = ES2NI_POS.get(key, "") |
| if pos_nom != "ADJ": |
| return ni_verb |
|
|
| |
| if ni_nom is not None: |
| return ni_nom |
| if ni_verb is not None: |
| return ni_verb |
| return None |
|
|
| def _has_explicit_3s_subject(left_context): |
| """¿Hay un sujeto 3S explícito en los últimos tokens (misma cláusula)? |
| |
| Solo retorna True para casos seguros y locales: |
| - Pronombre 3S (él/ella/usted/esto/eso/aquello) justo antes |
| - Determinante singular (el/la/un/mi/tu/su...) seguido de un sustantivo, |
| no precedido de preposición (para excluir locativos: "en la casa vivía") |
| - Bloqueado si "yo" aparece en los últimos 2 tokens |
| """ |
| ctx = [lower(x) for x in (left_context or []) if x] |
| if not ctx: |
| return False |
| tail = ctx[-5:] |
|
|
| |
| if any(tok in _FIRST_PERSON_SUBJECTS for tok in tail[-2:]): |
| return False |
|
|
| |
| if tail[-1] in _THIRD_SG_PRON_SUBJECTS: |
| return True |
|
|
| |
| |
| for j in range(len(tail)-2, -1, -1): |
| if tail[j] in _SINGULAR_NP_DETS: |
| if j > 0 and tail[j-1] in _PREP_CTX: |
| return False |
| if j < len(tail)-1: |
| return True |
| return False |
|
|
| def _adjust_ipfv_ambiguous_person(ni, left_context): |
| """Si el NI es 1S de un tiempo donde 1S=3S en español, quitar -n cuando contexto es 3S. |
| |
| Tiempos cubiertos (todos comparten forma 1S=3S en español): |
| -ska-n IPFV 1S → -ska IPFV 3S (vivía) |
| -tei-n COND 1S → -tei COND 3S (viviría) |
| -na-n SBJ 1S → -na SBJ 3S (viva) |
| -nabo-n SBJ_IPFV 1S → -nabo SBJ_IPFV 3S (viviera) |
| """ |
| if not (ni and isinstance(ni, str)): |
| return ni |
| ambiguous_suffixes = ("-ska-n", "-tei-n", "-na-n", "-nabo-n") |
| if any(ni.endswith(suf) for suf in ambiguous_suffixes): |
| if _has_explicit_3s_subject(left_context): |
| return ni[:-2] |
| return ni |
|
|
| out=[]; ib_toks=[] |
| i=0; prev_key="" |
| sent_start = True |
| left_context=[] |
| while i < len(toks): |
| t = toks[i] |
| if t in VISIBLE_PUNCT: |
| out.append(t); ib_toks.append(t); prev_key=""; i+=1 |
| if t in SENT_END: |
| sent_start = True |
| left_context=[] |
| elif t in CLAUSE_BREAKS: |
| left_context=[] |
| continue |
| if is_placeholder(t): |
| out.append(t); ib_toks.append(t); prev_key=""; i+=1 |
| sent_start = False |
| left_context.append(t) |
| continue |
| span, ni_surface = _longest_match(toks, i, ESPHRASE2NI) |
| if span > 1: |
| ni_surface = _adjust_ipfv_ambiguous_person(ni_surface, left_context) |
| out.append(ni_surface) |
| ib_toks.append(georgeos_keys(tokens_from_latin(ni_surface), ni_surface)) |
| prev_key = lower(toks[i+span-1]) if i+span-1 < len(toks) else "" |
| for k_idx in range(i, i+span): |
| left_context.append(toks[k_idx]) |
| i += span |
| sent_start = False |
| continue |
|
|
| next_key = "" |
| next_next_key = "" |
| j = i + 1 |
| |
| while j < len(toks): |
| if toks[j] in VISIBLE_PUNCT: |
| if toks[j] in SENT_END: |
| break |
| j += 1 |
| continue |
| next_key = lower(toks[j]) |
| break |
| |
| if next_key: |
| k = j + 1 |
| while k < len(toks): |
| if toks[k] in VISIBLE_PUNCT: |
| if toks[k] in SENT_END: |
| break |
| k += 1 |
| continue |
| next_next_key = lower(toks[k]) |
| break |
|
|
| key = lower(t) |
| ni = _choose_es_to_ni(t, prev_key, next_key, next_next_key, sent_start) |
| ni = _adjust_ipfv_ambiguous_person(ni, left_context) |
|
|
| if ni is not None: |
| out.append(ni) |
| ib_toks.append(georgeos_keys(tokens_from_latin(ni), ni)) |
| elif is_number(key): |
| ni_num = digit_to_ni(key) |
| out.append(ni_num); ib_toks.append(georgeos_keys(tokens_from_latin(ni_num), ni_num)) |
| else: |
| ph = f"[SIN-LEX:{t}]" |
| out.append(ph); ib_toks.append(ph) |
|
|
| prev_key = key |
| left_context.append(t) |
| i += 1 |
| sent_start = False |
|
|
| if MODAL_SUFFIX_ENABLE: |
| out = add_modal_suffixes_es2ni(out) |
| ib_toks = [] |
| for tt in out: |
| if tt in VISIBLE_PUNCT or tt.startswith("["): |
| ib_toks.append(tt) |
| else: |
| ib_toks.append(georgeos_keys(tokens_from_latin(tt), tt)) |
|
|
| ni_text = detokenize(out) |
| ib_html = "<div class='ib-line'>" + escape(render_ib_with_tridots(ib_toks)) + "</div>" |
| return ni_text, ib_html |
|
|
|
|
| def translate_ni_to_es_bi(text:str): |
| |
| lines = (text or "").split("\n") |
| if len(lines) > 1: |
| results = [translate_ni_to_es_bi(line) for line in lines] |
| return "\n".join(results) |
|
|
| toks = simple_tokenize(text) |
|
|
| if MODAL_SUFFIX_ENABLE: |
| toks = strip_modal_suffixes_ni(toks) |
|
|
| |
| |
| |
| |
| def _is_doge_hace_context(idx, tokens): |
| """Returns True if tokens[idx] is 'doge' acting as 'hace + tiempo' marker.""" |
| if idx >= len(tokens) or lower(tokens[idx]) != 'doge': |
| return False |
| |
| if idx+1 >= len(tokens): return False |
| nxt = lower(tokens[idx+1]) |
| if idx+2 >= len(tokens): return False |
| nnxt = lower(tokens[idx+2]) |
| |
| ni_time_roots = ('temket','lunbem','aniklun','dukman','śenmil','bukśar','śilśiŕ', |
| 'libit','ligimtir','natmaŕlom','golkormal','setlandis','daninmor') |
| nnxt_root = nnxt |
| if nnxt_root.startswith('ti-'): nnxt_root = nnxt_root[3:] |
| if nnxt_root.endswith('-k'): nnxt_root = nnxt_root[:-2] |
| if nnxt_root in ni_time_roots: |
| return True |
| |
| nnxt_es = NI2ES.get(nnxt, (None,))[0] or "" |
| if nnxt_es: |
| es_time_words = ('año','años','día','días','mes','meses','semana','semanas', |
| 'hora','horas','momento','momentos','rato','ratos','tiempo') |
| if nnxt_es.lower() in es_time_words: |
| return True |
| return False |
|
|
| def _is_at_sentence_start(idx, tokens): |
| """¿Está el token idx al inicio absoluto o tras puntuación final?""" |
| if idx == 0: |
| return True |
| prev = tokens[idx-1] |
| return prev in SENT_END or prev == "." or prev == "!" or prev == "?" |
|
|
| out=[] |
| i=0 |
| while i < len(toks): |
| t = toks[i] |
| if t in VISIBLE_PUNCT: |
| out.append(t); i+=1; continue |
| if is_placeholder(t): |
| out.append(t); i+=1; continue |
| span, es_surface = _longest_match(toks, i, NIPHRASE2ES) |
| if span > 1: |
| out.append(es_surface); i += span; continue |
|
|
| key = lower(t) |
| fkey = fold(key) |
|
|
| |
| if key == 'doge' and _is_doge_hace_context(i, toks): |
| if _is_at_sentence_start(i, toks): |
| out.append('Hace') |
| else: |
| out.append('hace') |
| i += 1 |
| continue |
|
|
| |
| |
| |
| |
| |
| |
| if key == 'galbi-ke' and i+1 < len(toks): |
| nxt = lower(toks[i+1]) |
| |
| is_part = nxt.endswith('-ir') or '-ir-' in nxt |
| if is_part: |
| out.append('ha') |
| i += 1 |
| continue |
|
|
| if key in NI2ES: |
| es = NI2ES[key][0] or "" |
| out.append(es if es else f"[?:{t}]") |
| elif fkey in NI_FOLD: |
| es = NI2ES[NI_FOLD[fkey]][0] or "" |
| out.append(es if es else f"[?:{t}]") |
| elif key in AMBIG_NI or fkey in AMBIG_NI and STRICT_BI_ENFORCE: |
| out.append(f"[AMB-NI:{t}]") |
| elif is_number(key): |
| out.append(t) |
| else: |
| out.append(f"[?:{t}]") |
| i += 1 |
|
|
| if MODAL_SUFFIX_ENABLE: |
| out = add_inverted_openers(out) |
|
|
| es_text = detokenize(out) |
| es_text = postprocess_spanish(es_text) |
| es_text = fuse_enclitics_es(es_text) |
| return es_text |
|
|
| |
| def diagnose_text(text, dir_label): |
| if not text or not text.strip(): |
| return "<em>Introduce texto para diagnosticar.</em>" |
|
|
| toks = simple_tokenize(text) |
| if dir_label.startswith("ES"): |
| toks = expand_enclitics(toks) |
| unknown=set(); asym=set(); amb=set() |
| total_tokens=0; covered=0 |
|
|
| if dir_label.startswith("ES"): |
| head = "ES→NI" |
| i=0 |
| while i < len(toks): |
| t = toks[i] |
| if t in VISIBLE_PUNCT or is_number(t): |
| i+=1; continue |
| total_tokens += 1 |
| span, _ = _longest_match(toks, i, ESPHRASE2NI) |
| if span > 1: |
| covered += 1; i += span; continue |
| k=lower(t) |
| fk=fold(k) |
| if k not in ES2NI and fk not in ES_FOLD: |
| unknown.add(t); i+=1; continue |
| if k not in ES2NI: k=ES_FOLD.get(fk, k) |
| covered += 1 |
| ni = ES2NI[k][0] |
| back = NI2ES.get(lower(ni)) |
| if back and lower(back[0]) != k: |
| asym.add(f"{t} → {ni} → {back[0]}") |
| i+=1 |
| else: |
| head = "NI→ES" |
| i=0 |
| while i < len(toks): |
| t = toks[i] |
| if t in VISIBLE_PUNCT or is_number(t): |
| i+=1; continue |
| total_tokens += 1 |
| span, _ = _longest_match(toks, i, NIPHRASE2ES) |
| if span > 1: |
| covered += 1; i += span; continue |
| k=lower(t) |
| fk=fold(k) |
| if k in AMBIG_NI or fk in AMBIG_NI: |
| amb.add(t); i+=1; continue |
| if k not in NI2ES and fk not in NI_FOLD: |
| unknown.add(t); i+=1; continue |
| if k not in NI2ES: k=NI_FOLD.get(fk, k) |
| covered += 1 |
| es = NI2ES[k][0] |
| back = ES2NI.get(lower(es)) |
| if back and lower(back[0]) != k: |
| asym.add(f"{t} → {es} → {back[0]}") |
| i+=1 |
|
|
| cov_pct = (covered/total_tokens*100) if total_tokens else 100.0 |
| cov_html = f"<div><b>Tokens (sin puntuación/numéricos):</b> {total_tokens} | <b>Cubiertos:</b> {covered} ({cov_pct:.1f}%)</div>" |
|
|
| unk_html = "".join(f"<li><code>{escape(u)}</code></li>" for u in sorted(unknown, key=lambda x: lower(x))) or "<li><i>—</i></li>" |
| amb_html = "".join(f"<li><code>{escape(a)}</code></li>" for a in sorted(amb, key=lambda x: lower(x))) or "<li><i>—</i></li>" |
| asy_html = "".join(f"<li><code>{escape(a)}</code></li>" for a in sorted(asym)) or "<li><i>—</i></li>" |
|
|
| return f"<b>Diagnóstico {head}</b>{cov_html}<b>Ambiguas (NI duplicada):</b><ul>{amb_html}</ul><b>Faltantes:</b><ul>{unk_html}</ul><b>Asimetrías:</b><ul>{asy_html}</ul>" |
|
|
| |
| LABELS={ |
| "ES":{ |
| "title":"Traductor Español ↔ Neoíbero", |
| "subtitle":"CSV estricto (BI-only 1:1; desambiguación ligera ES→NI; .gz) — determinista", |
| "in_label_es":"✏️ Entrada (Español)", |
| "in_label_ni":"✏️ Entrada (Neoíbero)", |
| "in_ph_es":"Escribe aquí. Ej.: Veo a Ana y doy pan a Marta.", |
| "in_ph_ni":"Idatzi hemen. Adib.: nuker-ke ni etxe-ka.", |
| "out_lat_esni":"📜 Salida: Neoíbero (latín)", |
| "out_lat_nies":"📜 Salida: Español", |
| "out_ib":"🗿 Línea ibérica", |
| "out_audio":"🔊 Locución (Audio)", |
| "btn":"🔄 Traducir", |
| "combo":"🌍 Idioma (UI + explicación)", |
| "dir":"🔁 Dirección", |
| "dir_opts":["ES → NI","NI → ES"], |
| "doc_header":"📚 Documentación y Referencia", |
| "acc_titles":[ |
| "🌍 ¿Qué es el neoíbero?", |
| "🔤 Fonología y escritura", |
| "📐 Sistema nominal: género, número y caso", |
| "🔄 Sistema verbal: TAM, persona y clíticos", |
| "🌿 Derivación y familias de palabras", |
| "🔢 Sistema numérico vigesimal", |
| "📝 Sintaxis básica y partículas", |
| "❓ Modalidad vascoide (-na / -ba)", |
| "⚙️ Pipeline del traductor (1:1 estricto)", |
| "📚 Bibliografía y créditos", |
| "🧾 Glosario técnico" |
| ] |
| }, |
| "EN":{ |
| "title":"Spanish ↔ Neo-Iberian Translator", |
| "subtitle":"Strict BI-only (1:1 surfaces; light ES→NI disambiguation; .gz) — deterministic", |
| "in_label_es":"✏️ Input (Spanish)", |
| "in_label_ni":"✏️ Input (Neo-Iberian)", |
| "in_ph_es":"Type here. E.g., Veo a Ana y doy pan a Marta.", |
| "in_ph_ni":"Type here. E.g., nuker-ke ni etxe-ka.", |
| "out_lat_esni":"📜 Output: Neo-Iberian (Latin)", |
| "out_lat_nies":"📜 Output: Spanish", |
| "out_ib":"🗿 Iberian line", |
| "out_audio":"🔊 Speech (Audio)", |
| "btn":"🔄 Translate", |
| "combo":"🌍 Language (UI + docs)", |
| "dir":"🔁 Direction", |
| "dir_opts":["ES → NI","NI → ES"], |
| "doc_header":"📚 Documentation & Reference", |
| "acc_titles":[ |
| "🌍 What is Neo-Iberian?", |
| "🔤 Phonology and writing", |
| "📐 Nominal system: gender, number & case", |
| "🔄 Verbal system: TAM, person & clitics", |
| "🌿 Derivation and word families", |
| "🔢 Vigesimal number system", |
| "📝 Basic syntax and particles", |
| "❓ Vascoid modality (-na / -ba)", |
| "⚙️ Translator pipeline (strict 1:1)", |
| "📚 Bibliography and credits", |
| "🧾 Technical glossary" |
| ] |
| } |
| } |
| DOC = { |
| "ES":[ |
| """**El neoíbero** es una lengua construida (conlang) inspirada en el antiguo íbero, la lengua prerromana hablada en la costa oriental de la península ibérica entre los siglos V–I a.C. Aunque el íbero histórico permanece sin descifrar por completo, el neoíbero reconstruye un sistema lingüístico coherente y aprendible a partir de lo que sí sabemos: su fonotaxis, su escritura semisilábica, los morfemas recurrentes en inscripciones, y los contactos documentados con otras lenguas mediterráneas. |
| |
| **Principios de diseño:** el neoíbero no pretende ser una reconstrucción filológica del íbero (imposible con los datos actuales), sino una conlang que *podría haber sido* su heredera natural. Se priorizan: regularidad morfológica, aprendibilidad, fidelidad a la fonotaxis atestiguada, y ausencia de influencia latina/romance en la gramática (la influencia indoeuropea, cuando existe, viene del contacto celta y griego documentado arqueológicamente). |
| |
| **Autoría:** conlang diseñada por David García. Generación léxica a través de scripts. Fuente tipográfica ibérica: Iberia-Georgeos de Joan Ferrer i Jané.""", |
|
|
| """**Inventario fonológico.** Consonantes: oclusivas *b, d, t, g, k* (sin /p/ superficial: p→b, rasgo atestiguado); fricativas *s, ś* (sibilante palatalizada); nasales *m, n*; líquidas *l, r, ŕ* (vibrante simple vs múltiple). Vocales: *a, e, i, o, u* (cinco timbres, como en español y vasco). |
| |
| **Fonotaxis.** Estructura silábica predominante CV(C), heredada de las inscripciones ibéricas. Los grupos consonánticos iniciales son raros; las codas admiten *-n, -s, -ś, -r, -ŕ, -l, -m, -k, -t*. |
| |
| **Escritura.** El neoíbero se escribe en tres sistemas: |
| - **Latín adaptado:** el sistema principal, con ś (s palatalizada) y ŕ (vibrante múltiple) como únicos diacríticos. |
| - **Línea ibérica:** semisilabario basado en la escritura levantina, con signos silábicos para *ba, be, bi, bo, bu; da, de…; ta, te…; ga, ge…; ka, ke…* y signos alfabéticos para vocales y consonantes de coda. |
| - **Claves compactas:** versión abreviada de la línea ibérica para uso digital. |
| |
| El guion (-) separa morfemas dentro de una palabra: *aŕen-ke-n* = ser-PRS-1SG = 'soy'.""", |
|
|
| """**Género.** Dos géneros: masculino (forma base, no marcada) y femenino (prefijo *ti-*, atestiguado en ibérico: eban/t-eban, olor/t-olor). |
| | | Singular | Plural | |
| |---|---|---| |
| | Masc. | *ituśder* (bueno) | *ituśder-k* | |
| | Fem. | *ti-ituśder* (buena) | *ti-ituśder-k* | |
| |
| **Número.** Dos marcadores de plural: *-k* (plural corto, el más frecuente) y *-nes* (plural largo, en sustantivos). La elección se fija léxicamente por cada raíz. |
| |
| **Determinantes.** |
| | Español | Neoíbero | |
| |---|---| |
| | el | *eno* | |
| | la | *ena* | |
| | los | *enos* | |
| | las | *enas* | |
| | un | *dan* | |
| |
| **Pronombres personales:** *nai* (yo), *śe* (tú), *eki* (nosotros). Los pronombres de 3ª persona se derivan de demostrativos. |
| |
| **Caso.** El neoíbero conserva vestigios de un sistema de sufijos casuales inspirado en las inscripciones: |
| - *-ka* (ergativo/agentivo): marcador de agente transitivo |
| - *-ar* (genitivo/asociativo): posesión y derivación agentiva |
| - *-en* (locativo/relacional): ubicación y adjetivos relacionales |
| - *-ta / tan* (ablativo/locativo): procedencia, ubicación estática""", |
|
|
| """**Raíz verbal.** Cada verbo tiene una raíz única e invariable. La conjugación es completamente regular y aglutinante: raíz + TAM + persona. |
| |
| **Sufijos TAM (tiempo-aspecto-modo):** |
| | TAM | Sufijo | Ejemplo (*niśnal* = hablar) | |
| |---|---|---| |
| | Presente | *-ke* | *niśnal-ke* (habla) | |
| | Pasado | *-bo* | *niśnal-bo* (habló) | |
| | Futuro | *-ta* | *niśnal-ta* (hablará) | |
| | Imperfecto | *-ska* | *niśnal-ska* (hablaba) | |
| | Condicional | *-tei* | *niśnal-tei* (hablaría) | |
| | Subjuntivo | *-na* | *niśnal-na* (hable) | |
| | Subj. imperfecto | *-nabo* | *niśnal-nabo* (hablara) | |
| | Imperativo | *-ka* | *niśnal-ka* (¡habla!) | |
| | Gerundio | *-min* | *niśnal-min* (hablando) | |
| | Participio | *-ir* | *niśnal-ir* (hablado) | |
| | Infinitivo | ∅ | *niśnal* (hablar) | |
| |
| **Sufijos de persona:** |
| | Persona | Sufijo | Ejemplo (PRS) | |
| |---|---|---| |
| | 1ª sing. | *-n* | *niśnal-ke-n* (hablo) | |
| | 2ª sing. | *-śe* | *niśnal-ke-śe* (hablas) | |
| | 3ª sing. | ∅ | *niśnal-ke* (habla) | |
| | 1ª pl. | *-ek* | *niśnal-ke-ek* (hablamos) | |
| | 2ª pl. | *-śek* | *niśnal-ke-śek* (habláis) | |
| | 3ª pl. | *-r* | *niśnal-ke-r* (hablan) | |
| |
| **Clíticos pronominales** (se añaden al imperativo en cadena): |
| *-nei* (me), *-sei* (te), *-śei* (se/reflexivo), *-guei* (nos), *-suei* (os), *-eki* (lo), *-eka* (la), *-ekik* (los), *-ekak* (las). |
| |
| Ejemplo: *saŕi-ka-śe-nei-eki* = llamar-IMP-2SG-1SG.DAT-3SG.ACC = 'llámamelo'. |
| |
| **Verbo pronominal:** infinitivo + *-śei*: *duŕkam-śei* (dormirse).""", |
|
|
| """El neoíbero genera familias de palabras mediante sufijos derivativos regulares aplicados a la raíz: |
| |
| | Sufijo | Función | Ejemplo | |
| |---|---|---| |
| | *-ar* | Agente / profesión | *tuŕtiśkal* (trabajador) | |
| | *-ist* | Especialista / -ista | *bomuś-ist* (florista) | |
| | *-la* | Lugar / establecimiento | *bomuś-ist-la* (floristería) | |
| | *-tun* | Acción / resultado | *tesekik-tun* (decisión) | |
| | *-ŕa* | Cualidad abstracta / -idad | *deśbiltik-ŕa* (realidad) | |
| | *-si* | Adjetival / cualidad | *deśbil-si* (real, adj.) | |
| | *-si-ka* | Adverbial / -mente | *deśbil-si-ka* (realmente) | |
| | *-en* | Relacional | uso locativo/adjetival | |
| |
| **Regla de oro:** primero se deriva la raíz, después se flexiona (género, número, persona, TAM). Ejemplo: *bomuś* (flor) → *bomuś-ist* (florista) → *bomuś-ist-la* (floristería) → *bomuś-ist-la-k* (floristerías). |
| |
| **Abstractos.** Para evitar colisiones entre adjetivos y nombres abstractos, el sistema productivo usa *-ŕa* como derivativo abstracto. Ej.: *kentikli* (indelicado, adj.) → *kentikli-ŕa* (indelicadeza) → *kentikli-ŕa-k* (indelicadezas).""", |
|
|
| """**Base vigesimal** inspirada en los numerales atestiguados en las inscripciones ibéricas y en paralelos vascos. Los diez primeros: |
| |
| | Nº | Neoíbero | | Nº | Neoíbero | |
| |---|---|---|---|---| |
| | 1 | *ban* | | 6 | *sei* | |
| | 2 | *bi* | | 7 | *sisbi* | |
| | 3 | *irur* | | 8 | *sorse* | |
| | 4 | *laur* | | 9 | *bedar* | |
| | 5 | *borste* | | 10 | *abar* | |
| |
| **Decenas:** 20 = *orkei*, 30 = *orkei-abar*, 40 = *binorkei*, 50 = *binorkei-abar*, 60 = *irurokei*, 100 = *atun*, 1000 = *bekśon*. |
| |
| **Composición:** decena + *-ke-* + unidad. Ej.: 45 = *binorkei-ke-borste*, 93 = *laurokei-abar-ke-irur*.""", |
|
|
| """**Orden.** El neoíbero es predominantemente SOV (sujeto-objeto-verbo), coherente con las lenguas de caso como el vasco y consistente con las hipótesis sobre el íbero. |
| |
| **Partículas y conectores:** |
| | Español | Neoíbero | Función | |
| |---|---|---| |
| | y | *ne* | Conjunción copulativa | |
| | o | *o* | Conjunción disyuntiva | |
| | de | *ta* | Genitivo / ablativo | |
| | en | *tan* | Locativo | |
| | con | *kin* | Comitativo | |
| | para | *kara* | Finalidad | |
| | no | *eś* | Negación | |
| |
| **Ejemplo:** *eno nulgarser ituśder-ke* = el perro bueno-es = 'el perro es bueno'.""", |
|
|
| """Sistema inspirado en la tipología vasca de partículas oracionales finales. En neoíbero: |
| - **-na** (sufijado a la última palabra de la oración): marca **interrogación**. Al traducir a español, se reconstruyen los signos ¿…? |
| - **-ba**: marca **exclamación**. Se reconstruyen ¡…! |
| |
| En dirección ES→NI, los signos ¿? y ¡! se eliminan y se sustituyen por el sufijo modal correspondiente. En NI→ES, el sufijo se elimina y se insertan los signos ortográficos españoles. |
| |
| Ejemplo: *niśnal-ke-na* → ¿Habla? / ¿Habla? → *niśnal-ke-na*""", |
|
|
| """**Motor de traducción.** El traductor opera en modo **BI-estricto 1:1**: cada superficie española y neoíbera se resuelve contra un CSV comprimido de superficies exactas. En **NI→ES** la traducción sigue siendo estrictamente determinista. En **ES→NI** se añade una capa ligera de desambiguación contextual para homógrafos nombre/verbo e infinitivos aislados (por ejemplo, artículos, pronombres de sujeto, negación, preposiciones típicas de infinitivo y posición tras puntuación). No hay análisis sintáctico profundo ni parser: la traducción sigue siendo de sustitución superficial con un filtro contextual mínimo. |
| |
| **Léxico.** ~270.000 pares ES↔NI generados mediante un pipeline que: (1) lee el diccionario de la RAE; (2) genera raíces NI únicas por hash determinista con anti-colisión; (3) conjuga 652 verbos con sus ~60 formas cada uno; (4) cierra paradigmas flexivos (género, plural único en *-k*, familias derivativas y abstractos en *-ŕa*); (5) verifica inyectividad 1:1 en ambas direcciones. |
| |
| **Códigos de diagnóstico:** |
| - `[SIN-LEX:palabra]` — palabra no encontrada en el léxico |
| - `[AMB-NI:forma]` — forma NI ambigua (bloqueada en modo estricto) |
| - `[?:forma]` — forma NI sin correspondencia ES""", |
|
|
| """**Sobre el íbero histórico.** El conocimiento actual del íbero procede de ~2.000 inscripciones en escritura paleohispánica, la mayoría breves (plomos, cerámicas, monedas). Se conoce su fonología, su escritura y un puñado de morfemas recurrentes, pero la lengua permanece sin descifrar. Las principales referencias: |
| |
| - **Jürgen Untermann** — *Monumenta Linguarum Hispanicarum* (1975–2000). Corpus fundamental de inscripciones paleohispánicas. |
| - **Javier de Hoz** — *Historia lingüística de la Península Ibérica en la Antigüedad* (2010–2011). Síntesis magistral del paisaje lingüístico prerromano. |
| - **Joan Ferrer i Jané** — Trabajos sobre el sistema dual de la escritura ibérica y desciframiento de numerales. Autor de la fuente tipográfica Iberia-Georgeos usada en esta aplicación. |
| - **José Antonio Correa** — Estudios sobre la escritura del sudoeste y las relaciones ibérico-tartesias. |
| - **Jesús Rodríguez Ramos** — Análisis morfológico y segmentación de textos ibéricos. |
| - **Javier Velaza** — Epigrafía ibérica y celtibérica, clasificación textual. |
| - **Eduardo Orduña** — Investigación sobre los numerales ibéricos y paralelos vascoibéricos. |
| |
| **Sobre el neoíbero.** Conlang diseñada y desarrollada por **David García** (2024–2026). Asistencia en generación léxica, depuración de pipeline y edición a través de scripts. El proyecto es de naturaleza artística y especulativa, no filológica: no pretende reconstruir el íbero real, sino imaginar cómo podría sonar una lengua heredera.""", |
|
|
| """| Sigla | Significado | |
| |---|---| |
| | ES | Español (superficie) | |
| | NI | Neoíbero (superficie) | |
| | TAM | Tiempo-Aspecto-Modo | |
| | PRS | Presente | |
| | PST | Pasado (perfecto) | |
| | FUT | Futuro | |
| | IPFV | Imperfecto | |
| | COND | Condicional | |
| | SBJ | Subjuntivo | |
| | SBJ.IPFV | Subjuntivo imperfecto | |
| | IMP | Imperativo | |
| | GER | Gerundio | |
| | PART | Participio | |
| | INF | Infinitivo | |
| | SG | Singular | |
| | PL | Plural | |
| | M | Masculino | |
| | F | Femenino | |
| | 1S/2S/3S | 1ª/2ª/3ª persona singular | |
| | 1P/2P/3P | 1ª/2ª/3ª persona plural | |
| | BI | Bilingüe (modo de CSV) | |
| | 1:1 | Correspondencia inyectiva (una superficie → una traducción) |""" |
| ], |
| "EN":[ |
| """**Neo-Iberian** is a constructed language (conlang) inspired by ancient Iberian, the pre-Roman language spoken along the eastern coast of the Iberian Peninsula (5th–1st c. BCE). While historical Iberian remains largely undeciphered, Neo-Iberian reconstructs a coherent and learnable linguistic system from what we do know: its phonotactics, semi-syllabic script, recurring morphemes in inscriptions, and documented contacts with other Mediterranean languages. |
| |
| Designed by David García (2024–2026). Iberian typeface: Iberia-Georgeos by Joan Ferrer i Jané.""", |
|
|
| """Consonants: *b, d, t, g, k* (no surface /p/: p→b); fricatives *s, ś*; nasals *m, n*; liquids *l, r, ŕ*. Five vowels: *a, e, i, o, u*. Syllable structure: CV(C). The hyphen (-) separates morphemes: *aŕen-ke-n* = be-PRS-1SG = 'I am'.""", |
|
|
| """Two genders (M base, F *-a*) and a single productive plural marker: *-k* for nouns and adjectives. Determiners: *eno* (the.M), *ena* (the.F), *enos* (the.M.PL), *enas* (the.F.PL). Case vestiges: *-ka* (ERG), *-ar* (GEN), *-en* (LOC), *-ta/tan* (ABL/LOC).""", |
|
|
| """Fully regular agglutinative conjugation: root + TAM + person. TAM: PRS *-ke*, PST *-bo*, FUT *-ta*, IPFV *-ska*, COND *-tei*, SBJ *-na*, SBJ.IPFV *-nabo*, IMP *-ka*, GER *-min*, PART *-ir*. Person: 1S *-n*, 2S *-śe*, 3S ∅, 1P *-ek*, 2P *-śek*, 3P *-r*. Pronominal clitics chain onto imperatives: *-nei* (me), *-sei* (you), *-śei* (refl.), *-eki* (him/it), *-eka* (her).""", |
|
|
| """Derivational suffixes: *-ar* (agent), *-ist* (-ist), *-la* (place), *-tun* (action), *-ŕa* (abstract quality), *-si* (adjectival), *-si-ka* (adverbial). Derivation precedes inflection. Abstract nouns systematically use *-ŕa* to stay distinct from adjectives, e.g. *kentikli* → *kentikli-ŕa* → *kentikli-ŕa-k*.""", |
|
|
| """Vigesimal base. 1–10: *ban, bi, irur, laur, borste, sei, sisbi, sorse, bedar, abar*. 20 = *orkei*, 100 = *atun*, 1000 = *bekśon*. Composition: decade + *-ke-* + unit.""", |
|
|
| """Predominantly SOV. Key particles: *ne* (and), *o* (or), *ta* (of), *tan* (in), *kin* (with), *kara* (for), *eś* (not).""", |
|
|
| """*-na* on the last word of the sentence marks interrogation; *-ba* marks exclamation. ES→NI strips ¿?¡! and adds the suffix. NI→ES reverses the process.""", |
|
|
| """Strict 1:1 bilingual mode. ~270,000 ES↔NI pairs. Deterministic surface substitution, no runtime morphology. Diagnostics: `[SIN-LEX:…]` (missing), `[AMB-NI:…]` (ambiguous), `[?:…]` (unknown NI).""", |
|
|
| """Core references: Untermann (*MLH*, 1975–2000); de Hoz (*Historia lingüística*, 2010–11); Ferrer i Jané (dual script, numerals); Correa (southwestern script); Rodríguez Ramos (morphological analysis); Velaza (epigraphy); Orduña (Iberian numerals). |
| |
| Conlang by **David García** (2024–2026).""", |
|
|
| """ES = Spanish surface | NI = Neo-Iberian surface | TAM = Tense-Aspect-Mood | PRS/PST/FUT/IPFV/COND/SBJ/IMP/GER/PART = TAM values | 1S/2S/3S/1P/2P/3P = person | BI = bilingual mode | 1:1 = injective mapping.""" |
| ] |
| } |
|
|
| |
| def build_css(): |
| b64=None |
| if os.path.exists("Iberia-Georgeos.ttf"): |
| with open("Iberia-Georgeos.ttf","rb") as f: |
| b64=base64.b64encode(f.read()).decode("ascii") |
| font_src = f"url(data:font/ttf;base64,{b64}) format('truetype')" if b64 else "local('sans-serif')" |
| return f""" |
| @font-face {{ |
| font-family: 'IberiaGeorgeos'; |
| src: {font_src}; |
| font-weight: normal; font-style: normal; |
| }} |
| :root {{ |
| --iberian-clay:#8B4513; --iberian-ochre:#CC7722; --iberian-stone:#5C5C5C; |
| --iberian-sand:#D2B48C; --iberian-rust:#A0522D; --iberian-bronze:#CD7F32; |
| }} |
| .gradio-container {{ background:linear-gradient(135deg,#f4e8d8 0%,#e8d5c4 50%,#d4c4b0 100%)!important; |
| font-family:'Georgia','Times New Roman',serif!important; }} |
| .gradio-container h1, .gradio-container h2, .gradio-container h3 {{ |
| color:var(--iberian-clay)!important; text-shadow:2px 2px 4px rgba(139,69,19,.15)!important; |
| border-bottom:3px solid var(--iberian-bronze)!important; padding-bottom:.5rem!important; letter-spacing:.5px!important; |
| }} |
| .gradio-container .gr-group {{ background:linear-gradient(to bottom,#f9f6f0,#ede6dc)!important; |
| border:2px solid var(--iberian-sand)!important; border-radius:8px!important; box-shadow:0 4px 12px rgba(139,69,19,.2), inset 0 1px 0 rgba(255,255,255,.5)!important; |
| padding:1.5rem!important; margin-bottom:0.2rem!important; }} |
| .gradio-container .gr-accordion {{ background:linear-gradient(145deg,#ebe3d5,#d9cec0)!important; |
| border:2px solid var(--iberian-rust)!important; border-radius:6px!important; margin-bottom:.8rem!important; box-shadow:2px 2px 6px rgba(0,0,0,.15)!important; }} |
| .gradio-container .gr-accordion .label-wrap {{ background:linear-gradient(to right,var(--iberian-ochre),var(--iberian-rust))!important; |
| color:#fff!important; font-weight:600!important; padding:.8rem 1rem!important; border-radius:4px!important; text-shadow:1px 1px 2px rgba(0,0,0,.3)!important; }} |
| .gradio-container .gr-textbox textarea, .gradio-container .gr-textbox input {{ background:linear-gradient(to bottom,#faf8f3,#f5f0e8)!important; |
| border:2px solid var(--iberian-sand)!important; border-radius:6px!important; color:var(--iberian-stone)!important; |
| font-family:'Georgia',serif!important; box-shadow:inset 2px 2px 4px rgba(139,69,19,.1)!important; }} |
| .gradio-container .gr-textbox textarea:focus, .gradio-container .gr-textbox input:focus {{ |
| border-color:var(--iberian-bronze)!important; box-shadow:inset 2px 2px 4px rgba(139,69,19,.1), 0 0 8px rgba(205,127,50,.3)!important; }} |
| .gradio-container .gr-button.gr-button-primary {{ background:linear-gradient(145deg,var(--iberian-bronze),var(--iberian-rust))!important; |
| border:2px solid var(--iberian-clay)!important; color:#fff!important; font-weight:bold!important; text-shadow:1px 2px 2px rgba(0,0,0,.4)!important; |
| box-shadow:0 4px 8px rgba(139,69,19,.3), inset 0 1px 0 rgba(255,255,255,.2)!important; border-radius:8px!important; padding:.8rem 1.5rem!important; transition:all .3s ease!important; }} |
| .gradio-container .gr-button.gr-button-primary:hover {{ background:linear-gradient(145deg,var(--iberian-rust),var(--iberian-bronze))!important; |
| transform:translateY(-2px)!important; box-shadow:0 6px 12px rgba(139,69,19,.4)!important; }} |
| .ib-line {{ font-family:'IberiaGeorgeos',monospace,sans-serif!important; font-size:1.9rem!important; line-height:2.4rem!important; white-space:pre-wrap!important; |
| background:linear-gradient(135deg,#e8dcc8 0%,#d4c4a8 50%,#c4b098 100%)!important; padding:24px!important; border-radius:10px!important; |
| border:3px solid var(--iberian-rust)!important; border-left:6px solid var(--iberian-bronze)!important; |
| box-shadow:0 4px 15px rgba(139,69,19,.25), inset 0 2px 4px rgba(0,0,0,.1)!important; color:var(--iberian-clay)!important; position:relative!important; }} |
| .ib-line::before {{ content:''!important; position:absolute!important; inset:0!important; |
| background-image:repeating-linear-gradient(0deg,transparent,transparent 2px, rgba(139,69,19,.03) 2px, rgba(139,69,19,.03) 4px)!important; |
| pointer-events:none!important; border-radius:10px!important; }} |
| @media (max-width:768px) {{ |
| .ib-line {{ font-size:1.5rem!important; line-height:2rem!important; padding:16px!important; }} |
| .gradio-container .gr-group {{ padding:1rem!important; }} |
| .gradio-container h1 {{ font-size:1.8rem!important; }} |
| }} |
| @media (max-width:480px) {{ |
| .ib-line {{ font-size:1.3rem!important; line-height:1.8rem!important; padding:12px!important; }} |
| .gradio-container h1 {{ font-size:1.5rem!important; }} |
| }} |
| /* ==== Estilos para las pestañas principales (Traductor/Mapa) ==== */ |
| /* Selector más general pero efectivo para los botones de tabs */ |
| .gradio-container button[role="tab"] {{ |
| background:linear-gradient(145deg,#ebe3d5,#d9cec0)!important; |
| border:2px solid var(--iberian-sand)!important; |
| border-bottom:none!important; |
| color:var(--iberian-clay)!important; |
| font-weight:600!important; |
| font-family:'Georgia','Times New Roman',serif!important; |
| font-size:1.05rem!important; |
| padding:0.8rem 2rem!important; |
| margin:0 0.3rem 0 0!important; |
| border-radius:8px 8px 0 0!important; |
| transition:all .25s ease!important; |
| box-shadow:2px 2px 6px rgba(0,0,0,.12)!important; |
| text-shadow:1px 1px 2px rgba(139,69,19,.08)!important; |
| }} |
| .gradio-container button[role="tab"]:hover {{ |
| background:linear-gradient(145deg,var(--iberian-ochre),#CC7722)!important; |
| color:#ffffff!important; |
| transform:translateY(-3px)!important; |
| box-shadow:0 5px 10px rgba(139,69,19,.25)!important; |
| text-shadow:1px 1px 3px rgba(0,0,0,.3)!important; |
| }} |
| .gradio-container button[role="tab"][aria-selected="true"] {{ |
| background:linear-gradient(145deg,var(--iberian-bronze),var(--iberian-rust))!important; |
| border:3px solid var(--iberian-clay)!important; |
| border-bottom:none!important; |
| color:#ffffff!important; |
| font-weight:700!important; |
| box-shadow:0 6px 12px rgba(139,69,19,.35), inset 0 1px 0 rgba(255,255,255,.25)!important; |
| text-shadow:1px 2px 3px rgba(0,0,0,.45)!important; |
| transform:translateY(0px)!important; |
| }} |
| /* Barra contenedora de tabs con fondo */ |
| .gradio-container div[role="tablist"] {{ |
| background:linear-gradient(145deg,#e8dcc8,#d9c4b0)!important; |
| border-bottom:4px solid var(--iberian-bronze)!important; |
| padding:0.5rem 1rem 0 1rem!important; |
| border-radius:10px 10px 0 0!important; |
| box-shadow:0 2px 8px rgba(139,69,19,.15)!important; |
| }} |
| """ |
| CSS = build_css() |
|
|
| |
| def _load_map_html() -> str: |
| for cand in ("mapa_iberos_neoibero.html", "salida/mapa_iberos_neoibero.html"): |
| if os.path.exists(cand): |
| with open(cand, "r", encoding="utf-8") as f: |
| return f.read() |
| |
| return """<!doctype html><meta charset=utf-8> |
| <title>Mapa</title> |
| <style>html,body,#m{height:100%;margin:0}#m{height:100vh}</style> |
| <link rel=stylesheet href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"> |
| <div id=m></div> |
| <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script> |
| <script>var map=L.map('m').setView([40,-2],6); |
| L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{maxZoom:18,attribution:'© OpenStreetMap'}).addTo(map); |
| L.circle([39,-0.3],{radius:70000}).addTo(map);</script>""" |
|
|
| MAP_SRC = _load_map_html() |
| MAP_DATA_URL = "data:text/html;base64," + base64.b64encode(MAP_SRC.encode("utf-8")).decode("ascii") |
|
|
| |
| with gr.Blocks(css=CSS, theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="purple")) as demo: |
| with gr.Group(): |
| title = gr.Markdown(f"# {LABELS['ES']['title']}") |
| subtitle = gr.Markdown(f"*{LABELS['ES']['subtitle']}*") |
|
|
| with gr.Group(): |
| with gr.Row(): |
| combo = gr.Dropdown(choices=["ES","EN"], value="ES", label=LABELS["ES"]["combo"]) |
| direction = gr.Radio(choices=LABELS["ES"]["dir_opts"], value="ES → NI", label=LABELS["ES"]["dir"]) |
|
|
| with gr.Group(): |
| es_in = gr.Textbox(label=LABELS["ES"]["in_label_es"], placeholder=LABELS["ES"]["in_ph_es"], lines=5) |
| with gr.Row(): |
| btn_tr = gr.Button(LABELS["ES"]["btn"], variant="primary") |
| btn_diag = gr.Button("🔎 Diagnosticar BI con este texto", variant="secondary") |
| with gr.Row(): |
| with gr.Column(scale=2): |
| ni_out = gr.Textbox(label=LABELS["ES"]["out_lat_esni"], lines=5, interactive=False) |
| loc_btn = gr.Button("🔊 Locutar", variant="secondary", visible=True) |
| audio_out = gr.Audio(label=LABELS["ES"]["out_audio"], type="numpy") |
| with gr.Column(scale=1): |
| ib_out = gr.HTML(label=LABELS["ES"]["out_ib"]) |
| diag_out = gr.HTML(value="") |
|
|
| with gr.Group(): |
| doc_header = gr.Markdown(f"## {LABELS['ES']['doc_header']}") |
| acc_titles = LABELS["ES"]["acc_titles"] |
| with gr.Accordion(acc_titles[0], open=False) as acc1: md1 = gr.Markdown(DOC["ES"][0]) |
| with gr.Accordion(acc_titles[1], open=False) as acc2: md2 = gr.Markdown(DOC["ES"][1]) |
| with gr.Accordion(acc_titles[2], open=False) as acc3: md3 = gr.Markdown(DOC["ES"][2]) |
| with gr.Accordion(acc_titles[3], open=False) as acc4: md4 = gr.Markdown(DOC["ES"][3]) |
| with gr.Accordion(acc_titles[4], open=False) as acc5: md5 = gr.Markdown(DOC["ES"][4]) |
| with gr.Accordion(acc_titles[5], open=False) as acc6: md6 = gr.Markdown(DOC["ES"][5]) |
| with gr.Accordion(acc_titles[6], open=False) as acc7: md7 = gr.Markdown(DOC["ES"][6]) |
| with gr.Accordion(acc_titles[7], open=False) as acc8: md8 = gr.Markdown(DOC["ES"][7]) |
| with gr.Accordion(acc_titles[8], open=False) as acc9: md9 = gr.Markdown(DOC["ES"][8]) |
| with gr.Accordion(acc_titles[9], open=False) as acc10: md10 = gr.Markdown(DOC["ES"][9]) |
| with gr.Accordion(acc_titles[10], open=False) as acc11: md11 = gr.Markdown(DOC["ES"][10]) |
| with gr.Accordion("🧪 Diagnóstico del CSV BI (al cargar)", open=False): |
| bi_diag_box = gr.HTML(value=BI_DIAG_HTML) |
|
|
| def do_translate(text, dir_label): |
| if not text or not text.strip(): |
| return (gr.update(value=""), |
| gr.update(value="<div class='ib-line'></div>"), |
| gr.update(visible=False), |
| gr.update(value=None), |
| gr.update(value="")) |
| if dir_label.startswith("ES"): |
| latin, ib = translate_es_to_ni_bi(text) |
| return (gr.update(label=LABELS["ES"]["out_lat_esni"], value=latin), |
| gr.update(value=ib), |
| gr.update(visible=True), |
| gr.update(value=None), |
| gr.update(value="")) |
| else: |
| es_text = translate_ni_to_es_bi(text) |
| return (gr.update(label=LABELS["ES"]["out_lat_nies"], value=es_text), |
| gr.update(value="<div class='ib-line'></div>"), |
| gr.update(visible=False), |
| gr.update(value=None), |
| gr.update(value="")) |
|
|
| btn_tr.click(do_translate, [es_in, direction], [ni_out, ib_out, loc_btn, audio_out, diag_out]) |
|
|
| def run_locution(latin_text, dir_label): |
| if dir_label.startswith("ES"): |
| return synthesize_speech(latin_text) |
| return None |
| loc_btn.click(run_locution, [ni_out, direction], audio_out) |
|
|
| def do_diagnose(text, dir_label): |
| return gr.update(value=diagnose_text(text, dir_label)) |
| btn_diag.click(do_diagnose, [es_in, direction], [diag_out]) |
|
|
| def switch_lang(sel_lang, dir_label): |
| L=LABELS[sel_lang]; T=L["acc_titles"]; D=DOC[sel_lang] |
| in_label = L["in_label_es"] if dir_label.startswith("ES") else L["in_label_ni"] |
| in_ph = L["in_ph_es"] if dir_label.startswith("ES") else L["in_ph_ni"] |
| out_lab = L["out_lat_esni"] if dir_label.startswith("ES") else L["out_lat_nies"] |
| return ( |
| gr.update(value=f"# {L['title']}"), |
| gr.update(value=f"*{L['subtitle']}*"), |
| gr.update(label=L["combo"], value=sel_lang), |
| gr.update(label=L["dir"], choices=L["dir_opts"], value=dir_label), |
| gr.update(value=f"## {L['doc_header']}"), |
| gr.update(label=T[0]), gr.update(value=D[0]), |
| gr.update(label=T[1]), gr.update(value=D[1]), |
| gr.update(label=T[2]), gr.update(value=D[2]), |
| gr.update(label=T[3]), gr.update(value=D[3]), |
| gr.update(label=T[4]), gr.update(value=D[4]), |
| gr.update(label=T[5]), gr.update(value=D[5]), |
| gr.update(label=T[6]), gr.update(value=D[6]), |
| gr.update(label=T[7]), gr.update(value=D[7]), |
| gr.update(label=T[8]), gr.update(value=D[8]), |
| gr.update(label=T[9]), gr.update(value=D[9]), |
| gr.update(label=T[10]), gr.update(value=D[10]), |
| gr.update(label=in_label, placeholder=in_ph), |
| gr.update(label=out_lab), |
| gr.update(label=L["out_ib"]), |
| gr.update(label=L["out_audio"]), |
| gr.update(value=L["btn"]) |
| ) |
| combo.change( |
| switch_lang, |
| [combo, direction], |
| [title, subtitle, combo, direction, doc_header, |
| acc1, md1, acc2, md2, acc3, md3, acc4, md4, acc5, md5, acc6, md6, acc7, md7, acc8, md8, acc9, md9, acc10, md10, acc11, md11, |
| es_in, ni_out, ib_out, audio_out, btn_tr] |
| ) |
|
|
| def switch_direction(dir_label, sel_lang): |
| L=LABELS[sel_lang] |
| in_label = L["in_label_es"] if dir_label.startswith("ES") else L["in_label_ni"] |
| in_ph = L["in_ph_es"] if dir_label.startswith("ES") else L["in_ph_ni"] |
| out_lab = L["out_lat_esni"] if dir_label.startswith("ES") else L["out_lat_nies"] |
| loc_vis = True if dir_label.startswith("ES") else False |
| return (gr.update(label=in_label, placeholder=in_ph), |
| gr.update(label=out_lab, value=""), |
| gr.update(value="<div class='ib-line'></div>"), |
| gr.update(visible=loc_vis), |
| gr.update(value=None), |
| gr.update(value="")) |
| direction.change( |
| switch_direction, |
| [direction, combo], |
| [es_in, ni_out, ib_out, loc_btn, audio_out, diag_out] |
| ) |
|
|
| |
| def _symmetry_smoketest(): |
| print("\n[SMOKE] Prueba ES↔NI (BI-estricto, determinista)…") |
| probes = [ |
| "nuker-ke ni etxe-ka ?", |
| "¿Pagaste 12,75 en la cafetería?", |
| "Marta llega a las 18:30.", |
| "[SIN-LEX:Tomás]-na euŕak-ke !" |
| ] |
| for p in probes: |
| es_from_ni = translate_ni_to_es_bi(p) |
| ni_round, _ = translate_es_to_ni_bi(es_from_ni) |
| print(" IN:", p) |
| print(" ES:", es_from_ni) |
| print(" NI:", ni_round) |
| print("---") |
|
|
| if DEBUG_MODE: |
| _symmetry_smoketest() |
|
|
| if __name__ == "__main__": |
| demo.queue().launch() |
