feat: support ALTO XML et PAGE XML comme GT dans l'upload de corpus

- Ajoute `_detect_xml_gt` qui détecte le format (ALTO/PAGE) et extrait le texte GT
- Ajoute `_extract_alto_text` : concatène les CONTENT des <String> par ligne
- Ajoute `_extract_page_text` : concatène les <Unicode> en ordre de lecture
- `_analyze_corpus_dir` reconnaît les paires image + .xml ; génère le .gt.txt
correspondant et indique le format dans chaque paire et le résumé global
- `_flatten_zip_to_dir` et l'endpoint upload acceptent désormais les .xml
- 11 nouveaux tests couvrant ALTO XML, PAGE XML, texte brut et XML inconnu

https://claude.ai/code/session_017gXea9mxBQqDTAsSQd7aAq

picarones/web/app.py

@@ -33,6 +33,7 @@ import shutil
 import threading
 import time
 import uuid
+import xml.etree.ElementTree as ET
 import zipfile
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
@@ -539,6 +540,76 @@ async def api_corpus_browse(path: str = Query(default=".", description="Chemin
 # API — corpus upload
 # ---------------------------------------------------------------------------
 
+def _detect_xml_gt(xml_bytes: bytes) -> tuple[str, str] | None:
+    """Détecte si xml_bytes est un fichier ALTO ou PAGE XML et extrait le texte GT.
+
+    Retourne (format_label, texte_gt) ou None si le format n'est pas reconnu.
+    """
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError:
+        return None
+
+    tag = root.tag  # peut être "{namespace}alto" ou "alto" ou "{ns}PcGts"
+
+    # --- ALTO XML ---
+    # Namespace contient loc.gov/standards/alto ou balise racine "alto"
+    ns_alto = "http://www.loc.gov/standards/alto"
+    is_alto = (
+        ns_alto in tag
+        or tag.lower() == "alto"
+        or (tag.startswith("{") and tag.split("}")[1].lower() in ("alto",))
+    )
+    if is_alto:
+        text = _extract_alto_text(root)
+        return ("ALTO XML", text)
+
+    # --- PAGE XML ---
+    # Balise racine PcGts (avec ou sans namespace)
+    local = tag.split("}")[-1] if "}" in tag else tag
+    if local == "PcGts":
+        text = _extract_page_text(root)
+        return ("PAGE XML", text)
+
+    return None
+
+
+def _extract_alto_text(root: ET.Element) -> str:
+    """Extrait le texte plein d'un arbre ALTO XML.
+
+    Concatène les attributs CONTENT des balises <String> dans l'ordre de lecture
+    (bloc → ligne → mot), avec un espace entre mots et une newline entre lignes.
+    """
+    # Chercher les éléments TextLine (avec ou sans namespace)
+    lines: list[str] = []
+    for elem in root.iter():
+        local = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag
+        if local == "TextLine":
+            words: list[str] = []
+            for child in elem.iter():
+                child_local = child.tag.split("}")[-1] if "}" in child.tag else child.tag
+                if child_local == "String":
+                    content = child.get("CONTENT", "")
+                    if content:
+                        words.append(content)
+            if words:
+                lines.append(" ".join(words))
+    return "\n".join(lines)
+
+
+def _extract_page_text(root: ET.Element) -> str:
+    """Extrait le texte plein d'un arbre PAGE XML.
+
+    Concatène le contenu des balises <Unicode> dans l'ordre de lecture.
+    """
+    texts: list[str] = []
+    for elem in root.iter():
+        local = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag
+        if local == "Unicode" and elem.text:
+            texts.append(elem.text.strip())
+    return "\n".join(t for t in texts if t)
+
+
 def _analyze_corpus_dir(path: Path) -> dict:
     """Analyse un dossier et retourne un résumé des paires image/GT détectées."""
     images = sorted(f.name for f in path.iterdir() if f.suffix.lower() in _IMAGE_EXTS)
@@ -546,11 +617,31 @@ def _analyze_corpus_dir(path: Path) -> dict:
     missing_gt: list[str] = []
     for img in images:
         stem = Path(img).stem
-        gt = path / (stem + ".gt.txt")
-        if gt.exists():
-            pairs.append({"image": img, "gt": stem + ".gt.txt"})
+        gt_txt = path / (stem + ".gt.txt")
+        gt_xml = path / (stem + ".xml")
+        if gt_txt.exists():
+            pairs.append({"image": img, "gt": stem + ".gt.txt", "gt_format": "texte brut"})
+        elif gt_xml.exists():
+            result = _detect_xml_gt(gt_xml.read_bytes())
+            if result is not None:
+                fmt, text = result
+                # Matérialiser le GT en .gt.txt pour le chargeur de corpus
+                gt_txt.write_text(text, encoding="utf-8")
+                pairs.append({"image": img, "gt": stem + ".gt.txt", "gt_format": fmt})
+            else:
+                missing_gt.append(img)
         else:
             missing_gt.append(img)
+
+    # Détecter le format dominant pour le résumé global
+    formats = {p["gt_format"] for p in pairs}
+    if len(formats) == 1:
+        dominant_format: str = formats.pop()
+    elif formats:
+        dominant_format = "mixte"
+    else:
+        dominant_format = "texte brut"
+
     return {
         "doc_count": len(pairs),
         "pairs": pairs[:20],
@@ -559,19 +650,20 @@ def _analyze_corpus_dir(path: Path) -> dict:
         "has_missing_gt": len(missing_gt) > 0,
         "warnings": [f"GT manquant : {img}" for img in missing_gt[:5]],
         "usable": len(pairs) > 0,
+        "gt_format": dominant_format,
     }
 
 
 def _flatten_zip_to_dir(zf: zipfile.ZipFile, dest: Path) -> None:
-    """Extrait un ZIP en aplatissant les paires image/.gt.txt dans dest."""
+    """Extrait un ZIP en aplatissant les paires image/.gt.txt/.xml dans dest."""
     dest.mkdir(parents=True, exist_ok=True)
     for member in zf.infolist():
         if member.is_dir():
             continue
         p = Path(member.filename)
         name = p.name
-        # Accepter images et .gt.txt
-        if p.suffix.lower() in _IMAGE_EXTS or name.endswith(".gt.txt"):
+        # Accepter images, .gt.txt et .xml (ALTO/PAGE)
+        if p.suffix.lower() in _IMAGE_EXTS or name.endswith(".gt.txt") or p.suffix.lower() == ".xml":
             data = zf.read(member.filename)
             (dest / name).write_bytes(data)
 
@@ -594,7 +686,7 @@ async def api_corpus_upload(files: list[UploadFile] = File(...)) -> dict:
                 import io
                 with zipfile.ZipFile(io.BytesIO(data)) as zf:
                     _flatten_zip_to_dir(zf, corpus_dir)
-            elif suffix in _IMAGE_EXTS or filename.endswith(".gt.txt") or suffix == ".txt":
+            elif suffix in _IMAGE_EXTS or filename.endswith(".gt.txt") or suffix in (".txt", ".xml"):
                 (corpus_dir / filename).write_bytes(data)
             # Ignorer les autres types
 

tests/test_sprint6_web_interface.py

@@ -1337,3 +1337,158 @@ class TestFastAPICorpusUpload:
         # corpus_id containing ".." (without slash — FastAPI strips slashes from path params)
         r = client.delete("/api/corpus/uploads/..malicious..")
         assert r.status_code in (400, 404)
+
+    # --- ALTO XML ---
+
+    @pytest.fixture
+    def alto_xml_bytes(self):
+        """Contenu d'un fichier ALTO XML minimal valide."""
+        return (
+            b'<?xml version="1.0" encoding="UTF-8"?>'
+            b'<alto xmlns="http://www.loc.gov/standards/alto/ns-v4#">'
+            b"<Layout><Page><PrintSpace>"
+            b"<TextBlock><TextLine>"
+            b'<String CONTENT="Bonjour"/>'
+            b'<String CONTENT="monde"/>'
+            b"</TextLine></TextBlock>"
+            b"</PrintSpace></Page></Layout>"
+            b"</alto>"
+        )
+
+    @pytest.fixture
+    def tmp_alto_zip(self, alto_xml_bytes):
+        """ZIP contenant une paire image + ALTO XML."""
+        import io
+        import zipfile
+
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as zf:
+            zf.writestr("page001.png", b"\x89PNG")
+            zf.writestr("page001.xml", alto_xml_bytes)
+        buf.seek(0)
+        return buf.getvalue()
+
+    def test_upload_alto_zip_returns_200(self, client, tmp_alto_zip):
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_alto_zip, "application/zip"))],
+        )
+        assert r.status_code == 200
+
+    def test_upload_alto_zip_doc_count(self, client, tmp_alto_zip):
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_alto_zip, "application/zip"))],
+        )
+        assert r.json()["doc_count"] == 1
+
+    def test_upload_alto_zip_format(self, client, tmp_alto_zip):
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_alto_zip, "application/zip"))],
+        )
+        d = r.json()
+        assert d["gt_format"] == "ALTO XML"
+        assert d["pairs"][0]["gt_format"] == "ALTO XML"
+
+    def test_upload_alto_individual_files(self, client, alto_xml_bytes):
+        files = [
+            ("files", ("img001.png", b"\x89PNG", "image/png")),
+            ("files", ("img001.xml", alto_xml_bytes, "application/xml")),
+        ]
+        r = client.post("/api/corpus/upload", files=files)
+        assert r.status_code == 200
+        assert r.json()["doc_count"] == 1
+        assert r.json()["gt_format"] == "ALTO XML"
+
+    def test_alto_text_extraction(self, alto_xml_bytes):
+        """_detect_xml_gt extrait correctement le texte depuis un ALTO XML."""
+        from picarones.web.app import _detect_xml_gt
+        result = _detect_xml_gt(alto_xml_bytes)
+        assert result is not None
+        fmt, text = result
+        assert fmt == "ALTO XML"
+        assert "Bonjour" in text
+        assert "monde" in text
+
+    # --- PAGE XML ---
+
+    @pytest.fixture
+    def page_xml_bytes(self):
+        """Contenu d'un fichier PAGE XML minimal valide."""
+        return (
+            b'<?xml version="1.0" encoding="UTF-8"?>'
+            b'<PcGts xmlns="http://schema.primaresearch.org/PAGE/gts/pagecontent/2013-07-15">'
+            b"<Page><TextRegion><TextLine>"
+            b"<TextEquiv><Unicode>Texte de la ligne</Unicode></TextEquiv>"
+            b"</TextLine></TextRegion></Page>"
+            b"</PcGts>"
+        )
+
+    @pytest.fixture
+    def tmp_page_zip(self, page_xml_bytes):
+        """ZIP contenant une paire image + PAGE XML."""
+        import io
+        import zipfile
+
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as zf:
+            zf.writestr("page002.png", b"\x89PNG")
+            zf.writestr("page002.xml", page_xml_bytes)
+        buf.seek(0)
+        return buf.getvalue()
+
+    def test_upload_page_zip_returns_200(self, client, tmp_page_zip):
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_page_zip, "application/zip"))],
+        )
+        assert r.status_code == 200
+
+    def test_upload_page_zip_format(self, client, tmp_page_zip):
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_page_zip, "application/zip"))],
+        )
+        d = r.json()
+        assert d["gt_format"] == "PAGE XML"
+        assert d["pairs"][0]["gt_format"] == "PAGE XML"
+
+    def test_page_text_extraction(self, page_xml_bytes):
+        """_detect_xml_gt extrait correctement le texte depuis un PAGE XML."""
+        from picarones.web.app import _detect_xml_gt
+        result = _detect_xml_gt(page_xml_bytes)
+        assert result is not None
+        fmt, text = result
+        assert fmt == "PAGE XML"
+        assert "Texte de la ligne" in text
+
+    # --- Texte brut ---
+
+    def test_upload_plain_txt_format_reported(self, client, tmp_corpus_zip):
+        """Un corpus .gt.txt classique doit indiquer 'texte brut' dans le résumé."""
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", tmp_corpus_zip, "application/zip"))],
+        )
+        assert r.status_code == 200
+        assert r.json()["gt_format"] == "texte brut"
+
+    # --- XML inconnu ignoré ---
+
+    def test_unknown_xml_not_valid_pair(self, client):
+        """Un XML non ALTO/PAGE ne crée pas de paire valide."""
+        import io
+        import zipfile
+
+        unknown_xml = b'<?xml version="1.0"?><root><item>foo</item></root>'
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as zf:
+            zf.writestr("pageX.png", b"\x89PNG")
+            zf.writestr("pageX.xml", unknown_xml)
+        buf.seek(0)
+        r = client.post(
+            "/api/corpus/upload",
+            files=[("files", ("corpus.zip", buf.getvalue(), "application/zip"))],
+        )
+        assert r.status_code == 422