#!/usr/bin/env python3 """Validate static website links, anchors, image assets, and JSON data. This is a local integrity check for the GitHub Pages / Hugging Face static website. It intentionally does not fetch external URLs; it verifies that the published local surface is internally coherent. """ from __future__ import annotations import argparse import json import re from dataclasses import dataclass from datetime import datetime, timezone from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urlsplit from PIL import Image ROOT = Path(__file__).resolve().parents[1] DEFAULT_DOCS = ROOT / "docs" DEFAULT_OUTPUT = DEFAULT_DOCS / "data/website_integrity.json" DEFAULT_SITE_BASE = "/ropedia-xperience-10m-task-suite/" LOCAL_ATTRS = {"href", "src"} SKIP_SCHEMES = {"http", "https", "mailto", "tel", "data", "javascript"} IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp"} @dataclass class Reference: source: Path tag: str attr: str raw: str path_part: str fragment: str class SiteParser(HTMLParser): def __init__(self, source: Path): super().__init__(convert_charrefs=True) self.source = source self.ids: list[str] = [] self.references: list[Reference] = [] self.images: list[str] = [] def handle_starttag(self, tag: str, attrs): attr_map = dict(attrs) element_id = attr_map.get("id") or attr_map.get("name") if element_id: self.ids.append(element_id) for attr in LOCAL_ATTRS: raw = attr_map.get(attr) if not raw: continue parsed = urlsplit(raw) if parsed.scheme in SKIP_SCHEMES or parsed.netloc: continue path_part = unquote(parsed.path) fragment = unquote(parsed.fragment) if not path_part and not fragment: continue self.references.append(Reference(self.source, tag, attr, raw, path_part, fragment)) if tag == "img" and attr == "src": self.images.append(path_part) def parse_html(path: Path) -> SiteParser: parser = SiteParser(path) parser.feed(path.read_text(encoding="utf-8", errors="ignore")) return parser def normalize_path_part(path_part: str, site_base: str) -> str: if path_part in {"", "/"}: return path_part normalized_base = "/" + site_base.strip("/") + "/" if path_part == normalized_base: return "/index.html" if path_part.startswith(normalized_base): return "/" + path_part[len(normalized_base):] return path_part def resolve_reference(docs_root: Path, source: Path, path_part: str, site_base: str) -> Path: path_part = normalize_path_part(path_part, site_base) if not path_part: return source base = docs_root if path_part.startswith("/") else source.parent resolved = (base / path_part.lstrip("/")).resolve() if resolved.is_dir(): return resolved / "index.html" return resolved def relative(path: Path, root: Path) -> str: try: return path.relative_to(root).as_posix() except ValueError: return path.as_posix() def duplicate_ids(ids: list[str]) -> list[dict]: counts: dict[str, int] = {} for item in ids: counts[item] = counts.get(item, 0) + 1 return [{"id": item, "count": count} for item, count in sorted(counts.items()) if count > 1] def image_record(path: Path, docs_root: Path) -> dict: suffix = path.suffix.lower() record = { "path": relative(path, docs_root), "exists": path.exists(), "bytes": path.stat().st_size if path.exists() else 0, } if not path.exists(): return record if suffix in IMAGE_SUFFIXES: with Image.open(path) as image: record.update({ "width": int(image.width), "height": int(image.height), "format": image.format, }) elif suffix == ".svg": text = path.read_text(encoding="utf-8", errors="ignore") viewbox = re.search(r'viewBox=["\']([^"\']+)["\']', text) record["format"] = "SVG" record["has_viewbox"] = bool(viewbox) return record def validate(docs_root: Path, site_base: str) -> dict: html_files = sorted(docs_root.glob("*.html")) parsers = {path: parse_html(path) for path in html_files} anchors_by_file = {path: set(parser.ids) for path, parser in parsers.items()} missing_targets = [] missing_anchors = [] local_references = [] external_reference_count = 0 image_paths: set[Path] = set() for path, parser in parsers.items(): text = path.read_text(encoding="utf-8", errors="ignore") external_reference_count += len(re.findall(r'https?://', text)) for ref in parser.references: target = resolve_reference(docs_root, path, ref.path_part, site_base) target_rel = relative(target, docs_root) source_rel = relative(ref.source, docs_root) local_references.append({ "source": source_rel, "tag": ref.tag, "attr": ref.attr, "raw": ref.raw, "target": target_rel, "fragment": ref.fragment, }) if not target.exists(): missing_targets.append({ "source": source_rel, "raw": ref.raw, "target": target_rel, }) continue if ref.tag == "img" and ref.attr == "src": image_paths.add(target) if ref.fragment: anchor_target = target if target.suffix.lower() == ".html" else path if ref.path_part and target.suffix.lower() != ".html": continue anchors = anchors_by_file.get(anchor_target) if anchors is None and anchor_target.exists() and anchor_target.suffix.lower() == ".html": anchors = set(parse_html(anchor_target).ids) anchors_by_file[anchor_target] = anchors if anchors is not None and ref.fragment not in anchors: missing_anchors.append({ "source": source_rel, "raw": ref.raw, "target": relative(anchor_target, docs_root), "fragment": ref.fragment, }) json_records = [] invalid_json = [] for path in sorted((docs_root / "data").glob("*.json")): rel_path = relative(path, docs_root) try: payload = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: invalid_json.append({"path": rel_path, "error": str(exc)}) continue json_records.append({ "path": rel_path, "bytes": path.stat().st_size, "top_level_type": type(payload).__name__, }) images = [] invalid_images = [] for path in sorted(image_paths): try: record = image_record(path, docs_root) images.append(record) if not record.get("exists") or record.get("bytes", 0) <= 0: invalid_images.append(record) if path.suffix.lower() in IMAGE_SUFFIXES and (record.get("width", 0) <= 0 or record.get("height", 0) <= 0): invalid_images.append(record) if path.suffix.lower() == ".svg" and not record.get("has_viewbox"): invalid_images.append(record) except Exception as exc: # noqa: BLE001 - report image validation failures. invalid_images.append({"path": relative(path, docs_root), "error": str(exc)}) duplicate_id_records = [ {"path": relative(path, docs_root), "duplicates": duplicate_ids(parser.ids)} for path, parser in parsers.items() ] duplicate_id_records = [item for item in duplicate_id_records if item["duplicates"]] semantic_checks = [] semantic_layout_failures = [] index_path = docs_root / "index.html" index_text = index_path.read_text(encoding="utf-8", errors="ignore") if index_path.exists() else "" suite_start = index_text.find('
') suite_end = index_text.find('
') suite_text = index_text[suite_start:suite_end] if suite_start >= 0 and suite_end > suite_start else "" semantic_rules = [ ( "suite_task_map_precedes_modality_atlas", '
', '
= 0 and after_pos >= 0 and marker_pos < after_pos detail = {"first_marker_index": marker_pos, "second_marker_index": after_pos} check = {"name": name, "status": "pass" if passed else "fail", "reason": reason, **detail} semantic_checks.append(check) if not passed: semantic_layout_failures.append(check) failures = { "missing_targets": missing_targets, "missing_anchors": missing_anchors, "duplicate_ids": duplicate_id_records, "invalid_json": invalid_json, "invalid_images": invalid_images, "semantic_layout": semantic_layout_failures, } failure_count = sum(len(items) for items in failures.values()) return { "status": "pass" if failure_count == 0 else "fail", "generated_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), "docs_root": str(docs_root), "site_base": site_base, "summary": { "html_pages": len(html_files), "local_references": len(local_references), "external_reference_count": external_reference_count, "json_files": len(json_records), "image_assets_referenced": len(images), "failure_count": failure_count, }, "failures": failures, "semantic_checks": semantic_checks, "html_pages": [ { "path": relative(path, docs_root), "id_count": len(parser.ids), "reference_count": len(parser.references), "image_count": len(parser.images), } for path, parser in parsers.items() ], "json_files": json_records, "images": images, } def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--docs-root", type=Path, default=DEFAULT_DOCS) parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) parser.add_argument("--site-base", default=DEFAULT_SITE_BASE) args = parser.parse_args() report = validate(args.docs_root.resolve(), args.site_base) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") print(f"{report['status'].upper()}: wrote {args.output}") if report["status"] != "pass": for kind, items in report["failures"].items(): for item in items[:20]: print(f"- {kind}: {item}") return 1 return 0 if __name__ == "__main__": raise SystemExit(main())