scripts/validate_utf8.py

#!/usr/bin/env python3
"""Validate that every .py file is plain UTF-8 and has no BOM/UTF-16 encoding.

This catches files like the old `scratch/old_product_views.py` which was saved
as UTF-16 LE with BOM and broke the Codacy security scan.

Usage:
    python scripts/validate_utf8.py
    python scripts/validate_utf8.py --path /path/to/repo

Exit codes:
    0 - all Python files are valid UTF-8 without BOM
    1 - one or more files are malformed
"""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

# Directories that should not be scanned
SKIP_DIRS = {
    ".git",
    ".venv",
    ".env",
    ".uv-cache",
    ".cache",
    "__pycache__",
    ".mypy_cache",
    ".pytest_cache",
    ".ruff_cache",
    ".tox",
    ".egg-info",
    "build",
    "dist",
    "node_modules",
}

# Byte markers that indicate BOMs
UTF8_BOM = b"\xef\xbb\xbf"
UTF16_LE_BOM = b"\xff\xfe"
UTF16_BE_BOM = b"\xfe\xff"


def should_skip(path: Path) -> bool:
    """Return True if the directory should be skipped."""
    return any(part in SKIP_DIRS for part in path.parts)


def validate_file(file_path: Path) -> list[str]:
    """Return a list of error messages for the file, or empty if valid."""
    errors: list[str] = []
    raw = file_path.read_bytes()

    # 1. Reject UTF-16 BOMs
    if raw.startswith(UTF16_LE_BOM):
        errors.append(f"{file_path}: UTF-16 LE BOM detected")
    elif raw.startswith(UTF16_BE_BOM):
        errors.append(f"{file_path}: UTF-16 BE BOM detected")
    elif raw.startswith(UTF8_BOM):
        errors.append(f"{file_path}: UTF-8 BOM detected")

    # 2. Reject null bytes
    if b"\x00" in raw:
        errors.append(f"{file_path}: contains null bytes")

    # 3. Validate strict UTF-8 decode
    if not errors:
        try:
            raw.decode("utf-8", errors="strict")
        except UnicodeDecodeError as exc:
            errors.append(f"{file_path}: {exc}")

    return errors


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Validate UTF-8 encoding for all .py files in a tree."
    )
    parser.add_argument(
        "--path",
        type=Path,
        default=Path.cwd(),
        help="Root directory to scan (default: current directory)",
    )
    parser.add_argument(
        "--skip",
        type=str,
        action="append",
        default=[],
        help="Additional directory names to skip (can be given multiple times)",
    )
    args = parser.parse_args()

    skip_dirs = SKIP_DIRS | set(args.skip)
    root: Path = args.path.resolve()

    all_errors: list[str] = []
    for py_path in root.rglob("*.py"):
        if should_skip(py_path):
            continue
        all_errors.extend(validate_file(py_path))

    if all_errors:
        print("Invalid encoding detected in the following files:", file=sys.stderr)
        for error in all_errors:
            print(f"  - {error}", file=sys.stderr)
        return 1

    print(f"Validated {len(list(root.rglob('*.py')))} Python files in {root}")
    print("All .py files are valid UTF-8 without BOM.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())