#!/usr/bin/env python3
"""
EPUB Converter - Compiles translated HTML files into EPUB format
Supports extraction of translated titles from chapter content
"""
import os
import sys
import io
import json
import mimetypes
import re
import zipfile
import unicodedata
import html as html_module
from xml.etree import ElementTree as ET
from typing import Dict, List, Tuple, Optional, Callable
from ebooklib import epub, ITEM_DOCUMENT
from bs4 import BeautifulSoup
from metadata_batch_translator import enhance_epub_compiler
from concurrent.futures import ThreadPoolExecutor, as_completed
try:
from unified_api_client import UnifiedClient
except ImportError:
UnifiedClient = None
try:
from translate_headers_standalone import run_translation as run_standalone_header_translation
except ImportError:
run_standalone_header_translation = None
# Configure stdout for UTF-8
def configure_utf8_output():
"""Configure stdout for UTF-8 encoding"""
try:
if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(encoding='utf-8', errors='ignore')
except AttributeError:
if sys.stdout is None:
devnull = open(os.devnull, "wb")
sys.stdout = io.TextIOWrapper(devnull, encoding='utf-8', errors='ignore')
elif hasattr(sys.stdout, 'buffer'):
try:
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='ignore')
except:
pass
# Global configuration
configure_utf8_output()
_global_log_callback = None
_stop_flag = False
def set_stop_flag(value: bool):
"""Set the stop flag for EPUB converter"""
global _stop_flag
_stop_flag = value
def is_stop_requested() -> bool:
"""Check if stop has been requested"""
global _stop_flag
return _stop_flag
def set_global_log_callback(callback: Optional[Callable]):
"""Set the global log callback for module-level functions"""
global _global_log_callback
_global_log_callback = callback
def log(message: str):
"""Module-level logging that works with or without callback"""
if _global_log_callback:
_global_log_callback(message)
else:
print(message)
class HTMLEntityDecoder:
"""Handles comprehensive HTML entity decoding with full Unicode support"""
# Comprehensive entity replacement dictionary
ENTITY_MAP = {
# Quotation marks and apostrophes
'"': '"', '"': '"',
''': "'", ''': "'",
'‘': '\u2018', '’': '\u2019',
'“': '\u201c', '”': '\u201d',
'‚': '‚', '„': '„',
'‹': '‹', '›': '›',
'«': '«', '»': '»',
# Spaces and dashes
' ': ' ', ' ': ' ',
' ': ' ', ' ': ' ',
' ': ' ', '': '\u200c',
'': '\u200d', '': '\u200e',
'': '\u200f',
'–': '–', '—': '—',
'−': '−', '‐': '‐',
# Common symbols
'…': '…', '…': '…',
'•': '•', '•': '•',
'·': '·', '·': '·',
'§': '§', '¶': '¶',
'†': '†', '‡': '‡',
'◊': '◊', '♦': '♦',
'♣': '♣', '♥': '♥',
'♠': '♠',
# Currency symbols
'¢': '¢', '£': '£',
'¥': '¥', '€': '€',
'¤': '¤',
# Mathematical symbols
'±': '±', '×': '×',
'÷': '÷', '⁄': '⁄',
'‰': '‰', '‱': '‱',
'′': '\u2032', '″': '\u2033',
'∞': '∞', '∅': '∅',
'∇': '∇', '&partial;': '∂',
'∑': '∑', '∏': '∏',
'∫': '∫', '√': '√',
'≈': '≈', '≠': '≠',
'≡': '≡', '≤': '≤',
'≥': '≥', '⊂': '⊂',
'⊃': '⊃', '⊄': '⊄',
'⊆': '⊆', '⊇': '⊇',
# Intellectual property
'©': '©', '©': '©',
'®': '®', '®': '®',
'™': '™', '™': '™',
}
# Common encoding fixes
ENCODING_FIXES = {
# UTF-8 decoded as Latin-1
'’': "'", 'â€Å"': '"', '�': '"',
'–': '–', 'â€â€': '—',
' ': ' ', 'ÂÂ': '',
'â': 'â', 'é': 'é', 'è': 'è',
'ä': 'ä', 'ö': 'ö', 'ü': 'ü',
'ñ': 'ñ', 'ç': 'ç',
# Common mojibake patterns
'’': "'", '“': '"', 'â€': '"',
'â€"': '—', 'â€"': '–',
'…': '…', '•': '•',
'â„¢': '™', '©': '©', '®': '®',
# Windows-1252 interpreted as UTF-8
'‘': '\u2018', '’': '\u2019',
'“': '\u201c', 'â€': '\u201d',
'•': '•', 'â€"': '–', 'â€"': '—',
}
@classmethod
def decode(cls, text: str) -> str:
"""Comprehensive HTML entity decoding - PRESERVES UNICODE"""
if text is None:
return ""
if not isinstance(text, str):
text = str(text)
if not text:
return text
# Fix common encoding issues first
for bad, good in cls.ENCODING_FIXES.items():
text = text.replace(bad, good)
# Multiple passes to handle nested/double-encoded entities
max_passes = 3
for _ in range(max_passes):
prev_text = text
# Use html module for standard decoding (this handles <, >, etc.)
text = html_module.unescape(text)
if text == prev_text:
break
# Apply any remaining entity replacements
for entity, char in cls.ENTITY_MAP.items():
text = text.replace(entity, char)
return text
@staticmethod
def _decode_decimal(match):
"""Decode decimal HTML entity"""
try:
code = int(match.group(1))
if XMLValidator.is_valid_char_code(code):
return chr(code)
except:
pass
return match.group(0)
@staticmethod
def _decode_hex(match):
"""Decode hexadecimal HTML entity"""
try:
code = int(match.group(1), 16)
if XMLValidator.is_valid_char_code(code):
return chr(code)
except:
pass
return match.group(0)
class XMLValidator:
"""Handles XML validation and character checking"""
@staticmethod
def is_valid_char_code(codepoint: int) -> bool:
"""Check if a codepoint is valid for XML"""
return (
codepoint == 0x9 or
codepoint == 0xA or
codepoint == 0xD or
(0x20 <= codepoint <= 0xD7FF) or
(0xE000 <= codepoint <= 0xFFFD) or
(0x10000 <= codepoint <= 0x10FFFF)
)
@staticmethod
def is_valid_char(c: str) -> bool:
"""Check if a character is valid for XML"""
return XMLValidator.is_valid_char_code(ord(c))
@staticmethod
def clean_for_xml(text: str) -> str:
"""Remove invalid XML characters"""
return ''.join(c for c in text if XMLValidator.is_valid_char(c))
class ContentProcessor:
"""Handles content cleaning and processing - UPDATED WITH UNICODE PRESERVATION"""
@staticmethod
def safe_escape(text: str) -> str:
"""Escape XML special characters for use in XHTML titles/attributes"""
if text is None:
return ""
if not isinstance(text, str):
try:
text = str(text)
except Exception:
return ""
# Use html.escape to handle &, <, > and quotes; then escape single quotes
escaped = html_module.escape(text, quote=True)
escaped = escaped.replace("'", "'")
return escaped
class TitleExtractor:
"""Handles extraction of titles from HTML content - UPDATED WITH UNICODE PRESERVATION"""
@staticmethod
def extract_from_html(html_content: str, chapter_num: Optional[int] = None,
filename: Optional[str] = None, allow_paragraph_fallback: bool = True,
allow_generic_chapter_fallback: bool = True) -> Tuple[str, float]:
"""Extract title from HTML content with confidence score - KEEP ALL HEADERS INCLUDING NUMBERS"""
try:
# Decode entities first - PRESERVES UNICODE
html_content = HTMLEntityDecoder.decode(html_content)
soup = BeautifulSoup(html_content, 'lxml', from_encoding='utf-8')
candidates = []
# Strategy 1: tag (highest confidence)
title_tag = soup.find('title')
if title_tag and title_tag.string:
title_text = HTMLEntityDecoder.decode(title_tag.string.strip())
if title_text and len(title_text) > 0 and title_text.lower() not in ['untitled', 'chapter', 'document']:
candidates.append((title_text, 0.95, "title_tag"))
# Strategy 2: h1 tags (very high confidence)
h1_tags = soup.find_all('h1')
for i, h1 in enumerate(h1_tags[:3]): # Check first 3 h1 tags
text = HTMLEntityDecoder.decode(h1.get_text(strip=True))
if text and len(text) < 300:
# First h1 gets highest confidence
confidence = 0.9 if i == 0 else 0.85
candidates.append((text, confidence, f"h1_tag_{i+1}"))
# Strategy 3: h2 tags (high confidence)
h2_tags = soup.find_all('h2')
for i, h2 in enumerate(h2_tags[:3]): # Check first 3 h2 tags
text = HTMLEntityDecoder.decode(h2.get_text(strip=True))
if text and len(text) < 250:
# First h2 gets highest confidence among h2s
confidence = 0.8 if i == 0 else 0.75
candidates.append((text, confidence, f"h2_tag_{i+1}"))
# Strategy 4: h3 tags (moderate confidence)
h3_tags = soup.find_all('h3')
for i, h3 in enumerate(h3_tags[:3]): # Check first 3 h3 tags
text = HTMLEntityDecoder.decode(h3.get_text(strip=True))
if text and len(text) < 200:
confidence = 0.7 if i == 0 else 0.65
candidates.append((text, confidence, f"h3_tag_{i+1}"))
# Strategy 5: Bold text in first elements (lower confidence)
# If paragraph fallback is disabled, avoid using <p> tags as title sources
body_title_tags = ['div'] if not allow_paragraph_fallback else ['p', 'div']
first_elements = soup.find_all(body_title_tags)[:5]
for elem in first_elements:
for bold in elem.find_all(['b', 'strong'])[:2]: # Limit to first 2 bold items
bold_text = HTMLEntityDecoder.decode(bold.get_text(strip=True))
if bold_text and 2 <= len(bold_text) <= 150:
candidates.append((bold_text, 0.6, "bold_text"))
# Strategy 6: Center-aligned text (common for chapter titles)
center_tags = ['center', 'div'] if not allow_paragraph_fallback else ['center', 'div', 'p']
center_elements = soup.find_all(center_tags,
attrs={'align': 'center'}) or \
soup.find_all(center_tags if not allow_paragraph_fallback else ['div', 'p'],
style=lambda x: x and 'text-align' in x and 'center' in x)
for center in center_elements[:3]: # Check first 3 centered elements
text = HTMLEntityDecoder.decode(center.get_text(strip=True))
if text and 2 <= len(text) <= 200:
candidates.append((text, 0.65, "centered_text"))
# Strategy 7: All-caps text (common for titles in older books)
all_caps_tags = ['h1', 'h2', 'h3', 'div'] if not allow_paragraph_fallback else ['h1', 'h2', 'h3', 'p', 'div']
for elem in soup.find_all(all_caps_tags)[:10]:
text = elem.get_text(strip=True)
# Check if text is mostly uppercase
if text and len(text) > 2 and text.isupper():
decoded_text = HTMLEntityDecoder.decode(text)
# Keep it as-is (don't convert to title case automatically)
candidates.append((decoded_text, 0.55, "all_caps_text"))
# Strategy 8: Patterns in first paragraph (optional)
if allow_paragraph_fallback:
first_p = soup.find('p')
if first_p:
p_text = HTMLEntityDecoder.decode(first_p.get_text(strip=True))
# Look for "Chapter X: Title" patterns
chapter_pattern = re.match(
r'^(Chapter\s+[\dIVXLCDM]+\s*[:\-\u2013\u2014]\s*)(.{2,100})(?:\.|$)',
p_text, re.IGNORECASE
)
if chapter_pattern:
# Extract just the title part after "Chapter X:"
title_part = chapter_pattern.group(2).strip()
if title_part:
candidates.append((title_part, 0.8, "paragraph_pattern_title"))
# Also add the full "Chapter X: Title" as a lower confidence option
full_title = chapter_pattern.group(0).strip().rstrip('.')
candidates.append((full_title, 0.75, "paragraph_pattern_full"))
elif len(p_text) <= 100 and len(p_text) > 2:
# Short first paragraph might be the title
candidates.append((p_text, 0.4, "paragraph_standalone"))
# Strategy 9: Filename
if filename:
filename_match = re.search(r'response_\d+_(.+?)\.html', filename)
if filename_match:
filename_title = filename_match.group(1).replace('_', ' ').title()
if len(filename_title) > 2:
candidates.append((filename_title, 0.3, "filename"))
# Filter and rank candidates
if candidates:
unique_candidates = {}
for title, confidence, source in candidates:
# Clean the title but keep roman numerals and short titles
title = TitleExtractor.clean_title(title)
# Don't reject short titles (like "III", "IX") - they're valid!
if title and len(title) > 0:
# Don't apply is_valid_title check too strictly
# Roman numerals and chapter numbers are valid titles
if title not in unique_candidates or unique_candidates[title][1] < confidence:
unique_candidates[title] = (title, confidence, source)
if unique_candidates:
sorted_candidates = sorted(unique_candidates.values(), key=lambda x: x[1], reverse=True)
best_title, best_confidence, best_source = sorted_candidates[0]
# Log what we found for debugging (only if debug mode is enabled)
import os
debug_mode_enabled = os.environ.get('DEBUG_MODE', '0') == '1'
if debug_mode_enabled:
log(f"[DEBUG] Best title candidate: '{best_title}' (confidence: {best_confidence:.2f}, source: {best_source})")
return best_title, best_confidence
# Fallback - only use generic chapter number if allowed and nothing found
if allow_generic_chapter_fallback and chapter_num:
return f"Chapter {chapter_num}", 0.1
return "Untitled Chapter", 0.0
except Exception as e:
log(f"[WARNING] Error extracting title: {e}")
if allow_generic_chapter_fallback and chapter_num:
return f"Chapter {chapter_num}", 0.1
return "Untitled Chapter", 0.0
@staticmethod
def clean_title(title: str) -> str:
"""Clean and normalize extracted title - trust HTML headers, minimal cleanup only"""
if not title:
return ""
# Decode HTML entities - PRESERVES UNICODE
title = HTMLEntityDecoder.decode(title)
# Remove HTML tags only
title = re.sub(r'<[^>]+>', '', title)
# Normalize whitespace (convert non-breaking spaces, multiple spaces, etc.)
title = re.sub(r'[\xa0\u2000-\u200a\u202f\u205f\u3000]+', ' ', title)
title = re.sub(r'\s+', ' ', title).strip()
# Normalize Unicode to NFC form
title = unicodedata.normalize('NFC', title)
# Remove invisible zero-width characters only
title = re.sub(r'[\u200b\u200c\u200d\u200e\u200f\ufeff]', '', title)
# Final whitespace cleanup
title = ' '.join(title.split())
# Truncate if excessively long (safety check only)
if len(title) > 200:
truncated = title[:197]
last_space = truncated.rfind(' ')
if last_space > 150:
truncated = truncated[:last_space]
title = truncated + "..."
return title
@staticmethod
def is_valid_title(title: str) -> bool:
"""Check if extracted title is valid - ACCEPT SHORT TITLES LIKE ROMAN NUMERALS"""
if not title:
return False
# Accept any non-empty title after cleaning
# Don't reject roman numerals or short titles
# Only reject truly invalid patterns
invalid_patterns = [
r'^untitled$', # Just "untitled"
r'^chapter$', # Just "chapter" without a number
r'^document$', # Just "document"
]
for pattern in invalid_patterns:
if re.match(pattern, title.lower().strip()):
return False
# Skip obvious filler phrases
filler_phrases = [
'click here', 'read more', 'continue reading', 'next chapter',
'previous chapter', 'table of contents', 'back to top'
]
title_lower = title.lower().strip()
if any(phrase in title_lower for phrase in filler_phrases):
return False
# Accept everything else, including roman numerals and short titles
return True
class XHTMLConverter:
"""Handles XHTML conversion and compliance"""
# Default language for generated XHTML (used for html[@lang] and html[@xml:lang])
# This will be synchronized with the EPUB book language by EPUBCompiler.
DEFAULT_LANG = "en"
@classmethod
def set_default_language(cls, lang_code: str):
"""Set default language code used for html lang/xml:lang attributes"""
if not lang_code:
return
try:
cls.DEFAULT_LANG = str(lang_code).strip() or "en"
except Exception:
cls.DEFAULT_LANG = "en"
@staticmethod
def ensure_compliance(html_content: str, title: str = "Chapter",
css_links: Optional[List[str]] = None) -> str:
"""Ensure HTML content is XHTML-compliant while PRESERVING story tags"""
try:
import html
import re
# Unescape HTML entities but PRESERVE < and > so fake angle brackets in narrative
# text don't become real tags (which breaks parsing across paragraphs like the sample).
if any(ent in html_content for ent in ['&', '"', '&#', '<', '>']):
# Temporarily protect < and > (both cases) from unescaping
placeholder_lt = '\ue000'
placeholder_gt = '\ue001'
html_content = html_content.replace('<', placeholder_lt).replace('<', placeholder_lt)
html_content = html_content.replace('>', placeholder_gt).replace('>', placeholder_gt)
# Unescape remaining entities
html_content = html.unescape(html_content)
# Restore protected angle bracket entities
html_content = html_content.replace(placeholder_lt, '<').replace(placeholder_gt, '>')
# Strip out ANY existing DOCTYPE, XML declaration, or html wrapper
# We only want the body content
# Try to extract just body content
body_match = re.search(r'<body[^>]*>(.*?)</body>', html_content, re.DOTALL | re.IGNORECASE)
if body_match:
html_content = body_match.group(1)
else:
# No body tags, strip any DOCTYPE/html tags if present
html_content = re.sub(r'<\?xml[^>]*\?>', '', html_content)
html_content = re.sub(r'<!DOCTYPE[^>]*>', '', html_content)
html_content = re.sub(r'</?html[^>]*>', '', html_content)
html_content = re.sub(r'<head[^>]*>.*?</head>', '', html_content, flags=re.DOTALL)
# Now process the content normally
# Fix broken attributes with ="" pattern
def fix_broken_attributes_only(match):
tag_content = match.group(0)
if '=""' in tag_content and tag_content.count('=""') > 2:
tag_match = re.match(r'<(\w+)', tag_content)
if tag_match:
tag_name = tag_match.group(1)
words = re.findall(r'(\w+)=""', tag_content)
if words:
content = ' '.join(words)
return f'<{tag_name}>{content}</{tag_name}>'
return ''
return tag_content
# Fix <p"Text... -> <p>"Text... and orphaned < inside <p>
def _fix_malformed_p_tags(text: str) -> str:
# Part 1: Fix <p"Text... -> <p>"Text...
# This handles the specific case where the opening p tag is malformed as <p"
text = re.sub(r'<p"([^>]*?</p>)', r'<p>"\1', text, flags=re.IGNORECASE | re.DOTALL)
# Part 1.5: Fix p>Text... -> <p>Text...
# Handles cases where the opening bracket is missing for the p tag
# Only matches if it starts a line or follows a closing tag, to be safe
text = re.sub(r'(^|>)\s*p>([^<]*?</p>)', r'\1<p>\2', text, flags=re.IGNORECASE | re.DOTALL)
# Part 2: Fix orphaned < inside <p> tags (e.g. <p><Yeah...)
def _process_p_content(m):
open_tag = m.group(1)
content = m.group(2)
close_tag = m.group(3)
new_content = []
last_pos = 0
# Iterate through all < characters in the content
# We use a manual loop to handle logic correctly
i = 0
while i < len(content):
if content[i] == '<':
# Found a <. Check if it's a valid tag or orphaned/text
# Append everything before this <
new_content.append(content[last_pos:i])
# Check for valid tag structure
# 1. Must have a closing >
next_gt = content.find('>', i)
should_escape = False
if next_gt == -1:
# No closing >, definitely orphaned
should_escape = True
else:
# Has closing >. Check interior.
inner_text = content[i+1:next_gt]
# If inner text contains <, then the first < is likely text
# (e.g. "x < y and y < z") unless it's nested tags which is rare inside simple P
# But simpler check: valid tag names start with alpha or /alpha
# and generally don't contain spaces immediately unless attributes
# Check if it looks like a tag: <tagname...> or </tagname...>
# Strip whitespace to handle <br >
inner_stripped = inner_text.strip()
tag_match = re.match(r'^/?([a-zA-Z0-9]+)', inner_stripped)
if not tag_match:
# < 3 or < . or < ? (though <? is processing instruction)
should_escape = True
else:
# It has a tag-like name.
# Check against known HTML tags if we want to be strict,
# but user said "if it's anything else, then it's a whole sentence"
# The case <Yeah... matches [a-zA-Z]+ but had NO >. We handled that above.
# What if we have <Yeah >?
# User's example was <Yeah... (no >).
# If we have <Yeah >, it is technically a tag "Yeah".
# But let's assume if it's not a standard HTML tag, it might be text?
# The user didn't explicitly ask to fix <Text>, only orphaned ones.
# So we stick to the "orphaned" logic (missing >) OR "obviously not a tag" logic.
# For the purpose of this fix, the "missing >" check handles the user's <Yeah example.
# The <I haven't example also has "missing >".
# One edge case: <p>Text <br> Text</p>
# <br> has > and matches [a-zA-Z]. Kept.
pass
if should_escape:
new_content.append('<')
last_pos = i + 1
else:
# It's a tag (or looks enough like one), keep the <
new_content.append('<')
last_pos = i + 1
i += 1
new_content.append(content[last_pos:])
return f"{open_tag}{''.join(new_content)}{close_tag}"
return re.sub(r'(<p[^>]*>)(.*?)(</p>)', _process_p_content, text, flags=re.IGNORECASE | re.DOTALL)
html_content = _fix_malformed_p_tags(html_content)
html_content = re.sub(r'<[^>]*?=\"\"[^>]*?>', fix_broken_attributes_only, html_content)
# Sanitize attributes that contain a colon (:) but are NOT valid namespaces.
# Example: <status effects:="" high="" temperature="" unconscious=""></status>
# becomes: <status data-effects="" high="" temperature="" unconscious=""></status>
def _sanitize_colon_attributes_in_tags(text: str) -> str:
# Process only inside start tags; skip closing tags, comments, doctypes, processing instructions
def _process_tag(tag_match):
tag = tag_match.group(0)
if tag.startswith('</') or tag.startswith('<!') or tag.startswith('<?'):
return tag
def _attr_repl(m):
before, name, eqval = m.group(1), m.group(2), m.group(3)
lname = name.lower()
# Preserve known namespace attributes
if (
lname.startswith('xml:') or lname.startswith('xlink:') or lname.startswith('epub:') or
lname == 'xmlns' or lname.startswith('xmlns:')
):
return m.group(0)
if ':' not in name:
return m.group(0)
# Replace colon(s) with dashes and prefix with data-
safe = re.sub(r'[:]+', '-', name).strip('-')
safe = re.sub(r'[^A-Za-z0-9_.-]', '-', safe) or 'attr'
if not safe.startswith('data-'):
safe = 'data-' + safe
return f'{before}{safe}{eqval}'
# Replace attributes with colon in the name (handles both single and double quoted values)
tag = re.sub(r'(\s)([A-Za-z_:][A-Za-z0-9_.:-]*:[A-Za-z0-9_.:-]*)(\s*=\s*(?:"[^"]*"|\'[^\']*\'))', _attr_repl, tag)
return tag
return re.sub(r'<[^>]+>', _process_tag, text)
html_content = _sanitize_colon_attributes_in_tags(html_content)
# Convert only "story tags" whose TAG NAME contains a colon (e.g., <System:Message>),
# but DO NOT touch valid HTML/SVG tags where colons appear in attributes (e.g., style="color:red" or xlink:href)
# and DO NOT touch namespaced tags like <svg:rect>.
allowed_ns_prefixes = {"svg", "math", "xlink", "xml", "xmlns", "epub"}
# NEW: Fix for "Empty Attribute Tags" (e.g. <unique ability=""></unique>)
# Transforms hallucinated patterns like <unique ability=""></unique> into <unique ability>
if os.getenv('FIX_EMPTY_ATTR_TAGS_EPUB', '0') == '1':
def _escape_empty_attr_tags(text: str) -> str:
# Known HTML tags to preserve
known_tags = {
'html','head','body','title','meta','link','style','script','noscript',
'p','div','span','br','hr','img','a','h1','h2','h3','h4','h5','h6',
'ul','ol','li','dl','dt','dd',
'pre','code','em','strong','b','i','u','s','strike','del','ins','mark','small','sub','sup',
'table','thead','tbody','tr','td','th','caption','col','colgroup',
'blockquote','q','cite',
'section','article','header','footer','nav','main','aside','details','summary',
'figure','figcaption',
'form','input','button','select','option','textarea','label','fieldset','legend',
'iframe','canvas','svg','math',
'video','audio','source','track','embed','object','param',
'map','area',
'center', 'font', 'base'
}
# Transform: <Tag Attr="">Content</Tag> --> <Tag Attr>Content
# This removes the empty attribute value and the closing tag, creating a visual "token" style representation
def _repl_pair(m):
tagname = m.group(1)
if tagname.lower() in known_tags:
return m.group(0)
attrname = m.group(2)
content = m.group(3)
return f"<{tagname} {attrname}>{content}"
# Match <Tag Attr=""></Tag> (allow whitespace and content)
text = re.sub(r'<([a-zA-Z0-9_\-]+)\s+([a-zA-Z0-9_\-]+)=""\s*>(.*?)</\1>', _repl_pair, text, flags=re.DOTALL)
return text
html_content = _escape_empty_attr_tags(html_content)
def _escape_story_tag(match):
full_tag = match.group(0) # Entire <...> or </...>
tag_name = match.group(1) # The tag name possibly containing ':'
prefix = tag_name.split(':', 1)[0].lower()
# If this is a known namespace prefix (e.g., svg:rect), leave it alone
if prefix in allowed_ns_prefixes:
return full_tag
# Otherwise, treat as a story/fake tag and replace angle brackets with Chinese brackets
return full_tag.replace('<', '《').replace('>', '》')
# Escape invalid story tags (tag names containing ':') so they render literally with angle brackets.
allowed_ns_prefixes = {"svg", "math", "xlink", "xml", "xmlns", "epub"}
def _escape_story_tag_entities(m):
tagname = m.group(1)
prefix = tagname.split(':', 1)[0].lower()
if prefix in allowed_ns_prefixes:
return m.group(0)
tag_text = m.group(0)
return tag_text.replace('<', '<').replace('>', '>')
# Apply in order: self-closing, opening, closing
html_content = re.sub(r'<([A-Za-z][\w.-]*:[\w.-]*)\s*([^>]*)/>', _escape_story_tag_entities, html_content)
html_content = re.sub(r'<([A-Za-z][\w.-]*:[\w.-]*)\s*([^>]*)>', _escape_story_tag_entities, html_content)
html_content = re.sub(r'</([A-Za-z][\w.-]*:[\w.-]*)\s*>', _escape_story_tag_entities, html_content)
# PREVENT malformed "fake tags" like <You are a farmer.> from being parsed as tags
# We only target angle-bracketed text that has spaces and NO '=' (so it's not real attributes)
# and ends with either '>' or the entity '>'.
def _escape_plaintext_angle_brackets(txt: str) -> str:
def repl(m):
inner = m.group(1)
# If looks like a real tag (has '=' or '/') keep it
# Check for start chars /!? or if it has attributes (=)
if '=' in inner or inner.strip().startswith(('/', '!', '?')):
return m.group(0)
# If the first token is a known HTML tag name, keep it
tokens = inner.strip().split()
if not tokens:
return m.group(0)
first = tokens[0].lower()
# Handle self-closing tags like <br/> by removing trailing slash
if first.endswith('/'):
first = first[:-1]
known = {
'html','head','body','title','meta','link','style','script','noscript',
'p','div','span','br','hr','img','a','h1','h2','h3','h4','h5','h6',
'ul','ol','li','dl','dt','dd',
'pre','code','em','strong','b','i','u','s','strike','del','ins','mark','small','sub','sup',
'table','thead','tbody','tr','td','th','caption','col','colgroup',
'blockquote','q','cite',
'section','article','header','footer','nav','main','aside','details','summary',
'figure','figcaption',
'form','input','button','select','option','textarea','label','fieldset','legend',
'iframe','canvas','svg','math',
'video','audio','source','track','embed','object','param',
'map','area',
'center', 'font', 'base'
}
if first in known:
return m.group(0)
# Otherwise, treat as narrative text in angle brackets and escape
return f'<{inner}>'
# Match <...> where content matches non-brackets.
# Allow single words without spaces (e.g. <luck>)
pattern = r'<([^<>]+)>'
txt = re.sub(pattern, repl, txt)
# Also handle cases where closing bracket is already an entity.
# IMPORTANT: Don't let this match across *real tags* like:
# <a href="<part0009.html#id>">...
# because it will convert the *start tag* into literal text (<a ...), breaking TOC links.
def repl_gt(m):
inner = m.group(1)
if '=' in inner or inner.strip().startswith(('/', '!', '?')):
return m.group(0)
tokens = inner.strip().split()
if not tokens:
return m.group(0)
first = tokens[0].lower()
if first.endswith('/'):
first = first[:-1]
known = {
'html','head','body','title','meta','link','style','script','noscript',
'p','div','span','br','hr','img','a','h1','h2','h3','h4','h5','h6',
'ul','ol','li','dl','dt','dd',
'pre','code','em','strong','b','i','u','s','strike','del','ins','mark','small','sub','sup',
'table','thead','tbody','tr','td','th','caption','col','colgroup',
'blockquote','q','cite',
'section','article','header','footer','nav','main','aside','details','summary',
'figure','figcaption',
'form','input','button','select','option','textarea','label','fieldset','legend',
'iframe','canvas','svg','math',
'video','audio','source','track','embed','object','param',
'map','area',
'center', 'font', 'base'
}
if first in known:
return m.group(0)
return f'<{inner}>'
pattern_gt = r'<([^<>]+)>'
txt = re.sub(pattern_gt, repl_gt, txt)
return txt
html_content = _escape_plaintext_angle_brackets(html_content)
# Parse with lxml
from lxml import html as lxml_html, etree
parser = lxml_html.HTMLParser(recover=True)
doc = lxml_html.document_fromstring(f"<div>{html_content}</div>", parser=parser)
# Fix common malformed link attributes produced by some EPUB sources/LLM output:
# href="<part0008.html#id>" -> href="part0008.html#id"
# href="<part0008.html#id>" -> href="part0008.html#id"
# If left as-is, XML serialization will escape the angle brackets, breaking navigation.
def _strip_angle_wrapped_url(v: str) -> str:
try:
if v is None:
return v
s = str(v).strip()
if not s:
return s
sl = s.lower()
# Full wrapper (entities)
if sl.startswith('<') and sl.endswith('>') and len(s) >= 8:
return s[4:-4].strip()
# Full wrapper (literal)
if s.startswith('<') and s.endswith('>') and len(s) >= 2:
return s[1:-1].strip()
# One-sided wrappers (best-effort)
if sl.startswith('<'):
s = s[4:]
if sl.endswith('>') and len(s) >= 4:
s = s[:-4]
if s.startswith('<'):
s = s[1:]
if s.endswith('>'):
s = s[:-1]
return s.strip()
except Exception:
return v
try:
for el in doc.iter():
# Common link attributes
for attr in ('href', 'src'):
try:
if attr in el.attrib:
old = el.attrib.get(attr)
new = _strip_angle_wrapped_url(old)
if new != old:
el.attrib[attr] = new
except Exception:
pass
# Namespaced link attributes (e.g., SVG)
try:
for attr in ("{http://www.w3.org/1999/xlink}href", 'xlink:href'):
if attr in el.attrib:
old = el.attrib.get(attr)
new = _strip_angle_wrapped_url(old)
if new != old:
el.attrib[attr] = new
except Exception:
pass
except Exception:
pass
# Get the content back
# Use HTML method if enabled (better whitespace preservation for buggy readers like Freda)
# but may reduce XHTML compliance. Default: xml (strict XHTML)
serialize_method = 'html' if os.getenv('EPUB_USE_HTML_METHOD', '0') == '1' else 'xml'
body_xhtml = etree.tostring(doc, method=serialize_method, encoding='unicode')
# Remove the wrapper div we added
body_xhtml = re.sub(r'^<div[^>]*>|</div>$', '', body_xhtml)
# Optionally replace angle-bracket entities with Chinese brackets
# Default behavior: keep them as entities (< >) so the output preserves the original text
bracket_style = os.getenv('ANGLE_BRACKET_OUTPUT', 'entity').lower()
if '<' in body_xhtml or '>' in body_xhtml:
if bracket_style in ('cjk', 'chinese', 'cjk_brackets'):
body_xhtml = body_xhtml.replace('<', '《').replace('>', '》')
# else: keep as entities
# Build our own clean XHTML document
return XHTMLConverter._build_xhtml(title, body_xhtml, css_links)
except Exception as e:
log(f"[WARNING] Failed to ensure XHTML compliance: {e}")
import traceback
log(f"[DEBUG] Full traceback:\n{traceback.format_exc()}")
log(f"[DEBUG] Failed chapter title: {title}")
log(f"[DEBUG] First 500 chars of input: {html_content[:500] if html_content else 'EMPTY'}")
return XHTMLConverter._build_fallback_xhtml(title)
@staticmethod
def _build_xhtml(title: str, body_content: str, css_links: Optional[List[str]] = None) -> str:
"""Build XHTML document"""
if not body_content.strip():
body_content = '<p>Empty chapter</p>'
title = ContentProcessor.safe_escape(title)
body_content = XHTMLConverter._ensure_xml_safe_readable(body_content)
xml_declaration = '<?xml version="1.0" encoding="utf-8"?>'
doctype = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">'
# Use class-level default language for html element language attributes
lang = getattr(XHTMLConverter, "DEFAULT_LANG", "en")
xhtml_parts = [
xml_declaration,
doctype,
f'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" xml:lang="{lang}" lang="{lang}">',
'<head>',
'<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />',
f'<title>{title}'
]
if css_links:
for css_link in css_links:
if css_link.startswith('')
xhtml_parts.extend([
'',
'',
body_content,
'',
'