"""
DoodleBook — HF ZeroGPU Version
Free T4 GPU on Hugging Face Spaces!
No Modal needed.
"""
import gradio as gr
import os
import sys
import torch
try:
import spaces
except ModuleNotFoundError:
# `spaces` only exists on HF ZeroGPU. Off-HF (local/dev) provide a no-op so
# the app still runs; generation then uses whatever local GPU/CPU exists.
class _SpacesShim:
@staticmethod
def GPU(*args, **kwargs):
if args and callable(args[0]): # bare @spaces.GPU
return args[0]
def deco(fn): # @spaces.GPU(duration=...)
return fn
return deco
spaces = _SpacesShim()
import json
import time
import tempfile
import logging
import struct
import re
sys.path.insert(0, os.path.dirname(__file__))
from config import (
FLUX_MODEL, STORY_MODEL, TTS_MODEL,
GENERATION_PARAMS, SAMPLE_BOOK_PATH, BASE_SEED, page_seed,
DEFAULT_VOICE, voice_design,
)
from book_builder import (
build_book_html, export_pdf, magic_loader_html,
build_coloring_html, export_coloring_pdf,
)
from ui.layout import create_layout
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# ZeroGPU sets SPACES_ZERO_GPU. On the Space we load models on cuda at IMPORT
# (a CUDA-emulation layer makes that work without a real GPU); lazy-loading
# inside @spaces.GPU is explicitly discouraged and was why FLUX kept failing
# → sketch. Guarded so a local/dev import doesn't try to pull ~20GB of weights.
ON_ZEROGPU = bool(os.environ.get("SPACES_ZERO_GPU"))
_FLUX_PIPE = None
_STORY_MODEL = None
_STORY_TOKENIZER = None
_TTS_MODEL = None
_LOAD_ERRORS = {}
def load_flux():
"""FLUX image pipeline placed on cuda at module scope (the ZeroGPU pattern).
No enable_model_cpu_offload() — that fights ZeroGPU's device management."""
global _FLUX_PIPE
if _FLUX_PIPE is None:
from diffusers import Flux2KleinPipeline
logger.info(f"Loading image model: {FLUX_MODEL.hub_id}")
pipe = Flux2KleinPipeline.from_pretrained(
FLUX_MODEL.hub_id, torch_dtype=torch.bfloat16,
)
pipe.to("cuda")
_FLUX_PIPE = pipe
return _FLUX_PIPE
def load_story():
global _STORY_MODEL, _STORY_TOKENIZER
if _STORY_MODEL is None:
from transformers import AutoTokenizer, AutoModelForCausalLM
logger.info(f"Loading story model: {STORY_MODEL.hub_id}")
_STORY_TOKENIZER = AutoTokenizer.from_pretrained(
STORY_MODEL.hub_id, trust_remote_code=True,
)
_STORY_MODEL = AutoModelForCausalLM.from_pretrained(
STORY_MODEL.hub_id, torch_dtype=torch.float16, trust_remote_code=True,
).to("cuda").eval()
return _STORY_MODEL, _STORY_TOKENIZER
def load_tts():
global _TTS_MODEL
if _TTS_MODEL is None:
from voxcpm import VoxCPM
logger.info(f"Loading TTS model: {TTS_MODEL.hub_id}")
_TTS_MODEL = VoxCPM.from_pretrained(TTS_MODEL.hub_id, load_denoiser=False)
return _TTS_MODEL
if ON_ZEROGPU:
for _name, _loader in (("flux", load_flux), ("story", load_story), ("tts", load_tts)):
try:
_loader()
except Exception as _e: # keep the Space booting
_LOAD_ERRORS[_name] = repr(_e)
logger.exception(f"Module-level load failed for {_name}")
COLOR_ART_STYLE = (
"hand-drawn crayon children's storybook illustration, soft waxy crayon "
"texture and visible crayon strokes, warm colorful crayon shading, "
"simple friendly shapes, looks drawn by hand with crayons"
)
COLOR_PAGE_SUFFIX = "full colorful background scene, the character clearly visible."
LINE_ART_STYLE = (
"children's coloring book page, pure black ink outlines on pure white paper, "
"clean contour lines, no color, no gray, no shading, no texture, "
"no hatching, no pencil marks, open spaces to color"
)
LINE_ART_SUFFIX = (
"simple clean background shapes, same composition, thick readable outlines, "
"no filled black areas, no extra sketch marks."
)
THEME_TEMPLATES = {
"brave adventure": [
("{hero} loved exploring new places.", "{hero} standing at the start of a bright adventure trail"),
("One morning, {hero} discovered something glowing nearby.", "{hero} spotting a magical glow in the distance"),
("Taking a deep breath, {hero} bravely went closer.", "{hero} walking forward with courage"),
("There, a new friend needed help.", "{hero} finding a small friend in trouble"),
("{hero} helped with kindness and a clever idea.", "{hero} helping the friend together"),
("Everyone cheered, and {hero} felt proud and brave.", "{hero} celebrating at sunset with the new friend"),
],
"making a new friend": [
("{hero} was playing alone in a sunny place.", "{hero} playing under a bright sky"),
("Then {hero} noticed someone shy nearby.", "{hero} seeing a shy new friend nearby"),
("{hero} smiled and said hello.", "{hero} waving with a friendly smile"),
("Soon they were sharing stories and laughs.", "{hero} and the new friend laughing together"),
("They played games all afternoon.", "{hero} and the new friend playing together"),
("By sunset, {hero} had made a wonderful new friend.", "{hero} and the new friend smiling together at sunset"),
],
}
FEW_SHOT_EXEMPLAR = """
Write a 6-page children's storybook for age 5 about Luna the cat with theme: brave adventure.
Return ONLY valid JSON:
{
"title": "Luna's Brave Adventure",
"character_description": "A small orange tabby cat named Luna with big green eyes, whiskers, and a tiny red scarf",
"pages": [
{"page": 1, "text": "Luna was a small orange cat who loved to explore.", "scene": "Luna sitting by the window looking outside"},
{"page": 2, "text": "One sunny morning, Luna saw something sparkling in the forest.", "scene": "Luna spotting a glow in the trees"},
{"page": 3, "text": "Bravely, Luna crept into the forest to investigate.", "scene": "Luna walking cautiously through trees"},
{"page": 4, "text": "It was a tiny fairy stuck in a spider web!", "scene": "Luna discovering a fairy in trouble"},
{"page": 5, "text": "Luna gently freed the fairy with her paw.", "scene": "Luna carefully helping the fairy"},
{"page": 6, "text": "The fairy thanked Luna and they became friends forever.", "scene": "Luna and fairy playing together at sunset"}
]
}
"""
def build_story_prompt(hero_name: str, theme: str, age: int) -> str:
return f"""{FEW_SHOT_EXEMPLAR}
Write a 6-page children's storybook for age {age} about {hero_name} with theme: {theme}.
Return ONLY valid JSON:
"""
def _validate_story_structure(story: dict) -> bool:
required_keys = ["title", "character_description", "pages"]
if not all(k in story for k in required_keys):
return False
pages = story.get("pages", [])
if not isinstance(pages, list) or len(pages) < 1:
return False
first_page = pages[0]
return all(k in first_page for k in ["page", "text", "scene"])
def _repair_json(json_str: str) -> str:
json_str = re.sub(r',\s*([}\]])', r'\1', json_str)
json_str = re.sub(r'//.*?$', '', json_str, flags=re.MULTILINE)
json_str = re.sub(r'/\*[\s\S]*?\*/', '', json_str)
json_str = re.sub(r'(?<=")\n(?=")', '\\n', json_str)
json_str = re.sub(r'(\s)(\w+)(\s*:)', r'\1"\2"\3', json_str)
return json_str
def parse_story_json(raw_output: str) -> dict | None:
match = re.search(r'\{[\s\S]*\}', raw_output or "")
if not match:
return None
raw_json = match.group(0)
for candidate in (raw_json, _repair_json(raw_json)):
try:
story = json.loads(candidate)
if _validate_story_structure(story):
return story
except Exception:
continue
return None
def _normalize_story(story: dict) -> dict:
pages = list(story.get("pages", []))[:6]
while len(pages) < 6:
pages.append({
"page": len(pages) + 1,
"text": "And the adventure continued happily.",
"scene": "Continuing adventure",
})
story["pages"] = pages
story.setdefault("title", "A Wonderful Adventure")
story.setdefault(
"character_description",
"A friendly children's storybook hero with bright colors and cheerful features",
)
return story
def build_story_locally(hero_name: str, theme: str) -> dict:
"""Fast, deterministic fallback story that avoids any Modal dependency."""
hero = (hero_name or "Little Hero").strip() or "Little Hero"
beats = THEME_TEMPLATES.get(theme, THEME_TEMPLATES["brave adventure"])
pages = [
{"page": i + 1, "text": text.format(hero=hero), "scene": scene.format(hero=hero)}
for i, (text, scene) in enumerate(beats)
]
return {
"title": f"{hero}'s Storybook Adventure",
"character_description": (
f"{hero}, a friendly children's storybook hero with bright colors, "
"bold outlines, and a cheerful expressive face"
),
"pages": pages,
}
def silent_wav_bytes(duration_seconds: int = 2, sample_rate: int = 24000) -> bytes:
"""Return a short silent WAV so the UI remains stable if TTS is unavailable."""
num_samples = sample_rate * duration_seconds
data_size = num_samples * 2
header = struct.pack(
"<4sI4s4sIHHIIHH4sI",
b"RIFF", 36 + data_size, b"WAVE",
b"fmt ", 16, 1, 1, sample_rate, sample_rate * 2, 2, 16,
b"data", data_size,
)
return header + (b"\x00" * data_size)
def _with_heartbeat(blocking_fn, frame_fn, poll=4.0):
import threading
box = {}
def _run():
try:
box["val"] = blocking_fn()
except BaseException as e:
box["err"] = e
th = threading.Thread(target=_run, daemon=True)
th.start()
t0 = time.time()
while th.is_alive():
th.join(timeout=poll)
if th.is_alive():
yield ("hb", frame_fn(int(time.time() - t0)))
if "err" in box:
raise box["err"]
yield ("done", box["val"])
# ============================================================================
# SAMPLE BOOK (loads instantly, no GPU needed)
# ============================================================================
SAMPLE_BOOK_HTML = None
def load_sample_book() -> str:
"""Load pre-generated sample book (C3: always ship sample)."""
global SAMPLE_BOOK_HTML
if SAMPLE_BOOK_HTML:
return SAMPLE_BOOK_HTML
sample_path = os.path.join(SAMPLE_BOOK_PATH, "sample.html")
if os.path.exists(sample_path):
with open(sample_path, "r", encoding="utf-8") as f:
SAMPLE_BOOK_HTML = f.read()
return SAMPLE_BOOK_HTML
return "
Loading sample book...
"
# ============================================================================
# ZEROGPU INFERENCE FUNCTIONS
# ============================================================================
@spaces.GPU(duration=60)
def generate_story_gpu(hero_name: str, theme: str, age: int = 5) -> dict:
"""Generate a story on ZeroGPU, falling back to a deterministic local story."""
try:
model, tok = load_story()
prompt = build_story_prompt(hero_name, theme, age)
inputs = tok.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
enable_thinking=False,
return_dict=True,
return_tensors="pt",
).to("cuda")
with torch.no_grad():
out = model.generate(
**inputs,
max_new_tokens=GENERATION_PARAMS.max_story_tokens,
do_sample=False,
)
response = tok.decode(
out[0][inputs["input_ids"].shape[1]:],
skip_special_tokens=True,
)
parsed = parse_story_json(response)
if parsed:
return _normalize_story(parsed)
logger.warning("Story parser failed; using deterministic local fallback")
except Exception as e:
logger.warning(f"ZeroGPU story generation failed: {e}")
return _normalize_story(build_story_locally(hero_name, theme))
@spaces.GPU(duration=150)
def generate_images_gpu(
character_desc: str,
scenes: list,
doodle_bytes: bytes = None,
seed: int = 42,
) -> list:
"""Generate all story pages with FLUX on ZeroGPU (two-stage: canonical
character from the doodle, then the same character in each scene)."""
import io
from PIL import Image
pipe = load_flux()
num_steps, guidance = 6, 1.0
canonical = None
if doodle_bytes:
try:
ref = Image.open(io.BytesIO(doodle_bytes)).convert("RGB")
canonical = pipe(
prompt=(f"Turn this child's drawing into a clean, friendly, full-body cartoon "
f"character for a children's storybook. Keep the EXACT same creature, "
f"face, and features as the drawing. {COLOR_ART_STYLE}, "
f"plain white background, full character visible, centered."),
image=ref, height=768, width=768, guidance_scale=guidance,
num_inference_steps=num_steps,
generator=torch.Generator("cuda").manual_seed(seed),
).images[0]
logger.info("Canonical character built from doodle")
except Exception as e:
logger.warning(f"Canonical build failed ({e}); text2img fallback")
canonical = None
images = []
for i, scene in enumerate(scenes):
if canonical is not None:
prompt = f"The same character. {scene}. {COLOR_ART_STYLE}, {COLOR_PAGE_SUFFIX}"
kw = dict(image=canonical, prompt=prompt)
else:
prompt = (f"{character_desc}. Scene: {scene}. {COLOR_ART_STYLE}, "
f"white background, centered, full character visible")
kw = dict(prompt=prompt)
kw.update(height=768, width=768, guidance_scale=guidance,
num_inference_steps=num_steps,
generator=torch.Generator("cuda").manual_seed(seed + i + 1))
images.append(pipe(**kw).images[0])
logger.info(f"Generated page {i+1}/{len(scenes)}")
return images
@spaces.GPU(duration=150)
def generate_coloring_images_gpu(color_pngs: list, seed: int = 7) -> list:
"""Coloring pages = FLUX redraws each finished COLOR page as clean line art
(img2img). This MATCHES the storybook composition and avoids the speckly
look of tracing crayon texture. Caller crispens the result to black-on-white."""
import io
from PIL import Image
pipe = load_flux()
prompt = f"{LINE_ART_STYLE}, {LINE_ART_SUFFIX}"
outs = []
for i, png in enumerate(color_pngs):
ref = Image.open(io.BytesIO(png)).convert("RGB")
base = dict(prompt=prompt, image=ref, height=768, width=768,
guidance_scale=1.0, num_inference_steps=6,
generator=torch.Generator("cuda").manual_seed(seed + i))
try: # strength may not be accepted
img = pipe(**base, strength=0.85).images[0]
except TypeError:
img = pipe(**base).images[0]
outs.append(img)
logger.info(f"Generated coloring page {i+1}/{len(color_pngs)}")
return outs
@spaces.GPU(duration=120)
def generate_tts_gpu(text: str, voice: str = DEFAULT_VOICE) -> bytes:
"""Narrate the book with VoxCPM2. Raises on failure so the caller can show
the real reason instead of silently shipping a silent clip."""
import io
import numpy as np
try:
model = load_tts()
design = voice_design(voice)
import re
chunks = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if s.strip()]
if not chunks:
chunks = [text.strip() or "The end."]
sr = model.tts_model.sample_rate
pause = np.zeros(int(sr * 0.35), dtype=np.float32)
pieces = []
for i, sentence in enumerate(chunks):
wav = model.generate(
text=f"{design} {sentence}",
cfg_value=2.0,
inference_timesteps=10,
)
pieces.append(np.asarray(wav, dtype=np.float32))
if i < len(chunks) - 1:
pieces.append(pause)
audio = np.concatenate(pieces)
import soundfile as sf
buf = io.BytesIO()
sf.write(buf, audio, sr, format="WAV")
return buf.getvalue()
except Exception as e:
# Surface the real reason (e.g. missing model) instead of a silent clip
# that looks like it worked. create_book records this in the trace.
logger.exception("TTS failed")
raise
# ============================================================================
# MAIN BOOK CREATION (Generator for streaming)
# ============================================================================
def create_book(doodle_image, character_name, theme, hero_name, voice=DEFAULT_VOICE, make_coloring=False):
"""ZeroGPU book flow: story → images → narration → PDFs → coloring book,
each a sequential @spaces.GPU call (ZeroGPU has one GPU per request)."""
t_total = time.perf_counter()
character_name = (character_name or "").strip() or "Little Hero"
hero_name = (hero_name or "").strip() or character_name
trace_data = {
"backend": "zerogpu",
"hero_name": hero_name,
"theme": theme,
"voice": voice,
"make_coloring": make_coloring,
"seed": BASE_SEED,
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
}
if _LOAD_ERRORS:
trace_data["model_load_errors"] = _LOAD_ERRORS
_no = gr.update(visible=False)
_keep = gr.update()
yield (
magic_loader_html("story", hero_name),
"Writing the story…",
None, _keep, {}, "", json.dumps(trace_data, indent=2),
_no, _keep,
)
t_story = time.perf_counter()
try:
story = generate_story_gpu(hero_name, theme)
except Exception as e:
logger.error(f"Story generation failed: {e}")
yield (
f"Error: {e}
",
f"Error: {e}",
None, _keep, {}, "", "",
_no, _keep,
)
return
trace_data["story_sec"] = round(time.perf_counter() - t_story, 2)
pages = story.get("pages", [])
char_desc = story.get("character_description", "")
title = story.get("title", "Untitled Story")
page_texts = [p.get("text", "") for p in pages]
scenes = [p.get("scene", "") for p in pages]
trace_data["title"] = title
trace_data["character_description"] = char_desc
yield (
magic_loader_html("images", hero_name),
f"{title} — illustrating on ZeroGPU…",
None, _keep, story, "", json.dumps(trace_data, indent=2),
_no, _keep,
)
doodle_bytes = None
if doodle_image is not None:
import io
from PIL import Image
img = Image.fromarray(doodle_image)
buf = io.BytesIO()
img.save(buf, format="PNG")
doodle_bytes = buf.getvalue()
full_text = f"{title}. {' '.join(page_texts)}"
# ---- NARRATION starts NOW, in PARALLEL with the images (it only needs the
# story text). The audio is surfaced the moment it's ready — usually
# before the illustrations finish — so narration appears first. ----
import threading
voice_box = {}
t_tts = time.perf_counter()
def _do_voice():
try:
voice_box["bytes"] = generate_tts_gpu(full_text, voice)
except Exception as e:
voice_box["err"] = e
voice_thread = threading.Thread(target=_do_voice, daemon=True)
voice_thread.start()
def _audio_now():
"""Write the narration to a temp wav once it's ready; return its path."""
if voice_box.get("bytes") and not voice_box.get("path"):
try:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
tmp.write(voice_box["bytes"])
voice_box["path"] = tmp.name
except Exception as e:
logger.warning(f"writing audio failed: {e}")
return voice_box.get("path")
# ---- IMAGES (FLUX on ZeroGPU), surfacing the narration as soon as it lands ----
img_bytes, engine, images = None, "sketch", None
t_images = time.perf_counter()
try:
for kind, payload in _with_heartbeat(
lambda: generate_images_gpu(char_desc, scenes, doodle_bytes, BASE_SEED),
lambda s: (
magic_loader_html("images", hero_name),
f"{title} — illustrating… {s}s"
+ (" · narration ready ▶" if _audio_now() else " · recording narration…"),
_audio_now(), _keep, story, "", json.dumps(trace_data, indent=2), _no, _keep,
),
):
if kind == "hb":
yield payload
else:
images = payload
import io
img_bytes = []
for img in images:
buf = io.BytesIO()
img.save(buf, format="PNG")
img_bytes.append(buf.getvalue())
engine = "flux"
except Exception as e:
logger.exception("Image generation failed")
trace_data["image_error"] = repr(e)
from services.images import generate_placeholder_images
img_bytes = generate_placeholder_images(char_desc, scenes, doodle_bytes)
engine = "sketch"
trace_data["images_sec"] = round(time.perf_counter() - t_images, 2)
trace_data["engine"] = engine
book_html = build_book_html(img_bytes, page_texts, title, engine)
# ---- collect the parallel narration (usually already finished) ----
while voice_thread.is_alive():
voice_thread.join(timeout=4)
if voice_thread.is_alive():
yield (book_html, f"{title} — finishing narration…",
_audio_now(), _keep, story, "", json.dumps(trace_data, indent=2),
_no, _keep)
audio_path = _audio_now()
if voice_box.get("err"):
logger.warning(f"TTS failed: {voice_box['err']}")
trace_data["tts_error"] = repr(voice_box["err"])
trace_data["tts_sec"] = round(time.perf_counter() - t_tts, 2)
pdf_path = None
t_pdf = time.perf_counter()
try:
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
pdf_path = export_pdf(img_bytes, page_texts, title, tmp.name)
except Exception as e:
logger.warning(f"PDF failed: {e}")
trace_data["pdf_sec"] = round(time.perf_counter() - t_pdf, 2)
coloring_html = ""
coloring_pdf_path = None
if make_coloring:
t_coloring = time.perf_counter()
try:
from services.coloring import _crispen
for kind, payload in _with_heartbeat(
lambda: generate_coloring_images_gpu(img_bytes, 7),
lambda s: (
book_html,
f"{title} — building coloring book… {s}s",
audio_path,
_keep,
story,
"",
json.dumps(trace_data, indent=2),
_no,
_keep,
),
):
if kind == "hb":
yield payload
else:
coloring_images = payload
import io
outlines = []
for img in coloring_images:
buf = io.BytesIO()
img.save(buf, format="PNG")
outlines.append(_crispen(buf.getvalue()))
coloring_html = build_coloring_html(outlines, page_texts, title)
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
coloring_pdf_path = export_coloring_pdf(outlines, page_texts, title, tmp.name)
trace_data["coloring_book"] = True
trace_data["coloring_engine"] = "flux-direct-lineart"
except Exception as e:
logger.exception("FLUX line-art coloring failed; using local trace fallback")
trace_data["coloring_error"] = repr(e)
try:
from services.coloring import _to_line_art_opencv
outlines = [_to_line_art_opencv(b) for b in img_bytes]
coloring_html = build_coloring_html(outlines, page_texts, title)
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp:
coloring_pdf_path = export_coloring_pdf(outlines, page_texts, title, tmp.name)
trace_data["coloring_book"] = True
trace_data["coloring_engine"] = "opencv-trace-fallback"
except Exception as e2:
logger.exception("Coloring fallback also failed")
trace_data["coloring_error2"] = repr(e2)
trace_data["coloring_sec"] = round(time.perf_counter() - t_coloring, 2)
trace_data["completed"] = True
trace_data["pages_generated"] = len(img_bytes)
trace_data["total_sec"] = round(time.perf_counter() - t_total, 2)
# Reveal the download buttons (they start visible=False) — value alone left
# them hidden, which is why there was "no download option" (incl. on mobile).
pdf_update = gr.update(value=pdf_path, visible=True) if pdf_path else _keep
coloring_pdf_update = (gr.update(value=coloring_pdf_path, visible=True)
if coloring_pdf_path else _keep)
coloring_display_update = (gr.update(visible=True, value=coloring_html) if coloring_html
else _no)
yield (
book_html,
f"Complete: {title} — {len(img_bytes)} pages · {'FLUX (ZeroGPU)' if engine == 'flux' else 'local sketch fallback'} · voice: {voice} · total {trace_data['total_sec']}s",
audio_path,
pdf_update,
story,
f"Pages: {len(img_bytes)} | Seed: {BASE_SEED} | Engine: {engine} | Story {trace_data.get('story_sec', 0)}s | Images {trace_data.get('images_sec', 0)}s | PDF {trace_data.get('pdf_sec', 0)}s | Coloring {trace_data.get('coloring_sec', 0)}s",
json.dumps(trace_data, indent=2),
coloring_display_update,
coloring_pdf_update,
)
# ============================================================================
# MAIN
# ============================================================================
if __name__ == "__main__":
demo = create_layout(
load_sample_fn=load_sample_book,
create_book_fn=create_book,
)
demo.queue(default_concurrency_limit=2, max_size=8)
# design_kwargs (theme/css/js/head) is non-empty on gradio 6 (moved to launch)
demo.launch(share=False, allowed_paths=[tempfile.gettempdir()], **demo.design_kwargs)