# backend/voice/engines/chatterbox_engine.py # §0.4 ENGINE 3: CHATTERBOX — cinematic intros, storytelling, presentations, demo videos # Used ONLY for non-realtime, pre-rendered content. NEVER inside the live wake-word loop. import logging import numpy as np import asyncio logger = logging.getLogger(__name__) from backend.voice.engines.kokoro_engine import inject_jarvis_pauses _chatterbox_model = None class ChatterboxSpecWrapper: """Wrapper to map spec parameters to actual Chatterbox implementation parameters.""" def __init__(self, model): self._model = model self.sample_rate = model.sr if hasattr(model, "sr") else 24000 def generate(self, text: str, voice_profile: str, pacing: float, expressiveness: str) -> np.ndarray: import os app_data = os.environ.get("JARVIS_APP_DATA_DIR", ".") speaker_wav = os.path.join(app_data, "voices", "jarvis.wav") # Map spec "expressiveness" to actual exaggeration parameter exag = 0.3 if expressiveness == "low" else 0.5 wav = self._model.generate( text, audio_prompt_path=speaker_wav if os.path.exists(speaker_wav) else None, exaggeration=exag, cfg_weight=0.5 ) try: return wav.squeeze().cpu().numpy().astype(np.float32) except AttributeError: return np.array(wav, dtype=np.float32) def _get_chatterbox(): global _chatterbox_model if _chatterbox_model is None: from chatterbox.tts import ChatterboxTTS raw_model = ChatterboxTTS.from_pretrained(device="cpu") _chatterbox_model = ChatterboxSpecWrapper(raw_model) logger.info("[Chatterbox] Model loaded.") return _chatterbox_model def concatenate_with_pause_markers(chunks: list[tuple[str, int]]) -> str: parts = [] for clause_text, pause_ms in chunks: parts.append(clause_text) return " ".join(parts) async def synthesize_chatterbox(text: str) -> bytes: """ §0.4 Chatterbox synthesis tuned to the JARVIS acoustic profile. Used for cinematic / pre-rendered content only. """ import io import scipy.io.wavfile from backend.voice.jarvis_voice_profile import JARVIS_VOICE_PROFILE from backend.voice.post_process import apply_jarvis_dsp_chain def _sync(): chatterbox_model = _get_chatterbox() chunks = inject_jarvis_pauses(text, JARVIS_VOICE_PROFILE) # Exact API spec mapping audio = chatterbox_model.generate( text=concatenate_with_pause_markers(chunks), voice_profile="deep_male_calm", # closest preset/clone target available in Chatterbox pacing=0.82, # same WPM target as XTTS expressiveness="low" # near-flat intonation ) audio = apply_jarvis_dsp_chain(audio, sr=chatterbox_model.sample_rate) out = io.BytesIO() scipy.io.wavfile.write(out, chatterbox_model.sample_rate, np.int16(audio * 32767)) return out.getvalue() loop = asyncio.get_running_loop() return await loop.run_in_executor(None, _sync)