feat: add Chatterbox TTS engine for multilingual voice cloning

- New ChatterboxTTSBackend wrapping ChatterboxMultilingualTTS (ResembleAI/chatterbox)
- Supports 23 languages including Hebrew, forces CPU on macOS (MPS issue)
- Monkey-patches torch.load for CPU loading, forces eager attention for compatibility
- trim_tts_output utility cuts trailing silence/hallucination from Chatterbox output
- Full engine integration: /generate, /generate/stream, model status/download/delete
- Hebrew (he) added to supported languages in frontend and backend validation
- Single flat model dropdown extended with Chatterbox option in both generation UIs
- ModelManagement UI groups LuxTTS and Chatterbox under 'Other Voice Models' section

backend/backends/__init__.py

@@ -121,6 +121,7 @@ _stt_backend: Optional[STTBackend] = None
 TTS_ENGINES = {
     "qwen": "Qwen TTS",
     "luxtts": "LuxTTS",
+    "chatterbox": "Chatterbox TTS",
 }
 
 
@@ -167,6 +168,9 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
         elif engine == "luxtts":
             from .luxtts_backend import LuxTTSBackend
             backend = LuxTTSBackend()
+        elif engine == "chatterbox":
+            from .chatterbox_backend import ChatterboxTTSBackend
+            backend = ChatterboxTTSBackend()
         else:
             raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
         

backend/backends/chatterbox_backend.py

@@ -0,0 +1,318 @@
+"""
+Chatterbox TTS backend implementation.
+
+Wraps ChatterboxMultilingualTTS from chatterbox-tts for zero-shot
+voice cloning. Supports 23 languages including Hebrew. Forces CPU
+on macOS due to known MPS tensor issues.
+"""
+
+import asyncio
+import logging
+import platform
+import threading
+from pathlib import Path
+from typing import ClassVar, List, Optional, Tuple
+
+import numpy as np
+
+from . import TTSBackend
+from ..utils.audio import normalize_audio, load_audio
+from ..utils.progress import get_progress_manager
+from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
+from ..utils.tasks import get_task_manager
+
+logger = logging.getLogger(__name__)
+
+CHATTERBOX_HF_REPO = "ResembleAI/chatterbox"
+
+# Files that must be present for the multilingual model
+_MTL_WEIGHT_FILES = [
+    "t3_mtl23ls_v2.safetensors",
+    "s3gen.pt",
+    "ve.pt",
+]
+
+
+class ChatterboxTTSBackend:
+    """Chatterbox Multilingual TTS backend for voice cloning."""
+
+    # Class-level lock for torch.load monkey-patching
+    _load_lock: ClassVar[threading.Lock] = threading.Lock()
+
+    def __init__(self):
+        self.model = None
+        self.model_size = "default"
+        self._device = None
+        self._model_load_lock = asyncio.Lock()
+
+    def _get_device(self) -> str:
+        """Get the best available device. Forces CPU on macOS (MPS issue)."""
+        if platform.system() == "Darwin":
+            return "cpu"
+        try:
+            import torch
+
+            if torch.cuda.is_available():
+                return "cuda"
+        except ImportError:
+            pass
+        return "cpu"
+
+    def is_loaded(self) -> bool:
+        return self.model is not None
+
+    def _get_model_path(self, model_size: str = "default") -> str:
+        return CHATTERBOX_HF_REPO
+
+    def _is_model_cached(self, model_size: str = "default") -> bool:
+        """Check if the Chatterbox multilingual model is cached locally."""
+        try:
+            from huggingface_hub import constants as hf_constants
+
+            repo_cache = Path(hf_constants.HF_HUB_CACHE) / (
+                "models--" + CHATTERBOX_HF_REPO.replace("/", "--")
+            )
+
+            if not repo_cache.exists():
+                return False
+
+            blobs_dir = repo_cache / "blobs"
+            if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
+                return False
+
+            # Check for multilingual weight files
+            snapshots_dir = repo_cache / "snapshots"
+            if snapshots_dir.exists():
+                for fname in _MTL_WEIGHT_FILES:
+                    if not any(snapshots_dir.rglob(fname)):
+                        return False
+                return True
+
+            return False
+        except Exception as e:
+            logger.warning(f"Error checking Chatterbox cache: {e}")
+            return False
+
+    async def load_model(self, model_size: str = "default") -> None:
+        """Load the Chatterbox multilingual model."""
+        if self.model is not None:
+            return
+        async with self._model_load_lock:
+            if self.model is not None:
+                return
+            await asyncio.to_thread(self._load_model_sync)
+
+    def _load_model_sync(self):
+        """Synchronous model loading."""
+        progress_manager = get_progress_manager()
+        task_manager = get_task_manager()
+        model_name = "chatterbox-tts"
+
+        is_cached = self._is_model_cached()
+
+        try:
+            progress_callback = create_hf_progress_callback(model_name, progress_manager)
+            tracker = HFProgressTracker(progress_callback, filter_non_downloads=is_cached)
+
+            if not is_cached:
+                task_manager.start_download(model_name)
+                progress_manager.update_progress(
+                    model_name=model_name,
+                    current=0,
+                    total=0,
+                    filename="Downloading Chatterbox model...",
+                    status="downloading",
+                )
+
+            with tracker.patch_download():
+                device = self._get_device()
+                self._device = device
+
+                logger.info(f"Loading Chatterbox Multilingual TTS on {device}...")
+
+                import torch
+                from chatterbox.mtl_tts import ChatterboxMultilingualTTS
+
+                # Monkey-patch torch.load for CPU loading. The model's .pt files
+                # were saved on CUDA; from_pretrained() doesn't pass map_location
+                # so loading on CPU fails without this.
+                if device == "cpu":
+                    _orig_torch_load = torch.load
+
+                    def _patched_load(*args, **kwargs):
+                        kwargs.setdefault("map_location", "cpu")
+                        return _orig_torch_load(*args, **kwargs)
+
+                    with ChatterboxTTSBackend._load_lock:
+                        torch.load = _patched_load
+                        try:
+                            self.model = ChatterboxMultilingualTTS.from_pretrained(
+                                device=device,
+                            )
+                        finally:
+                            torch.load = _orig_torch_load
+                else:
+                    self.model = ChatterboxMultilingualTTS.from_pretrained(
+                        device=device,
+                    )
+
+                # Fix: transformers >= 4.36 defaults LlamaModel to sdpa attention
+                # which doesn't support output_attentions=True (needed by
+                # Chatterbox's AlignmentStreamAnalyzer). Force eager attention.
+                t3_tfmr = self.model.t3.tfmr
+                if hasattr(t3_tfmr, "config") and hasattr(
+                    t3_tfmr.config, "_attn_implementation"
+                ):
+                    t3_tfmr.config._attn_implementation = "eager"
+                    for layer in getattr(t3_tfmr, "layers", []):
+                        if hasattr(layer, "self_attn"):
+                            layer.self_attn._attn_implementation = "eager"
+
+            if not is_cached:
+                progress_manager.mark_complete(model_name)
+                task_manager.complete_download(model_name)
+
+            logger.info("Chatterbox Multilingual TTS loaded successfully")
+
+        except ImportError as e:
+            logger.error(
+                "chatterbox-tts package not found. "
+                "Install with: pip install chatterbox-tts"
+            )
+            progress_manager.mark_error(model_name, str(e))
+            task_manager.error_download(model_name, str(e))
+            raise
+        except Exception as e:
+            logger.error(f"Failed to load Chatterbox: {e}")
+            progress_manager.mark_error(model_name, str(e))
+            task_manager.error_download(model_name, str(e))
+            raise
+
+    def unload_model(self) -> None:
+        """Unload model to free memory."""
+        if self.model is not None:
+            device = self._device
+            del self.model
+            self.model = None
+            self._device = None
+            if device == "cuda":
+                import torch
+
+                torch.cuda.empty_cache()
+            logger.info("Chatterbox unloaded")
+
+    async def create_voice_prompt(
+        self,
+        audio_path: str,
+        reference_text: str,
+        use_cache: bool = True,
+    ) -> Tuple[dict, bool]:
+        """
+        Create voice prompt from reference audio.
+
+        Chatterbox processes reference audio at generation time, so the
+        prompt just stores the file path. The actual audio is loaded by
+        model.generate() via audio_prompt_path.
+        """
+        voice_prompt = {
+            "ref_audio": str(audio_path),
+            "ref_text": reference_text,
+        }
+        return voice_prompt, False
+
+    async def combine_voice_prompts(
+        self,
+        audio_paths: List[str],
+        reference_texts: List[str],
+    ) -> Tuple[np.ndarray, str]:
+        """Combine multiple reference samples."""
+        combined_audio = []
+        for path in audio_paths:
+            audio, _sr = load_audio(path)
+            audio = normalize_audio(audio)
+            combined_audio.append(audio)
+
+        mixed = np.concatenate(combined_audio)
+        mixed = normalize_audio(mixed)
+        combined_text = " ".join(reference_texts)
+        return mixed, combined_text
+
+    # Per-language generation defaults. Lower temp + higher cfg = clearer speech.
+    _LANG_DEFAULTS: ClassVar[dict] = {
+        "he": {
+            "exaggeration": 0.4,
+            "cfg_weight": 0.7,
+            "temperature": 0.65,
+            "repetition_penalty": 2.5,
+        },
+    }
+    _GLOBAL_DEFAULTS: ClassVar[dict] = {
+        "exaggeration": 0.5,
+        "cfg_weight": 0.5,
+        "temperature": 0.8,
+        "repetition_penalty": 2.0,
+    }
+
+    async def generate(
+        self,
+        text: str,
+        voice_prompt: dict,
+        language: str = "en",
+        seed: Optional[int] = None,
+        instruct: Optional[str] = None,
+    ) -> Tuple[np.ndarray, int]:
+        """
+        Generate audio using Chatterbox Multilingual TTS.
+
+        Args:
+            text: Text to synthesize
+            voice_prompt: Dict with ref_audio path
+            language: BCP-47 language code
+            seed: Random seed for reproducibility
+            instruct: Unused (protocol compatibility)
+
+        Returns:
+            Tuple of (audio_array, sample_rate)
+        """
+        await self.load_model()
+
+        ref_audio = voice_prompt.get("ref_audio")
+        if ref_audio and not Path(ref_audio).exists():
+            logger.warning(f"Reference audio not found: {ref_audio}")
+            ref_audio = None
+
+        # Merge language-specific defaults with global defaults
+        lang_defaults = self._LANG_DEFAULTS.get(language, self._GLOBAL_DEFAULTS)
+
+        def _generate_sync():
+            import torch
+
+            if seed is not None:
+                torch.manual_seed(seed)
+
+            logger.info(f"[Chatterbox] Generating: lang={language}")
+
+            wav = self.model.generate(
+                text,
+                language_id=language,
+                audio_prompt_path=ref_audio,
+                exaggeration=lang_defaults["exaggeration"],
+                cfg_weight=lang_defaults["cfg_weight"],
+                temperature=lang_defaults["temperature"],
+                repetition_penalty=lang_defaults["repetition_penalty"],
+            )
+
+            # Convert tensor -> numpy
+            if isinstance(wav, torch.Tensor):
+                audio = wav.squeeze().cpu().numpy().astype(np.float32)
+            else:
+                audio = np.asarray(wav, dtype=np.float32)
+
+            sample_rate = (
+                getattr(self.model, "sr", None)
+                or getattr(self.model, "sample_rate", 24000)
+            )
+
+            return audio, sample_rate
+
+        return await asyncio.to_thread(_generate_sync)

backend/main.py

@@ -676,6 +676,29 @@ async def generate_speech(
                 )
 
             await tts_model.load_model()
+        elif engine == "chatterbox":
+            if not tts_model._is_model_cached():
+                model_name = "chatterbox-tts"
+
+                async def download_chatterbox_background():
+                    try:
+                        await tts_model.load_model()
+                    except Exception as e:
+                        task_manager.error_download(model_name, str(e))
+
+                task_manager.start_download(model_name)
+                asyncio.create_task(download_chatterbox_background())
+
+                raise HTTPException(
+                    status_code=202,
+                    detail={
+                        "message": "Chatterbox model is being downloaded. Please wait and try again.",
+                        "model_name": model_name,
+                        "downloading": True,
+                    },
+                )
+
+            await tts_model.load_model()
 
         # Create voice prompt from profile
         voice_prompt = await profiles.create_voice_prompt_for_profile(
@@ -693,6 +716,11 @@ async def generate_speech(
             data.instruct,
         )
 
+        # Trim trailing silence/hallucination for Chatterbox output
+        if engine == "chatterbox":
+            from .utils.audio import trim_tts_output
+            audio = trim_tts_output(audio, sample_rate)
+
         # Calculate duration
         duration = len(audio) / sample_rate
 
@@ -763,6 +791,13 @@ async def stream_speech(
                 detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
             )
         await tts_model.load_model()
+    elif engine == "chatterbox":
+        if not tts_model._is_model_cached():
+            raise HTTPException(
+                status_code=400,
+                detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.",
+            )
+        await tts_model.load_model()
 
     voice_prompt = await profiles.create_voice_prompt_for_profile(
         data.profile_id, db, engine=engine,
@@ -776,6 +811,11 @@ async def stream_speech(
         data.instruct,
     )
 
+    # Trim trailing silence/hallucination for Chatterbox output
+    if engine == "chatterbox":
+        from .utils.audio import trim_tts_output
+        audio = trim_tts_output(audio, sample_rate)
+
     wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
 
     async def _wav_stream():
@@ -1384,6 +1424,15 @@ async def get_model_status():
         except Exception:
             return False
 
+    # Check if Chatterbox backend is loaded
+    def check_chatterbox_loaded():
+        try:
+            from .backends import get_tts_backend_for_engine
+            backend = get_tts_backend_for_engine("chatterbox")
+            return backend.is_loaded()
+        except Exception:
+            return False
+
     model_configs = [
         {
             "model_name": "qwen-tts-1.7B",
@@ -1406,6 +1455,13 @@ async def get_model_status():
             "model_size": "default",
             "check_loaded": check_luxtts_loaded,
         },
+        {
+            "model_name": "chatterbox-tts",
+            "display_name": "Chatterbox TTS (Multilingual)",
+            "hf_repo_id": "ResembleAI/chatterbox",
+            "model_size": "default",
+            "check_loaded": check_chatterbox_loaded,
+        },
         {
             "model_name": "whisper-base",
             "display_name": "Whisper Base",
@@ -1557,6 +1613,7 @@ async def get_model_status():
             statuses.append(models.ModelStatus(
                 model_name=config["model_name"],
                 display_name=config["display_name"],
+                hf_repo_id=config["hf_repo_id"],
                 downloaded=downloaded,
                 downloading=is_downloading,
                 size_mb=size_mb,
@@ -1575,6 +1632,7 @@ async def get_model_status():
             statuses.append(models.ModelStatus(
                 model_name=config["model_name"],
                 display_name=config["display_name"],
+                hf_repo_id=config["hf_repo_id"],
                 downloaded=False,  # Assume not downloaded if check failed
                 downloading=is_downloading,
                 size_mb=None,
@@ -1606,6 +1664,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
             "model_size": "default",
             "load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
         },
+        "chatterbox-tts": {
+            "model_size": "default",
+            "load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(),
+        },
         "whisper-base": {
             "model_size": "base",
             "load_func": lambda: transcribe.get_whisper_model().load_model("base"),
@@ -1723,6 +1785,11 @@ async def delete_model(model_name: str):
             "model_size": "default",
             "model_type": "luxtts",
         },
+        "chatterbox-tts": {
+            "hf_repo_id": "ResembleAI/chatterbox",
+            "model_size": "default",
+            "model_type": "chatterbox",
+        },
         "whisper-base": {
             "hf_repo_id": "openai/whisper-base",
             "model_size": "base",
@@ -1762,6 +1829,11 @@ async def delete_model(model_name: str):
             luxtts = get_tts_backend_for_engine("luxtts")
             if luxtts.is_loaded():
                 luxtts.unload_model()
+        elif config["model_type"] == "chatterbox":
+            from .backends import get_tts_backend_for_engine
+            chatterbox = get_tts_backend_for_engine("chatterbox")
+            if chatterbox.is_loaded():
+                chatterbox.unload_model()
         elif config["model_type"] == "whisper":
             whisper_model = transcribe.get_whisper_model()
             if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:

backend/models.py

@@ -11,7 +11,7 @@ class VoiceProfileCreate(BaseModel):
     """Request model for creating a voice profile."""
     name: str = Field(..., min_length=1, max_length=100)
     description: Optional[str] = Field(None, max_length=500)
-    language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it)$")
+    language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
 
 
 class VoiceProfileResponse(BaseModel):
@@ -53,11 +53,11 @@ class GenerationRequest(BaseModel):
     """Request model for voice generation."""
     profile_id: str
     text: str = Field(..., min_length=1, max_length=5000)
-    language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it)$")
+    language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
     seed: Optional[int] = Field(None, ge=0)
     model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
     instruct: Optional[str] = Field(None, max_length=500)
-    engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts)$")
+    engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox)$")
 
 
 class GenerationResponse(BaseModel):
@@ -135,6 +135,7 @@ class ModelStatus(BaseModel):
     """Response model for model status."""
     model_name: str
     display_name: str
+    hf_repo_id: Optional[str] = None  # HuggingFace repository ID
     downloaded: bool
     downloading: bool = False  # True if download is in progress
     size_mb: Optional[float] = None

backend/requirements.txt

@@ -21,6 +21,9 @@ qwen-tts>=0.0.5
 linacodec @ git+https://github.com/ysharma3501/LinaCodec.git
 Zipvoice @ git+https://github.com/ysharma3501/LuxTTS.git
 
+# Chatterbox TTS (multilingual voice cloning, includes Hebrew)
+chatterbox-tts>=0.1.0
+
 # Audio processing
 librosa>=0.10.0
 soundfile>=0.12.0

backend/utils/audio.py

@@ -80,6 +80,95 @@ def save_audio(
     sf.write(path, audio, sample_rate)
 
 
+def trim_tts_output(
+    audio: np.ndarray,
+    sample_rate: int = 24000,
+    frame_ms: int = 20,
+    silence_threshold_db: float = -40.0,
+    min_silence_ms: int = 200,
+    max_internal_silence_ms: int = 1000,
+    fade_ms: int = 30,
+) -> np.ndarray:
+    """
+    Trim trailing silence and post-silence hallucination from TTS output.
+
+    Chatterbox sometimes produces ``[speech][silence][hallucinated noise]``.
+    This detects internal silence gaps longer than *max_internal_silence_ms*
+    and cuts the audio at that boundary, then trims trailing silence and
+    applies a short cosine fade-out.
+
+    Args:
+        audio: Input audio array (mono float32)
+        sample_rate: Sample rate in Hz
+        frame_ms: Frame size for RMS energy calculation
+        silence_threshold_db: dB threshold below which a frame is silence
+        min_silence_ms: Minimum trailing silence to keep
+        max_internal_silence_ms: Cut after any silence gap longer than this
+        fade_ms: Cosine fade-out duration in ms
+
+    Returns:
+        Trimmed audio array
+    """
+    frame_len = int(sample_rate * frame_ms / 1000)
+    if frame_len == 0 or len(audio) < frame_len:
+        return audio
+
+    n_frames = len(audio) // frame_len
+    threshold_linear = 10 ** (silence_threshold_db / 20)
+
+    # Compute per-frame RMS
+    rms = np.array(
+        [
+            np.sqrt(np.mean(audio[i * frame_len : (i + 1) * frame_len] ** 2))
+            for i in range(n_frames)
+        ]
+    )
+    is_speech = rms >= threshold_linear
+
+    # Find first speech frame
+    first_speech = 0
+    for i, s in enumerate(is_speech):
+        if s:
+            first_speech = max(0, i - 1)  # keep 1 frame padding
+            break
+
+    # Walk forward from first speech; cut at long internal silence gaps
+    max_silence_frames = int(max_internal_silence_ms / frame_ms)
+    consecutive_silence = 0
+    cut_frame = n_frames
+
+    for i in range(first_speech, n_frames):
+        if is_speech[i]:
+            consecutive_silence = 0
+        else:
+            consecutive_silence += 1
+            if consecutive_silence >= max_silence_frames:
+                cut_frame = i - consecutive_silence + 1
+                break
+
+    # Trim trailing silence from the cut point
+    min_silence_frames = int(min_silence_ms / frame_ms)
+    end_frame = cut_frame
+    while end_frame > first_speech and not is_speech[end_frame - 1]:
+        end_frame -= 1
+    # Keep a short tail
+    end_frame = min(end_frame + min_silence_frames, cut_frame)
+
+    # Convert frames back to samples
+    start_sample = first_speech * frame_len
+    end_sample = min(end_frame * frame_len, len(audio))
+
+    trimmed = audio[start_sample:end_sample].copy()
+
+    # Cosine fade-out
+    fade_samples = int(sample_rate * fade_ms / 1000)
+    if fade_samples > 0 and len(trimmed) > fade_samples:
+        fade = np.cos(np.linspace(0, np.pi / 2, fade_samples)) ** 2
+        trimmed[-fade_samples:] *= fade
+
+    return trimmed
+
+
 def validate_reference_audio(
     audio_path: str,
     min_duration: float = 2.0,