feat: add LuxTTS as second TTS engine with multi-engine support

Introduce LuxTTS (ZipVoice) alongside Qwen TTS, enabling users to choose
between engines at generation time. LuxTTS offers fast, English-focused
voice cloning at 48kHz with ~1GB VRAM.

Backend:
- Add LuxTTSBackend with encode_prompt/generate_speech integration
- Multi-engine registry (get_tts_backend_for_engine) replacing singleton
- Engine-prefixed voice prompt cache keys to avoid collisions
- Engine field on GenerationRequest (default 'qwen' for backward compat)
- Engine dispatch in /generate and /generate/stream endpoints
- LuxTTS in model status, download, and delete maps

Frontend:
- TTS Engine selector dropdown in GenerationForm (Qwen TTS / LuxTTS)
- Conditionally hide Model Size and Delivery Instructions for LuxTTS
- Engine field added to TypeScript types and Zod schema
- LuxTTS section in Model Management page

backend/backends/__init__.py

@@ -112,29 +112,57 @@ class STTBackend(Protocol):
 
 # Global backend instances
 _tts_backend: Optional[TTSBackend] = None
+_tts_backends: dict[str, TTSBackend] = {}
 _stt_backend: Optional[STTBackend] = None
 
+# Supported TTS engines
+TTS_ENGINES = {
+    "qwen": "Qwen TTS",
+    "luxtts": "LuxTTS",
+}
+
 
 def get_tts_backend() -> TTSBackend:
     """
-    Get or create TTS backend instance based on platform.
+    Get or create the default (Qwen) TTS backend instance based on platform.
     
     Returns:
         TTS backend instance (MLX or PyTorch)
     """
-    global _tts_backend
+    return get_tts_backend_for_engine("qwen")
+
+
+def get_tts_backend_for_engine(engine: str) -> TTSBackend:
+    """
+    Get or create a TTS backend for the given engine.
     
-    if _tts_backend is None:
+    Args:
+        engine: Engine name ("qwen" or "luxtts")
+    
+    Returns:
+        TTS backend instance
+    """
+    global _tts_backends
+    
+    if engine in _tts_backends:
+        return _tts_backends[engine]
+    
+    if engine == "qwen":
         backend_type = get_backend_type()
-        
         if backend_type == "mlx":
             from .mlx_backend import MLXTTSBackend
-            _tts_backend = MLXTTSBackend()
+            backend = MLXTTSBackend()
         else:
             from .pytorch_backend import PyTorchTTSBackend
-            _tts_backend = PyTorchTTSBackend()
+            backend = PyTorchTTSBackend()
+    elif engine == "luxtts":
+        from .luxtts_backend import LuxTTSBackend
+        backend = LuxTTSBackend()
+    else:
+        raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
     
-    return _tts_backend
+    _tts_backends[engine] = backend
+    return backend
 
 
 def get_stt_backend() -> STTBackend:
@@ -161,6 +189,7 @@ def get_stt_backend() -> STTBackend:
 
 def reset_backends():
     """Reset backend instances (useful for testing)."""
-    global _tts_backend, _stt_backend
+    global _tts_backend, _tts_backends, _stt_backend
     _tts_backend = None
+    _tts_backends.clear()
     _stt_backend = None

backend/backends/luxtts_backend.py

@@ -0,0 +1,264 @@
+"""
+LuxTTS backend implementation.
+
+Wraps the LuxTTS (ZipVoice) model for zero-shot voice cloning.
+~1GB VRAM, 48kHz output, 150x realtime on CPU.
+"""
+
+import asyncio
+import logging
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+
+from . import TTSBackend
+from ..utils.audio import normalize_audio, load_audio
+from ..utils.cache import get_cache_key, get_cached_voice_prompt, cache_voice_prompt
+from ..utils.progress import get_progress_manager
+from ..utils.tasks import get_task_manager
+
+logger = logging.getLogger(__name__)
+
+# HuggingFace repo for model weight detection
+LUXTTS_HF_REPO = "YatharthS/LuxTTS"
+
+
+class LuxTTSBackend:
+    """LuxTTS backend for zero-shot voice cloning."""
+
+    def __init__(self):
+        self.model = None
+        self.model_size = "default"  # LuxTTS has only one model size
+        self._device = None
+
+    def _get_device(self) -> str:
+        """Get the best available device."""
+        import torch
+
+        if torch.cuda.is_available():
+            return "cuda"
+        if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            return "mps"
+        return "cpu"
+
+    def is_loaded(self) -> bool:
+        return self.model is not None
+
+    @property
+    def device(self) -> str:
+        if self._device is None:
+            self._device = self._get_device()
+        return self._device
+
+    def _get_model_path(self, model_size: str) -> str:
+        return LUXTTS_HF_REPO
+
+    def _is_model_cached(self, model_size: str = "default") -> bool:
+        """Check if LuxTTS model weights are cached locally."""
+        try:
+            from huggingface_hub import constants as hf_constants
+
+            repo_cache = (
+                Path(hf_constants.HF_HUB_CACHE)
+                / ("models--" + LUXTTS_HF_REPO.replace("/", "--"))
+            )
+
+            if not repo_cache.exists():
+                return False
+
+            blobs_dir = repo_cache / "blobs"
+            if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
+                return False
+
+            snapshots_dir = repo_cache / "snapshots"
+            if snapshots_dir.exists():
+                has_weights = any(snapshots_dir.rglob("*.pt")) or any(
+                    snapshots_dir.rglob("*.safetensors")
+                ) or any(snapshots_dir.rglob("*.onnx")) or any(
+                    snapshots_dir.rglob("*.bin")
+                )
+                return has_weights
+
+            return False
+        except Exception as e:
+            logger.warning(f"Error checking LuxTTS cache: {e}")
+            return False
+
+    async def load_model(self, model_size: str = "default") -> None:
+        """Load the LuxTTS model."""
+        if self.model is not None:
+            return
+
+        await asyncio.to_thread(self._load_model_sync)
+
+    def _load_model_sync(self):
+        """Synchronous model loading."""
+        progress_manager = get_progress_manager()
+        task_manager = get_task_manager()
+        model_name = "luxtts"
+
+        is_cached = self._is_model_cached()
+
+        if not is_cached:
+            task_manager.start_download(model_name)
+            progress_manager.update_progress(
+                model_name=model_name,
+                current=0,
+                total=0,
+                filename="Downloading LuxTTS model...",
+                status="downloading",
+            )
+
+        try:
+            from zipvoice.luxvoice import LuxTTS
+
+            device = self.device
+            logger.info(f"Loading LuxTTS on {device}...")
+
+            # LuxTTS constructor downloads model and loads everything
+            if device == "cpu":
+                import os
+                threads = os.cpu_count() or 4
+                self.model = LuxTTS(
+                    model_path=LUXTTS_HF_REPO,
+                    device="cpu",
+                    threads=min(threads, 8),
+                )
+            else:
+                self.model = LuxTTS(
+                    model_path=LUXTTS_HF_REPO,
+                    device=device,
+                )
+
+            if not is_cached:
+                progress_manager.mark_complete(model_name)
+                task_manager.complete_download(model_name)
+
+            logger.info("LuxTTS loaded successfully")
+
+        except Exception as e:
+            logger.error(f"Failed to load LuxTTS: {e}")
+            if not is_cached:
+                progress_manager.mark_error(model_name, str(e))
+                task_manager.error_download(model_name, str(e))
+            raise
+
+    def unload_model(self) -> None:
+        """Unload model to free memory."""
+        if self.model is not None:
+            del self.model
+            self.model = None
+
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
+            logger.info("LuxTTS unloaded")
+
+    async def create_voice_prompt(
+        self,
+        audio_path: str,
+        reference_text: str,
+        use_cache: bool = True,
+    ) -> Tuple[dict, bool]:
+        """
+        Create voice prompt from reference audio.
+
+        LuxTTS uses its own encode_prompt() which runs Whisper ASR internally
+        to transcribe the reference. The reference_text parameter is not used
+        by LuxTTS itself, but we include it in the cache key for consistency.
+        """
+        await self.load_model()
+
+        if use_cache:
+            # Include "luxtts" in the cache key so it doesn't collide with Qwen prompts
+            cache_key = "luxtts_" + get_cache_key(audio_path, reference_text)
+            cached = get_cached_voice_prompt(cache_key)
+            if cached is not None and isinstance(cached, dict):
+                return cached, True
+
+        def _encode_sync():
+            return self.model.encode_prompt(
+                prompt_audio=str(audio_path),
+                duration=5,
+                rms=0.01,
+            )
+
+        encoded = await asyncio.to_thread(_encode_sync)
+
+        if use_cache:
+            cache_key = "luxtts_" + get_cache_key(audio_path, reference_text)
+            cache_voice_prompt(cache_key, encoded)
+
+        return encoded, False
+
+    async def combine_voice_prompts(
+        self,
+        audio_paths: List[str],
+        reference_texts: List[str],
+    ) -> Tuple[np.ndarray, str]:
+        """
+        Combine multiple reference samples.
+
+        LuxTTS doesn't have native multi-prompt support, so we concatenate
+        the audio and let encode_prompt handle the combined clip.
+        """
+        combined_audio = []
+        for path in audio_paths:
+            audio, sr = load_audio(path, sample_rate=24000)
+            audio = normalize_audio(audio)
+            combined_audio.append(audio)
+
+        mixed = np.concatenate(combined_audio)
+        mixed = normalize_audio(mixed)
+        combined_text = " ".join(reference_texts)
+
+        return mixed, combined_text
+
+    async def generate(
+        self,
+        text: str,
+        voice_prompt: dict,
+        language: str = "en",
+        seed: Optional[int] = None,
+        instruct: Optional[str] = None,
+    ) -> Tuple[np.ndarray, int]:
+        """
+        Generate audio from text using LuxTTS.
+
+        Args:
+            text: Text to synthesize
+            voice_prompt: Encoded prompt dict from encode_prompt()
+            language: Language code (LuxTTS is English-focused)
+            seed: Random seed for reproducibility
+            instruct: Not supported by LuxTTS (ignored)
+
+        Returns:
+            Tuple of (audio_array, sample_rate)
+        """
+        await self.load_model()
+
+        def _generate_sync():
+            import torch
+
+            if seed is not None:
+                torch.manual_seed(seed)
+                if torch.cuda.is_available():
+                    torch.cuda.manual_seed(seed)
+
+            wav = self.model.generate_speech(
+                text=text,
+                encode_dict=voice_prompt,
+                num_steps=4,
+                guidance_scale=3.0,
+                t_shift=0.5,
+                speed=1.0,
+                return_smooth=False,  # 48kHz output
+            )
+
+            # LuxTTS returns a tensor, convert to numpy
+            audio = wav.numpy().squeeze()
+            return audio, 48000
+
+        return await asyncio.to_thread(_generate_sync)

backend/main.py

@@ -602,47 +602,69 @@ async def generate_speech(
             raise HTTPException(status_code=404, detail="Profile not found")
         
         # Generate audio
+        from .backends import get_tts_backend_for_engine
 
-        # Resolve model size and load the correct model FIRST.
-        # This must happen before create_voice_prompt_for_profile because that
-        # function calls load_model_async(None), which falls back to self.model_size.
-        # If the model is already loaded with the right size at that point, it
-        # returns immediately and the voice prompt is created by the correct model.
-        tts_model = tts.get_tts_model()
+        engine = data.engine or "qwen"
+        tts_model = get_tts_backend_for_engine(engine)
+
+        # Resolve model size (only relevant for Qwen engine)
         model_size = data.model_size or "1.7B"
 
         # Check if model needs to be downloaded first
-        model_path = tts_model._get_model_path(model_size)
-        if not tts_model._is_model_cached(model_size):
-            # Model is not fully cached — kick off a background download and tell
-            # the client to retry once it's ready.
-            model_name = f"qwen-tts-{model_size}"
+        if engine == "qwen":
+            if not tts_model._is_model_cached(model_size):
+                model_name = f"qwen-tts-{model_size}"
 
-            async def download_model_background():
-                try:
-                    await tts_model.load_model_async(model_size)
-                except Exception as e:
-                    task_manager.error_download(model_name, str(e))
+                async def download_model_background():
+                    try:
+                        await tts_model.load_model_async(model_size)
+                    except Exception as e:
+                        task_manager.error_download(model_name, str(e))
 
-            task_manager.start_download(model_name)
-            asyncio.create_task(download_model_background())
+                task_manager.start_download(model_name)
+                asyncio.create_task(download_model_background())
 
-            raise HTTPException(
-                status_code=202,
-                detail={
-                    "message": f"Model {model_size} is being downloaded. Please wait and try again.",
-                    "model_name": model_name,
-                    "downloading": True,
-                },
-            )
+                raise HTTPException(
+                    status_code=202,
+                    detail={
+                        "message": f"Model {model_size} is being downloaded. Please wait and try again.",
+                        "model_name": model_name,
+                        "downloading": True,
+                    },
+                )
 
-        # Load (or switch to) the requested model before building the voice prompt
-        await tts_model.load_model_async(model_size)
+            # Load (or switch to) the requested model
+            await tts_model.load_model_async(model_size)
+        elif engine == "luxtts":
+            if not tts_model._is_model_cached():
+                model_name = "luxtts"
 
-        # Create voice prompt from profile (model is already loaded with correct size)
+                async def download_luxtts_background():
+                    try:
+                        await tts_model.load_model()
+                    except Exception as e:
+                        task_manager.error_download(model_name, str(e))
+
+                task_manager.start_download(model_name)
+                asyncio.create_task(download_luxtts_background())
+
+                raise HTTPException(
+                    status_code=202,
+                    detail={
+                        "message": "LuxTTS model is being downloaded. Please wait and try again.",
+                        "model_name": model_name,
+                        "downloading": True,
+                    },
+                )
+
+            await tts_model.load_model()
+
+        # Create voice prompt from profile
         voice_prompt = await profiles.create_voice_prompt_for_profile(
             data.profile_id,
             db,
+            use_cache=True,
+            engine=engine,
         )
 
         audio, sample_rate = await tts_model.generate(
@@ -699,23 +721,34 @@ async def stream_speech(
     playing audio before the entire file has been received.  This endpoint
     does NOT create a history entry — use /generate for that.
     """
+    from .backends import get_tts_backend_for_engine
+
     profile = await profiles.get_profile(data.profile_id, db)
     if not profile:
         raise HTTPException(status_code=404, detail="Profile not found")
 
-    tts_model = tts.get_tts_model()
+    engine = data.engine or "qwen"
+    tts_model = get_tts_backend_for_engine(engine)
     model_size = data.model_size or "1.7B"
 
-    if not tts_model._is_model_cached(model_size):
-        raise HTTPException(
-            status_code=400,
-            detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
-        )
+    if engine == "qwen":
+        if not tts_model._is_model_cached(model_size):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
+            )
+        await tts_model.load_model_async(model_size)
+    elif engine == "luxtts":
+        if not tts_model._is_model_cached():
+            raise HTTPException(
+                status_code=400,
+                detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
+            )
+        await tts_model.load_model()
 
-    # Load the correct model before building the voice prompt (fixes issue #96)
-    await tts_model.load_model_async(model_size)
-
-    voice_prompt = await profiles.create_voice_prompt_for_profile(data.profile_id, db)
+    voice_prompt = await profiles.create_voice_prompt_for_profile(
+        data.profile_id, db, engine=engine,
+    )
 
     audio, sample_rate = await tts_model.generate(
         data.text,
@@ -1324,6 +1357,15 @@ async def get_model_status():
         whisper_medium_id = "openai/whisper-medium"
         whisper_large_id = "openai/whisper-large-v3"
     
+    # Check if LuxTTS backend is loaded
+    def check_luxtts_loaded():
+        try:
+            from .backends import get_tts_backend_for_engine
+            backend = get_tts_backend_for_engine("luxtts")
+            return backend.is_loaded()
+        except Exception:
+            return False
+
     model_configs = [
         {
             "model_name": "qwen-tts-1.7B",
@@ -1339,6 +1381,13 @@ async def get_model_status():
             "model_size": "0.6B",
             "check_loaded": lambda: check_tts_loaded("0.6B"),
         },
+        {
+            "model_name": "luxtts",
+            "display_name": "LuxTTS (Fast, CPU-friendly)",
+            "hf_repo_id": "YatharthS/LuxTTS",
+            "model_size": "default",
+            "check_loaded": check_luxtts_loaded,
+        },
         {
             "model_name": "whisper-base",
             "display_name": "Whisper Base",
@@ -1521,6 +1570,7 @@ async def get_model_status():
 async def trigger_model_download(request: models.ModelDownloadRequest):
     """Trigger download of a specific model."""
     import asyncio
+    from .backends import get_tts_backend_for_engine
     
     task_manager = get_task_manager()
     progress_manager = get_progress_manager()
@@ -1534,6 +1584,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
             "model_size": "0.6B",
             "load_func": lambda: tts.get_tts_model().load_model("0.6B"),
         },
+        "luxtts": {
+            "model_size": "default",
+            "load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
+        },
         "whisper-base": {
             "model_size": "base",
             "load_func": lambda: transcribe.get_whisper_model().load_model("base"),
@@ -1646,6 +1700,11 @@ async def delete_model(model_name: str):
             "model_size": "0.6B",
             "model_type": "tts",
         },
+        "luxtts": {
+            "hf_repo_id": "YatharthS/LuxTTS",
+            "model_size": "default",
+            "model_type": "luxtts",
+        },
         "whisper-base": {
             "hf_repo_id": "openai/whisper-base",
             "model_size": "base",
@@ -1680,6 +1739,11 @@ async def delete_model(model_name: str):
             tts_model = tts.get_tts_model()
             if tts_model.is_loaded() and tts_model.model_size == config["model_size"]:
                 tts.unload_tts_model()
+        elif config["model_type"] == "luxtts":
+            from .backends import get_tts_backend_for_engine
+            luxtts = get_tts_backend_for_engine("luxtts")
+            if luxtts.is_loaded():
+                luxtts.unload_model()
         elif config["model_type"] == "whisper":
             whisper_model = transcribe.get_whisper_model()
             if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:

backend/models.py

@@ -57,6 +57,7 @@ class GenerationRequest(BaseModel):
     seed: Optional[int] = Field(None, ge=0)
     model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
     instruct: Optional[str] = Field(None, max_length=500)
+    engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts)$")
 
 
 class GenerationResponse(BaseModel):

backend/profiles.py

@@ -327,6 +327,7 @@ async def create_voice_prompt_for_profile(
     profile_id: str,
     db: Session,
     use_cache: bool = True,
+    engine: str = "qwen",
 ) -> dict:
     """
     Create a combined voice prompt from all samples in a profile.
@@ -335,17 +336,20 @@ async def create_voice_prompt_for_profile(
         profile_id: Profile ID
         db: Database session
         use_cache: Whether to use cached prompts
+        engine: TTS engine to create prompt for ("qwen" or "luxtts")
 
     Returns:
         Voice prompt dictionary
     """
+    from .backends import get_tts_backend_for_engine
+
     # Get all samples for profile
     samples = db.query(DBProfileSample).filter_by(profile_id=profile_id).all()
 
     if not samples:
         raise ValueError(f"No samples found for profile {profile_id}")
 
-    tts_model = get_tts_model()
+    tts_model = get_tts_backend_for_engine(engine)
 
     if len(samples) == 1:
         # Single sample - use directly

backend/requirements.txt

@@ -9,11 +9,19 @@ alembic>=1.13.0
 
 # ML models
 torch>=2.1.0
-transformers>=4.36.0
+transformers>=4.36.0,<=4.57.6
 accelerate>=0.26.0
 huggingface_hub>=0.20.0
 qwen-tts>=0.0.5
 
+# LuxTTS (voice cloning engine)
+Zipvoice @ git+https://github.com/ysharma3501/LuxTTS.git
+onnxruntime>=1.16.0
+piper-phonemize>=1.1.0
+lhotse>=1.20.0
+pydub>=0.25.0
+inflect>=7.0.0
+
 # Audio processing
 librosa>=0.10.0
 soundfile>=0.12.0