@@ -305,7 +346,9 @@ export function ModelManagement() {
}}
onCancel={() => handleCancel(model.model_name)}
isDownloading={downloadingModel === model.model_name}
- isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
+ isCancelling={
+ cancelMutation.isPending && cancelMutation.variables === model.model_name
+ }
isDismissed={dismissedErrors.has(model.model_name)}
erroredDownload={erroredDownloads.get(model.model_name)}
formatSize={formatSize}
@@ -353,12 +396,16 @@ export function ModelManagement() {
{dl.error ? (
<>
{': '}
- {dl.error}
+
+ {dl.error}
+
) : (
<>
{': '}
- No error details available. Try downloading again.
+
+ No error details available. Try downloading again.
+
)}
@@ -422,21 +469,31 @@ interface ModelItemProps {
model_name: string;
display_name: string;
downloaded: boolean;
- downloading?: boolean; // From server - true if download in progress
+ downloading?: boolean; // From server - true if download in progress
size_mb?: number;
loaded: boolean;
};
onDownload: () => void;
onDelete: () => void;
onCancel: () => void;
- isDownloading: boolean; // Local state - true if user just clicked download
+ isDownloading: boolean; // Local state - true if user just clicked download
isCancelling: boolean;
isDismissed: boolean;
erroredDownload?: ActiveDownloadTask;
formatSize: (sizeMb?: number) => string;
}
-function ModelItem({ model, onDownload, onDelete, onCancel, isDownloading, isCancelling, isDismissed, erroredDownload, formatSize }: ModelItemProps) {
+function ModelItem({
+ model,
+ onDownload,
+ onDelete,
+ onCancel,
+ isDownloading,
+ isCancelling,
+ isDismissed,
+ erroredDownload,
+ formatSize,
+}: ModelItemProps) {
// Use server's downloading state OR local state (for immediate feedback before server updates)
// Suppress downloading if user just dismissed/cancelled this model
const showDownloading = (model.downloading || isDownloading) && !erroredDownload && !isDismissed;
diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts
index 0baeb52a..4041a318 100644
--- a/app/src/lib/api/types.ts
+++ b/app/src/lib/api/types.ts
@@ -34,6 +34,8 @@ export interface GenerationRequest {
language: LanguageCode;
seed?: number;
model_size?: '1.7B' | '0.6B';
+ engine?: 'qwen' | 'luxtts';
+ instruct?: string;
}
export interface GenerationResponse {
diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts
index c6fdba50..1a44b24f 100644
--- a/app/src/lib/hooks/useGenerationForm.ts
+++ b/app/src/lib/hooks/useGenerationForm.ts
@@ -16,6 +16,7 @@ const generationSchema = z.object({
seed: z.number().int().optional(),
modelSize: z.enum(['1.7B', '0.6B']).optional(),
instruct: z.string().max(500).optional(),
+ engine: z.enum(['qwen', 'luxtts']).optional(),
});
export type GenerationFormValues = z.infer;
@@ -47,6 +48,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
seed: undefined,
modelSize: '1.7B',
instruct: '',
+ engine: 'qwen',
...options.defaultValues,
},
});
@@ -67,8 +69,14 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
try {
setIsGenerating(true);
- const modelName = `qwen-tts-${data.modelSize}`;
- const displayName = data.modelSize === '1.7B' ? 'Qwen TTS 1.7B' : 'Qwen TTS 0.6B';
+ const engine = data.engine || 'qwen';
+ const modelName = engine === 'luxtts' ? 'luxtts' : `qwen-tts-${data.modelSize}`;
+ const displayName =
+ engine === 'luxtts'
+ ? 'LuxTTS'
+ : data.modelSize === '1.7B'
+ ? 'Qwen TTS 1.7B'
+ : 'Qwen TTS 0.6B';
try {
const modelStatus = await apiClient.getModelStatus();
@@ -87,8 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
text: data.text,
language: data.language,
seed: data.seed,
- model_size: data.modelSize,
- instruct: data.instruct || undefined,
+ model_size: engine === 'luxtts' ? undefined : data.modelSize,
+ engine,
+ instruct: engine === 'luxtts' ? undefined : data.instruct || undefined,
});
toast({
@@ -99,7 +108,14 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
const audioUrl = apiClient.getAudioUrl(result.id);
setAudioWithAutoPlay(audioUrl, result.id, selectedProfileId, data.text.substring(0, 50));
- form.reset();
+ form.reset({
+ text: '',
+ language: data.language,
+ seed: undefined,
+ modelSize: data.modelSize,
+ instruct: '',
+ engine: data.engine,
+ });
options.onSuccess?.(result.id);
} catch (error) {
toast({
diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py
index f7c47ba9..8f4dce0c 100644
--- a/backend/backends/__init__.py
+++ b/backend/backends/__init__.py
@@ -4,6 +4,7 @@ Backend abstraction layer for TTS and STT.
Provides a unified interface for MLX and PyTorch backends.
"""
+import threading
from typing import Protocol, Optional, Tuple, List
from typing_extensions import runtime_checkable
import numpy as np
@@ -112,29 +113,65 @@ class STTBackend(Protocol):
# Global backend instances
_tts_backend: Optional[TTSBackend] = None
+_tts_backends: dict[str, TTSBackend] = {}
+_tts_backends_lock = threading.Lock()
_stt_backend: Optional[STTBackend] = None
+# Supported TTS engines
+TTS_ENGINES = {
+ "qwen": "Qwen TTS",
+ "luxtts": "LuxTTS",
+}
+
def get_tts_backend() -> TTSBackend:
"""
- Get or create TTS backend instance based on platform.
+ Get or create the default (Qwen) TTS backend instance based on platform.
Returns:
TTS backend instance (MLX or PyTorch)
"""
- global _tts_backend
+ return get_tts_backend_for_engine("qwen")
+
+
+def get_tts_backend_for_engine(engine: str) -> TTSBackend:
+ """
+ Get or create a TTS backend for the given engine.
- if _tts_backend is None:
- backend_type = get_backend_type()
+ Args:
+ engine: Engine name ("qwen" or "luxtts")
+
+ Returns:
+ TTS backend instance
+ """
+ global _tts_backends
+
+ # Fast path: check without lock
+ if engine in _tts_backends:
+ return _tts_backends[engine]
+
+ # Slow path: create with lock to avoid duplicate instantiation
+ with _tts_backends_lock:
+ # Double-check after acquiring lock
+ if engine in _tts_backends:
+ return _tts_backends[engine]
- if backend_type == "mlx":
- from .mlx_backend import MLXTTSBackend
- _tts_backend = MLXTTSBackend()
+ if engine == "qwen":
+ backend_type = get_backend_type()
+ if backend_type == "mlx":
+ from .mlx_backend import MLXTTSBackend
+ backend = MLXTTSBackend()
+ else:
+ from .pytorch_backend import PyTorchTTSBackend
+ backend = PyTorchTTSBackend()
+ elif engine == "luxtts":
+ from .luxtts_backend import LuxTTSBackend
+ backend = LuxTTSBackend()
else:
- from .pytorch_backend import PyTorchTTSBackend
- _tts_backend = PyTorchTTSBackend()
-
- return _tts_backend
+ raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
+
+ _tts_backends[engine] = backend
+ return backend
def get_stt_backend() -> STTBackend:
@@ -161,6 +198,7 @@ def get_stt_backend() -> STTBackend:
def reset_backends():
"""Reset backend instances (useful for testing)."""
- global _tts_backend, _stt_backend
+ global _tts_backend, _tts_backends, _stt_backend
_tts_backend = None
+ _tts_backends.clear()
_stt_backend = None
diff --git a/backend/backends/luxtts_backend.py b/backend/backends/luxtts_backend.py
new file mode 100644
index 00000000..7e692139
--- /dev/null
+++ b/backend/backends/luxtts_backend.py
@@ -0,0 +1,264 @@
+"""
+LuxTTS backend implementation.
+
+Wraps the LuxTTS (ZipVoice) model for zero-shot voice cloning.
+~1GB VRAM, 48kHz output, 150x realtime on CPU.
+"""
+
+import asyncio
+import logging
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import numpy as np
+
+from . import TTSBackend
+from ..utils.audio import normalize_audio, load_audio
+from ..utils.cache import get_cache_key, get_cached_voice_prompt, cache_voice_prompt
+from ..utils.progress import get_progress_manager
+from ..utils.tasks import get_task_manager
+
+logger = logging.getLogger(__name__)
+
+# HuggingFace repo for model weight detection
+LUXTTS_HF_REPO = "YatharthS/LuxTTS"
+
+
+class LuxTTSBackend:
+ """LuxTTS backend for zero-shot voice cloning."""
+
+ def __init__(self):
+ self.model = None
+ self.model_size = "default" # LuxTTS has only one model size
+ self._device = None
+
+ def _get_device(self) -> str:
+ """Get the best available device."""
+ import torch
+
+ if torch.cuda.is_available():
+ return "cuda"
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ return "mps"
+ return "cpu"
+
+ def is_loaded(self) -> bool:
+ return self.model is not None
+
+ @property
+ def device(self) -> str:
+ if self._device is None:
+ self._device = self._get_device()
+ return self._device
+
+ def _get_model_path(self, model_size: str) -> str:
+ return LUXTTS_HF_REPO
+
+ def _is_model_cached(self, model_size: str = "default") -> bool:
+ """Check if LuxTTS model weights are cached locally."""
+ try:
+ from huggingface_hub import constants as hf_constants
+
+ repo_cache = (
+ Path(hf_constants.HF_HUB_CACHE)
+ / ("models--" + LUXTTS_HF_REPO.replace("/", "--"))
+ )
+
+ if not repo_cache.exists():
+ return False
+
+ blobs_dir = repo_cache / "blobs"
+ if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
+ return False
+
+ snapshots_dir = repo_cache / "snapshots"
+ if snapshots_dir.exists():
+ has_weights = any(snapshots_dir.rglob("*.pt")) or any(
+ snapshots_dir.rglob("*.safetensors")
+ ) or any(snapshots_dir.rglob("*.onnx")) or any(
+ snapshots_dir.rglob("*.bin")
+ )
+ return has_weights
+
+ return False
+ except Exception as e:
+ logger.warning(f"Error checking LuxTTS cache: {e}")
+ return False
+
+ async def load_model(self, model_size: str = "default") -> None:
+ """Load the LuxTTS model."""
+ if self.model is not None:
+ return
+
+ await asyncio.to_thread(self._load_model_sync)
+
+ def _load_model_sync(self):
+ """Synchronous model loading."""
+ progress_manager = get_progress_manager()
+ task_manager = get_task_manager()
+ model_name = "luxtts"
+
+ is_cached = self._is_model_cached()
+
+ if not is_cached:
+ task_manager.start_download(model_name)
+ progress_manager.update_progress(
+ model_name=model_name,
+ current=0,
+ total=0,
+ filename="Downloading LuxTTS model...",
+ status="downloading",
+ )
+
+ try:
+ from zipvoice.luxvoice import LuxTTS
+
+ device = self.device
+ logger.info(f"Loading LuxTTS on {device}...")
+
+ # LuxTTS constructor downloads model and loads everything
+ if device == "cpu":
+ import os
+ threads = os.cpu_count() or 4
+ self.model = LuxTTS(
+ model_path=LUXTTS_HF_REPO,
+ device="cpu",
+ threads=min(threads, 8),
+ )
+ else:
+ self.model = LuxTTS(
+ model_path=LUXTTS_HF_REPO,
+ device=device,
+ )
+
+ if not is_cached:
+ progress_manager.mark_complete(model_name)
+ task_manager.complete_download(model_name)
+
+ logger.info("LuxTTS loaded successfully")
+
+ except Exception as e:
+ logger.error(f"Failed to load LuxTTS: {e}")
+ if not is_cached:
+ progress_manager.mark_error(model_name, str(e))
+ task_manager.error_download(model_name, str(e))
+ raise
+
+ def unload_model(self) -> None:
+ """Unload model to free memory."""
+ if self.model is not None:
+ del self.model
+ self.model = None
+
+ import torch
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+ logger.info("LuxTTS unloaded")
+
+ async def create_voice_prompt(
+ self,
+ audio_path: str,
+ reference_text: str,
+ use_cache: bool = True,
+ ) -> Tuple[dict, bool]:
+ """
+ Create voice prompt from reference audio.
+
+ LuxTTS uses its own encode_prompt() which runs Whisper ASR internally
+ to transcribe the reference. The reference_text parameter is not used
+ by LuxTTS itself, but we include it in the cache key for consistency.
+ """
+ await self.load_model()
+
+ # Compute cache key once for both lookup and storage
+ cache_key = ("luxtts_" + get_cache_key(audio_path, reference_text)) if use_cache else None
+
+ if cache_key:
+ cached = get_cached_voice_prompt(cache_key)
+ if cached is not None and isinstance(cached, dict):
+ return cached, True
+
+ def _encode_sync():
+ return self.model.encode_prompt(
+ prompt_audio=str(audio_path),
+ duration=5,
+ rms=0.01,
+ )
+
+ encoded = await asyncio.to_thread(_encode_sync)
+
+ if cache_key:
+ cache_voice_prompt(cache_key, encoded)
+
+ return encoded, False
+
+ async def combine_voice_prompts(
+ self,
+ audio_paths: List[str],
+ reference_texts: List[str],
+ ) -> Tuple[np.ndarray, str]:
+ """
+ Combine multiple reference samples.
+
+ LuxTTS doesn't have native multi-prompt support, so we concatenate
+ the audio and let encode_prompt handle the combined clip.
+ """
+ combined_audio = []
+ for path in audio_paths:
+ audio, _sr = load_audio(path, sample_rate=24000)
+ audio = normalize_audio(audio)
+ combined_audio.append(audio)
+
+ mixed = np.concatenate(combined_audio)
+ mixed = normalize_audio(mixed)
+ combined_text = " ".join(reference_texts)
+
+ return mixed, combined_text
+
+ async def generate(
+ self,
+ text: str,
+ voice_prompt: dict,
+ language: str = "en",
+ seed: Optional[int] = None,
+ instruct: Optional[str] = None,
+ ) -> Tuple[np.ndarray, int]:
+ """
+ Generate audio from text using LuxTTS.
+
+ Args:
+ text: Text to synthesize
+ voice_prompt: Encoded prompt dict from encode_prompt()
+ language: Language code (LuxTTS is English-focused)
+ seed: Random seed for reproducibility
+ instruct: Not supported by LuxTTS (ignored)
+
+ Returns:
+ Tuple of (audio_array, sample_rate)
+ """
+ await self.load_model()
+
+ def _generate_sync():
+ import torch
+
+ if seed is not None:
+ torch.manual_seed(seed)
+ if torch.cuda.is_available():
+ torch.cuda.manual_seed(seed)
+
+ wav = self.model.generate_speech(
+ text=text,
+ encode_dict=voice_prompt,
+ num_steps=4,
+ guidance_scale=3.0,
+ t_shift=0.5,
+ speed=1.0,
+ return_smooth=False, # 48kHz output
+ )
+
+ # LuxTTS returns a tensor (may be on GPU/MPS), move to CPU first
+ audio = wav.detach().cpu().numpy().squeeze()
+ return audio, 48000
+
+ return await asyncio.to_thread(_generate_sync)
diff --git a/backend/main.py b/backend/main.py
index 4f6be44f..d4dd87c5 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -48,6 +48,18 @@ from .utils.tasks import get_task_manager
from .utils.cache import clear_voice_prompt_cache
from .platform_detect import get_backend_type
+# Keep references to fire-and-forget background tasks to prevent GC
+_background_tasks: set = set()
+
+
+def _create_background_task(coro) -> asyncio.Task:
+ """Create a background task and prevent it from being garbage collected."""
+ task = asyncio.create_task(coro)
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+ return task
+
+
app = FastAPI(
title="voicebox API",
description="Production-quality Qwen3-TTS voice cloning API",
@@ -608,47 +620,69 @@ async def generate_speech(
raise HTTPException(status_code=404, detail="Profile not found")
# Generate audio
+ from .backends import get_tts_backend_for_engine
- # Resolve model size and load the correct model FIRST.
- # This must happen before create_voice_prompt_for_profile because that
- # function calls load_model_async(None), which falls back to self.model_size.
- # If the model is already loaded with the right size at that point, it
- # returns immediately and the voice prompt is created by the correct model.
- tts_model = tts.get_tts_model()
+ engine = data.engine or "qwen"
+ tts_model = get_tts_backend_for_engine(engine)
+
+ # Resolve model size (only relevant for Qwen engine)
model_size = data.model_size or "1.7B"
# Check if model needs to be downloaded first
- model_path = tts_model._get_model_path(model_size)
- if not tts_model._is_model_cached(model_size):
- # Model is not fully cached — kick off a background download and tell
- # the client to retry once it's ready.
- model_name = f"qwen-tts-{model_size}"
+ if engine == "qwen":
+ if not tts_model._is_model_cached(model_size):
+ model_name = f"qwen-tts-{model_size}"
- async def download_model_background():
- try:
- await tts_model.load_model_async(model_size)
- except Exception as e:
- task_manager.error_download(model_name, str(e))
+ async def download_model_background():
+ try:
+ await tts_model.load_model_async(model_size)
+ except Exception as e:
+ task_manager.error_download(model_name, str(e))
- task_manager.start_download(model_name)
- asyncio.create_task(download_model_background())
+ task_manager.start_download(model_name)
+ _create_background_task(download_model_background())
- raise HTTPException(
- status_code=202,
- detail={
- "message": f"Model {model_size} is being downloaded. Please wait and try again.",
- "model_name": model_name,
- "downloading": True,
- },
- )
+ raise HTTPException(
+ status_code=202,
+ detail={
+ "message": f"Model {model_size} is being downloaded. Please wait and try again.",
+ "model_name": model_name,
+ "downloading": True,
+ },
+ )
- # Load (or switch to) the requested model before building the voice prompt
- await tts_model.load_model_async(model_size)
+ # Load (or switch to) the requested model
+ await tts_model.load_model_async(model_size)
+ elif engine == "luxtts":
+ if not tts_model._is_model_cached():
+ model_name = "luxtts"
- # Create voice prompt from profile (model is already loaded with correct size)
+ async def download_luxtts_background():
+ try:
+ await tts_model.load_model()
+ except Exception as e:
+ task_manager.error_download(model_name, str(e))
+
+ task_manager.start_download(model_name)
+ _create_background_task(download_luxtts_background())
+
+ raise HTTPException(
+ status_code=202,
+ detail={
+ "message": "LuxTTS model is being downloaded. Please wait and try again.",
+ "model_name": model_name,
+ "downloading": True,
+ },
+ )
+
+ await tts_model.load_model()
+
+ # Create voice prompt from profile
voice_prompt = await profiles.create_voice_prompt_for_profile(
data.profile_id,
db,
+ use_cache=True,
+ engine=engine,
)
audio, sample_rate = await tts_model.generate(
@@ -705,23 +739,34 @@ async def stream_speech(
playing audio before the entire file has been received. This endpoint
does NOT create a history entry — use /generate for that.
"""
+ from .backends import get_tts_backend_for_engine
+
profile = await profiles.get_profile(data.profile_id, db)
if not profile:
raise HTTPException(status_code=404, detail="Profile not found")
- tts_model = tts.get_tts_model()
+ engine = data.engine or "qwen"
+ tts_model = get_tts_backend_for_engine(engine)
model_size = data.model_size or "1.7B"
- if not tts_model._is_model_cached(model_size):
- raise HTTPException(
- status_code=400,
- detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
- )
+ if engine == "qwen":
+ if not tts_model._is_model_cached(model_size):
+ raise HTTPException(
+ status_code=400,
+ detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
+ )
+ await tts_model.load_model_async(model_size)
+ elif engine == "luxtts":
+ if not tts_model._is_model_cached():
+ raise HTTPException(
+ status_code=400,
+ detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
+ )
+ await tts_model.load_model()
- # Load the correct model before building the voice prompt (fixes issue #96)
- await tts_model.load_model_async(model_size)
-
- voice_prompt = await profiles.create_voice_prompt_for_profile(data.profile_id, db)
+ voice_prompt = await profiles.create_voice_prompt_for_profile(
+ data.profile_id, db, engine=engine,
+ )
audio, sample_rate = await tts_model.generate(
data.text,
@@ -959,7 +1004,7 @@ async def transcribe_audio(
get_task_manager().error_download(progress_model_name, str(e))
get_task_manager().start_download(progress_model_name)
- asyncio.create_task(download_whisper_background())
+ _create_background_task(download_whisper_background())
# Return 202 Accepted
raise HTTPException(
@@ -1330,6 +1375,15 @@ async def get_model_status():
whisper_medium_id = "openai/whisper-medium"
whisper_large_id = "openai/whisper-large-v3"
+ # Check if LuxTTS backend is loaded
+ def check_luxtts_loaded():
+ try:
+ from .backends import get_tts_backend_for_engine
+ backend = get_tts_backend_for_engine("luxtts")
+ return backend.is_loaded()
+ except Exception:
+ return False
+
model_configs = [
{
"model_name": "qwen-tts-1.7B",
@@ -1345,6 +1399,13 @@ async def get_model_status():
"model_size": "0.6B",
"check_loaded": lambda: check_tts_loaded("0.6B"),
},
+ {
+ "model_name": "luxtts",
+ "display_name": "LuxTTS (Fast, CPU-friendly)",
+ "hf_repo_id": "YatharthS/LuxTTS",
+ "model_size": "default",
+ "check_loaded": check_luxtts_loaded,
+ },
{
"model_name": "whisper-base",
"display_name": "Whisper Base",
@@ -1527,6 +1588,7 @@ async def get_model_status():
async def trigger_model_download(request: models.ModelDownloadRequest):
"""Trigger download of a specific model."""
import asyncio
+ from .backends import get_tts_backend_for_engine
task_manager = get_task_manager()
progress_manager = get_progress_manager()
@@ -1540,6 +1602,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
"model_size": "0.6B",
"load_func": lambda: tts.get_tts_model().load_model("0.6B"),
},
+ "luxtts": {
+ "model_size": "default",
+ "load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
+ },
"whisper-base": {
"model_size": "base",
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
@@ -1591,7 +1657,7 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
)
# Start download in background task (don't await)
- asyncio.create_task(download_in_background())
+ _create_background_task(download_in_background())
# Return immediately - frontend should poll progress endpoint
return {"message": f"Model {request.model_name} download started"}
@@ -1652,6 +1718,11 @@ async def delete_model(model_name: str):
"model_size": "0.6B",
"model_type": "tts",
},
+ "luxtts": {
+ "hf_repo_id": "YatharthS/LuxTTS",
+ "model_size": "default",
+ "model_type": "luxtts",
+ },
"whisper-base": {
"hf_repo_id": "openai/whisper-base",
"model_size": "base",
@@ -1686,6 +1757,11 @@ async def delete_model(model_name: str):
tts_model = tts.get_tts_model()
if tts_model.is_loaded() and tts_model.model_size == config["model_size"]:
tts.unload_tts_model()
+ elif config["model_type"] == "luxtts":
+ from .backends import get_tts_backend_for_engine
+ luxtts = get_tts_backend_for_engine("luxtts")
+ if luxtts.is_loaded():
+ luxtts.unload_model()
elif config["model_type"] == "whisper":
whisper_model = transcribe.get_whisper_model()
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:
@@ -1831,7 +1907,7 @@ async def download_cuda_backend():
import logging
logging.getLogger(__name__).error(f"CUDA download failed: {e}")
- asyncio.create_task(_download())
+ _create_background_task(_download())
return {"message": "CUDA backend download started", "progress_key": "cuda-backend"}
diff --git a/backend/models.py b/backend/models.py
index 3d8261b2..9ecf4510 100644
--- a/backend/models.py
+++ b/backend/models.py
@@ -57,6 +57,7 @@ class GenerationRequest(BaseModel):
seed: Optional[int] = Field(None, ge=0)
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
instruct: Optional[str] = Field(None, max_length=500)
+ engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts)$")
class GenerationResponse(BaseModel):
diff --git a/backend/profiles.py b/backend/profiles.py
index 7b4cc931..86f9bf59 100644
--- a/backend/profiles.py
+++ b/backend/profiles.py
@@ -344,6 +344,7 @@ async def create_voice_prompt_for_profile(
profile_id: str,
db: Session,
use_cache: bool = True,
+ engine: str = "qwen",
) -> dict:
"""
Create a combined voice prompt from all samples in a profile.
@@ -352,17 +353,20 @@ async def create_voice_prompt_for_profile(
profile_id: Profile ID
db: Database session
use_cache: Whether to use cached prompts
+ engine: TTS engine to create prompt for ("qwen" or "luxtts")
Returns:
Voice prompt dictionary
"""
+ from .backends import get_tts_backend_for_engine
+
# Get all samples for profile
samples = db.query(DBProfileSample).filter_by(profile_id=profile_id).all()
if not samples:
raise ValueError(f"No samples found for profile {profile_id}")
- tts_model = get_tts_model()
+ tts_model = get_tts_backend_for_engine(engine)
if len(samples) == 1:
# Single sample - use directly
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 661bf6de..e57ddd94 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -9,11 +9,18 @@ alembic>=1.13.0
# ML models
torch>=2.1.0
-transformers>=4.36.0
+transformers>=4.36.0,<=4.57.6
accelerate>=0.26.0
huggingface_hub>=0.20.0
qwen-tts>=0.0.5
+# LuxTTS (voice cloning engine)
+# piper-phonemize needs custom index (no PyPI wheels)
+--find-links https://k2-fsa.github.io/icefall/piper_phonemize.html
+# linacodec is a git-only dep of Zipvoice (uv-only source, pip can't resolve it)
+linacodec @ git+https://github.com/ysharma3501/LinaCodec.git
+Zipvoice @ git+https://github.com/ysharma3501/LuxTTS.git
+
# Audio processing
librosa>=0.10.0
soundfile>=0.12.0
diff --git a/justfile b/justfile
new file mode 100644
index 00000000..b0e1981f
--- /dev/null
+++ b/justfile
@@ -0,0 +1,189 @@
+# Voicebox development commands
+# Install: brew install just (or cargo install just)
+# Usage: just --list
+
+# Directories
+backend_dir := "backend"
+tauri_dir := "tauri"
+app_dir := "app"
+web_dir := "web"
+venv := backend_dir / "venv"
+venv_bin := venv / "bin"
+python := venv_bin / "python"
+pip := venv_bin / "pip"
+
+# Detect best python for venv creation
+system_python := `command -v python3.12 2>/dev/null || command -v python3.13 2>/dev/null || echo python3`
+
+# ─── Setup ────────────────────────────────────────────────────────────
+
+# Full project setup (python venv + JS deps + dev sidecar)
+setup: setup-python setup-js
+ @echo ""
+ @echo "Setup complete! Run: just dev"
+
+# Create venv and install Python dependencies
+setup-python:
+ #!/usr/bin/env bash
+ set -euo pipefail
+ if [ ! -d "{{ venv }}" ]; then
+ echo "Creating Python virtual environment..."
+ PY_MINOR=$({{ system_python }} -c "import sys; print(sys.version_info[1])")
+ if [ "$PY_MINOR" -gt 13 ]; then
+ echo "Warning: Python 3.$PY_MINOR detected. ML packages may not be compatible."
+ echo "Recommended: brew install python@3.12"
+ fi
+ {{ system_python }} -m venv {{ venv }}
+ fi
+ echo "Installing Python dependencies..."
+ {{ pip }} install --upgrade pip -q
+ {{ pip }} install -r {{ backend_dir }}/requirements.txt
+ # Apple Silicon: install MLX backend
+ if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
+ echo "Detected Apple Silicon — installing MLX dependencies..."
+ {{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt
+ fi
+ {{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git
+ echo "Python environment ready."
+
+# Install JavaScript dependencies
+setup-js:
+ bun install
+
+# ─── Development ──────────────────────────────────────────────────────
+
+# Start backend + frontend for development (two processes, one terminal)
+dev: _ensure-venv _ensure-sidecar
+ #!/usr/bin/env bash
+ set -euo pipefail
+ trap 'kill 0' EXIT
+
+ echo "Starting backend on http://localhost:17493 ..."
+ {{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 &
+ sleep 2
+
+ echo "Starting Tauri desktop app..."
+ cd {{ tauri_dir }} && bun run tauri dev &
+
+ wait
+
+# Start backend only
+dev-backend: _ensure-venv
+ {{ venv_bin }}/uvicorn backend.main:app --reload --port 17493
+
+# Start Tauri desktop app only (backend must be running separately)
+dev-frontend: _ensure-sidecar
+ cd {{ tauri_dir }} && bun run tauri dev
+
+# Start backend + web app (no Tauri)
+dev-web: _ensure-venv
+ #!/usr/bin/env bash
+ set -euo pipefail
+ trap 'kill 0' EXIT
+ {{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 &
+ sleep 2
+ cd {{ web_dir }} && bun run dev &
+ wait
+
+# Kill all dev processes
+kill:
+ -pkill -f "uvicorn backend.main:app" 2>/dev/null || true
+ -pkill -f "vite" 2>/dev/null || true
+ @echo "Dev processes killed."
+
+# ─── Build ────────────────────────────────────────────────────────────
+
+# Build everything (server binary + desktop app)
+build: build-server build-tauri
+
+# Build Python server binary
+build-server: _ensure-venv
+ PATH="{{ venv_bin }}:$PATH" ./scripts/build-server.sh
+
+# Build Tauri desktop app
+build-tauri:
+ cd {{ tauri_dir }} && bun run tauri build
+
+# Build web app
+build-web:
+ cd {{ web_dir }} && bun run build
+
+# ─── Code Quality ────────────────────────────────────────────────────
+
+# Run all checks (lint + format + typecheck)
+check:
+ bun run check
+
+# Lint with Biome
+lint:
+ bun run lint
+
+# Format with Biome
+format:
+ bun run format
+
+# Fix lint + format issues
+fix:
+ bun run check:fix
+
+# ─── Database ─────────────────────────────────────────────────────────
+
+# Initialize SQLite database
+db-init: _ensure-venv
+ cd {{ backend_dir }} && {{ python }} -c "from database import init_db; init_db()"
+
+# Reset database (delete + reinit)
+db-reset:
+ rm -f {{ backend_dir }}/data/voicebox.db
+ just db-init
+
+# ─── Utilities ────────────────────────────────────────────────────────
+
+# Generate TypeScript API client (backend must be running)
+generate-api:
+ ./scripts/generate-api.sh
+
+# Open API docs in browser
+docs:
+ open http://localhost:17493/docs 2>/dev/null || xdg-open http://localhost:17493/docs
+
+# Tail backend logs
+logs:
+ tail -f {{ backend_dir }}/logs/*.log 2>/dev/null || echo "No log files found"
+
+# ─── Clean ────────────────────────────────────────────────────────────
+
+# Clean build artifacts
+clean:
+ rm -rf {{ tauri_dir }}/src-tauri/target/release
+ rm -rf {{ web_dir }}/dist
+ rm -rf {{ app_dir }}/dist
+
+# Clean Python venv and cache
+clean-python:
+ rm -rf {{ venv }}
+ find {{ backend_dir }} -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+
+# Nuclear clean (everything including node_modules)
+clean-all: clean clean-python
+ rm -rf node_modules
+ rm -rf {{ app_dir }}/node_modules
+ rm -rf {{ tauri_dir }}/node_modules
+ rm -rf {{ web_dir }}/node_modules
+ cd {{ tauri_dir }}/src-tauri && cargo clean
+
+# ─── Internal ─────────────────────────────────────────────────────────
+
+# Ensure venv exists (prompt to run setup if not)
+[private]
+_ensure-venv:
+ #!/usr/bin/env bash
+ if [ ! -d "{{ venv }}" ]; then
+ echo "Python venv not found. Run: just setup"
+ exit 1
+ fi
+
+# Ensure Tauri dev sidecar placeholder exists
+[private]
+_ensure-sidecar:
+ bun run setup:dev
diff --git a/scripts/setup-dev-sidecar.js b/scripts/setup-dev-sidecar.js
index 6d5d5524..0fb9e327 100644
--- a/scripts/setup-dev-sidecar.js
+++ b/scripts/setup-dev-sidecar.js
@@ -1,4 +1,5 @@
#!/usr/bin/env node
+
/**
* Creates placeholder sidecar binaries for development mode.
*
@@ -9,10 +10,10 @@
* The actual server should be started separately with `bun run dev:server`.
*/
-import { existsSync, mkdirSync, writeFileSync, statSync } from 'fs';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
import { execSync } from 'child_process';
+import { existsSync, mkdirSync, statSync, writeFileSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -55,7 +56,9 @@ function createPlaceholderBinary(targetTriple) {
try {
const stats = statSync(binaryPath);
if (stats.size > MIN_REAL_BINARY_SIZE) {
- console.log(`Real binary already exists: ${binaryName} (${(stats.size / 1024 / 1024).toFixed(1)} MB)`);
+ console.log(
+ `Real binary already exists: ${binaryName} (${(stats.size / 1024 / 1024).toFixed(1)} MB)`,
+ );
return;
}
} catch {
@@ -73,52 +76,275 @@ function createPlaceholderBinary(targetTriple) {
// This is the smallest valid PE that Windows will accept
const minimalPE = Buffer.from([
// DOS Header
- 0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00,
- 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
+ 0x4d,
+ 0x5a,
+ 0x90,
+ 0x00,
+ 0x03,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x04,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0xff,
+ 0xff,
+ 0x00,
+ 0x00,
+ 0xb8,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x40,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x80,
+ 0x00,
+ 0x00,
+ 0x00,
// DOS Stub
- 0x0E, 0x1F, 0xBA, 0x0E, 0x00, 0xB4, 0x09, 0xCD, 0x21, 0xB8, 0x01, 0x4C, 0xCD, 0x21, 0x54, 0x68,
- 0x69, 0x73, 0x20, 0x70, 0x72, 0x6F, 0x67, 0x72, 0x61, 0x6D, 0x20, 0x63, 0x61, 0x6E, 0x6E, 0x6F,
- 0x74, 0x20, 0x62, 0x65, 0x20, 0x72, 0x75, 0x6E, 0x20, 0x69, 0x6E, 0x20, 0x44, 0x4F, 0x53, 0x20,
- 0x6D, 0x6F, 0x64, 0x65, 0x2E, 0x0D, 0x0D, 0x0A, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x0e,
+ 0x1f,
+ 0xba,
+ 0x0e,
+ 0x00,
+ 0xb4,
+ 0x09,
+ 0xcd,
+ 0x21,
+ 0xb8,
+ 0x01,
+ 0x4c,
+ 0xcd,
+ 0x21,
+ 0x54,
+ 0x68,
+ 0x69,
+ 0x73,
+ 0x20,
+ 0x70,
+ 0x72,
+ 0x6f,
+ 0x67,
+ 0x72,
+ 0x61,
+ 0x6d,
+ 0x20,
+ 0x63,
+ 0x61,
+ 0x6e,
+ 0x6e,
+ 0x6f,
+ 0x74,
+ 0x20,
+ 0x62,
+ 0x65,
+ 0x20,
+ 0x72,
+ 0x75,
+ 0x6e,
+ 0x20,
+ 0x69,
+ 0x6e,
+ 0x20,
+ 0x44,
+ 0x4f,
+ 0x53,
+ 0x20,
+ 0x6d,
+ 0x6f,
+ 0x64,
+ 0x65,
+ 0x2e,
+ 0x0d,
+ 0x0d,
+ 0x0a,
+ 0x24,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
// PE Signature
- 0x50, 0x45, 0x00, 0x00,
+ 0x50,
+ 0x45,
+ 0x00,
+ 0x00,
// COFF Header (x64)
- 0x64, 0x86, // Machine: AMD64
- 0x01, 0x00, // NumberOfSections: 1
- 0x00, 0x00, 0x00, 0x00, // TimeDateStamp
- 0x00, 0x00, 0x00, 0x00, // PointerToSymbolTable
- 0x00, 0x00, 0x00, 0x00, // NumberOfSymbols
- 0xF0, 0x00, // SizeOfOptionalHeader
- 0x22, 0x00, // Characteristics: EXECUTABLE_IMAGE | LARGE_ADDRESS_AWARE
+ 0x64,
+ 0x86, // Machine: AMD64
+ 0x01,
+ 0x00, // NumberOfSections: 1
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // TimeDateStamp
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // PointerToSymbolTable
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // NumberOfSymbols
+ 0xf0,
+ 0x00, // SizeOfOptionalHeader
+ 0x22,
+ 0x00, // Characteristics: EXECUTABLE_IMAGE | LARGE_ADDRESS_AWARE
// Optional Header (PE32+)
- 0x0B, 0x02, // Magic: PE32+
- 0x00, 0x00, // Linker version
- 0x00, 0x00, 0x00, 0x00, // SizeOfCode
- 0x00, 0x00, 0x00, 0x00, // SizeOfInitializedData
- 0x00, 0x00, 0x00, 0x00, // SizeOfUninitializedData
- 0x00, 0x10, 0x00, 0x00, // AddressOfEntryPoint
- 0x00, 0x00, 0x00, 0x00, // BaseOfCode
- 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, // ImageBase
- 0x00, 0x10, 0x00, 0x00, // SectionAlignment
- 0x00, 0x02, 0x00, 0x00, // FileAlignment
- 0x06, 0x00, 0x00, 0x00, // OS version
- 0x00, 0x00, 0x00, 0x00, // Image version
- 0x06, 0x00, 0x00, 0x00, // Subsystem version
- 0x00, 0x00, 0x00, 0x00, // Win32VersionValue
- 0x00, 0x20, 0x00, 0x00, // SizeOfImage
- 0x00, 0x02, 0x00, 0x00, // SizeOfHeaders
- 0x00, 0x00, 0x00, 0x00, // CheckSum
- 0x03, 0x00, // Subsystem: CONSOLE
- 0x60, 0x01, // DllCharacteristics
+ 0x0b,
+ 0x02, // Magic: PE32+
+ 0x00,
+ 0x00, // Linker version
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // SizeOfCode
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // SizeOfInitializedData
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // SizeOfUninitializedData
+ 0x00,
+ 0x10,
+ 0x00,
+ 0x00, // AddressOfEntryPoint
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // BaseOfCode
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x40,
+ 0x01,
+ 0x00,
+ 0x00,
+ 0x00, // ImageBase
+ 0x00,
+ 0x10,
+ 0x00,
+ 0x00, // SectionAlignment
+ 0x00,
+ 0x02,
+ 0x00,
+ 0x00, // FileAlignment
+ 0x06,
+ 0x00,
+ 0x00,
+ 0x00, // OS version
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // Image version
+ 0x06,
+ 0x00,
+ 0x00,
+ 0x00, // Subsystem version
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // Win32VersionValue
+ 0x00,
+ 0x20,
+ 0x00,
+ 0x00, // SizeOfImage
+ 0x00,
+ 0x02,
+ 0x00,
+ 0x00, // SizeOfHeaders
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // CheckSum
+ 0x03,
+ 0x00, // Subsystem: CONSOLE
+ 0x60,
+ 0x01, // DllCharacteristics
// Stack/Heap sizes (8 bytes each for PE32+)
- 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, // LoaderFlags
- 0x10, 0x00, 0x00, 0x00, // NumberOfRvaAndSizes
+ 0x00,
+ 0x00,
+ 0x10,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x10,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00,
+ 0x00, // LoaderFlags
+ 0x10,
+ 0x00,
+ 0x00,
+ 0x00, // NumberOfRvaAndSizes
]);
// Pad to 512 bytes minimum for valid PE
@@ -138,19 +364,8 @@ exit 1
}
function main() {
- console.log('Setting up development sidecar...');
- console.log('');
-
const targetTriple = getTargetTriple();
- console.log(`Platform: ${targetTriple}`);
-
createPlaceholderBinary(targetTriple);
-
- console.log('');
- console.log('Sidecar setup complete.');
- console.log('For development, start the Python server in a separate terminal:');
- console.log(' bun run dev:server');
- console.log('');
}
main();