diff --git a/app/src/components/Generation/EngineModelSelector.tsx b/app/src/components/Generation/EngineModelSelector.tsx new file mode 100644 index 00000000..fc58e828 --- /dev/null +++ b/app/src/components/Generation/EngineModelSelector.tsx @@ -0,0 +1,96 @@ +import type { UseFormReturn } from 'react-hook-form'; +import { FormControl } from '@/components/ui/form'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { getLanguageOptionsForEngine } from '@/lib/constants/languages'; +import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm'; + +/** + * Engine/model options and their display metadata. + * Adding a new engine means adding one entry here. + */ +const ENGINE_OPTIONS = [ + { value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B' }, + { value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B' }, + { value: 'luxtts', label: 'LuxTTS' }, + { value: 'chatterbox', label: 'Chatterbox' }, + { value: 'chatterbox_turbo', label: 'Chatterbox Turbo' }, +] as const; + +const ENGINE_DESCRIPTIONS: Record = { + qwen: 'Multi-language, two sizes', + luxtts: 'Fast, English-focused', + chatterbox: '23 languages, incl. Hebrew', + chatterbox_turbo: 'English, [laugh] [cough] tags', +}; + +/** Engines that only support English and should force language to 'en' on select. */ +const ENGLISH_ONLY_ENGINES = new Set(['luxtts', 'chatterbox_turbo']); + +function getSelectValue(engine: string, modelSize?: string): string { + if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`; + return engine; +} + +function handleEngineChange(form: UseFormReturn, value: string) { + if (value.startsWith('qwen:')) { + const [, modelSize] = value.split(':'); + form.setValue('engine', 'qwen'); + form.setValue('modelSize', modelSize as '1.7B' | '0.6B'); + } else { + form.setValue('engine', value as GenerationFormValues['engine']); + if (ENGLISH_ONLY_ENGINES.has(value)) { + form.setValue('language', 'en'); + } else { + // If current language isn't supported by the new engine, reset to first available + const currentLang = form.getValues('language'); + const available = getLanguageOptionsForEngine(value); + if (!available.some((l) => l.value === currentLang)) { + form.setValue('language', available[0]?.value ?? 'en'); + } + } + } +} + +interface EngineModelSelectorProps { + form: UseFormReturn; + compact?: boolean; +} + +export function EngineModelSelector({ form, compact }: EngineModelSelectorProps) { + const engine = form.watch('engine') || 'qwen'; + const modelSize = form.watch('modelSize'); + const selectValue = getSelectValue(engine, modelSize); + + const itemClass = compact ? 'text-xs text-muted-foreground' : undefined; + const triggerClass = compact + ? 'h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all' + : undefined; + + return ( + + ); +} + +/** Returns a human-readable description for the currently selected engine. */ +export function getEngineDescription(engine: string): string { + return ENGINE_DESCRIPTIONS[engine] ?? ''; +} diff --git a/app/src/components/Generation/FloatingGenerateBox.tsx b/app/src/components/Generation/FloatingGenerateBox.tsx index 005bcdca..1ba4f0fe 100644 --- a/app/src/components/Generation/FloatingGenerateBox.tsx +++ b/app/src/components/Generation/FloatingGenerateBox.tsx @@ -22,6 +22,7 @@ import { cn } from '@/lib/utils/cn'; import { useGenerationStore } from '@/stores/generationStore'; import { useStoryStore } from '@/stores/storyStore'; import { useUIStore } from '@/stores/uiStore'; +import { EngineModelSelector } from './EngineModelSelector'; import { ParalinguisticInput } from './ParalinguisticInput'; interface FloatingGenerateBoxProps { @@ -455,58 +456,7 @@ export function FloatingGenerateBox({ /> - + diff --git a/app/src/components/Generation/GenerationForm.tsx b/app/src/components/Generation/GenerationForm.tsx index e326bec4..225e8dfa 100644 --- a/app/src/components/Generation/GenerationForm.tsx +++ b/app/src/components/Generation/GenerationForm.tsx @@ -23,6 +23,7 @@ import { getLanguageOptionsForEngine } from '@/lib/constants/languages'; import { useGenerationForm } from '@/lib/hooks/useGenerationForm'; import { useProfile } from '@/lib/hooks/useProfiles'; import { useUIStore } from '@/stores/uiStore'; +import { EngineModelSelector, getEngineDescription } from './EngineModelSelector'; import { ParalinguisticInput } from './ParalinguisticInput'; export function GenerationForm() { @@ -117,53 +118,9 @@ export function GenerationForm() {
Model - + - {form.watch('engine') === 'luxtts' - ? 'Fast, English-focused' - : form.watch('engine') === 'chatterbox' - ? '23 languages, incl. Hebrew' - : form.watch('engine') === 'chatterbox_turbo' - ? 'English, [laugh] [cough] tags' - : 'Multi-language, two sizes'} + {getEngineDescription(form.watch('engine') || 'qwen')} diff --git a/app/src/components/Sidebar.tsx b/app/src/components/Sidebar.tsx index 88659399..5a37bd64 100644 --- a/app/src/components/Sidebar.tsx +++ b/app/src/components/Sidebar.tsx @@ -1,7 +1,10 @@ import { Link, useMatchRoute } from '@tanstack/react-router'; import { AudioLines, Box, Mic, Server, Speaker, Volume2, Wand2 } from 'lucide-react'; +import { useEffect, useState } from 'react'; import voiceboxLogo from '@/assets/voicebox-logo.png'; import { cn } from '@/lib/utils/cn'; +import { usePlatform } from '@/platform/PlatformContext'; +import type { UpdateStatus } from '@/platform/types'; import { usePlayerStore } from '@/stores/playerStore'; import { version } from '../../package.json'; @@ -22,6 +25,10 @@ const tabs = [ export function Sidebar({ isMacOS }: SidebarProps) { const matchRoute = useMatchRoute(); const isPlayerOpen = !!usePlayerStore((s) => s.audioUrl); + const platform = usePlatform(); + + const [updateStatus, setUpdateStatus] = useState(platform.updater.getStatus()); + useEffect(() => platform.updater.subscribe(setUpdateStatus), [platform.updater]); return (
- v{version} + v{version} + {updateStatus.available && ( + + Update + + )}
); diff --git a/backend/REFACTOR_PLAN.md b/backend/REFACTOR_PLAN.md new file mode 100644 index 00000000..19b7cea2 --- /dev/null +++ b/backend/REFACTOR_PLAN.md @@ -0,0 +1,158 @@ +# Backend Refactor Plan + +## Current State + +2,856-line god file (`main.py`), ~500 lines of copy-pasted backend methods, 3x duplicated generation orchestration, dead modules, fake async, scattered constants. 72 routes all registered in one file. Works fine, but will fight us on every new feature. + +--- + +## Phase 1: Dead Code & Low-Hanging Fruit + +Remove noise so the real structure is easier to see. + +- Delete `studio.py` (66 lines, every method raises `NotImplementedError`, never imported) +- Delete `migrate_add_instruct.py` (48 lines, superseded by `database.py` migrations) +- Delete `utils/validation.py` (66 lines, none of the 3 functions are called anywhere) +- Remove duplicate `_profile_to_response()` in `main.py:1983-2005`, use the one from `profiles.py` +- Remove duplicate `import asyncio` in `main.py` +- Remove pointless one-line wrappers (`_get_profiles_dir()`, `_get_generations_dir()`) in `profiles.py`, `history.py`, `export_import.py` — call `config.*` directly +- Deduplicate `LANGUAGE_CODE_TO_NAME` (defined in both `pytorch_backend.py:18` and `mlx_backend.py:24`) — move to `backends/__init__.py` +- Deduplicate `WHISPER_HF_REPOS` (defined in both `pytorch_backend.py:379` and `mlx_backend.py:416`) — move to `backends/__init__.py` +- Update `README.md` to reflect actual file structure (it still references `studio.py` and the old two-backend layout) + +--- + +## Phase 2: Backend Deduplication + +The backends have 5-7 copies of identical or near-identical methods. This is the highest-value structural change because it removes ~500 lines and makes adding new engines trivial. + +### Extract shared methods + +Create `backends/base.py` with: + +- **`is_model_cached(hf_repo, hf_revision)`** — the HuggingFace cache directory check. Currently copy-pasted in `pytorch_backend.py:81`, `mlx_backend.py:68`, `chatterbox_backend.py:66`, `chatterbox_turbo_backend.py:66`, `luxtts_backend.py:57`, and both STT backends. One function, parameterized by repo/revision. + +- **`combine_voice_prompts(samples, sample_rate, backend_type)`** — load audio, normalize, concatenate, join texts. Identical in all 5 TTS backends (`pytorch:301`, `mlx:291`, `chatterbox:266`, `chatterbox_turbo:269`, `luxtts:208`). The only variation is which audio loading function is used (torchaudio vs mlx_audio) — pass the loader as a parameter or detect from backend type. + +- **`get_device(backend_type)`** — device detection. Currently 5 slightly different implementations. Parameterize the differences: + - PyTorch: checks CUDA > XPU > DirectML > MPS > CPU + - Chatterbox/Chatterbox Turbo: forces CPU on macOS, otherwise CUDA > CPU + - LuxTTS: checks MPS > CUDA > CPU + +- **`model_load_wrapper(load_fn, ...)`** — the progress tracking boilerplate shared by all 7 `_load_model_sync` implementations. Every backend does the same setup/teardown dance with `progress_manager`, `task_manager`, `HFProgressTracker`, and tqdm patching. Extract the wrapper, backends just supply the actual model loading callable. + +### Extract Chatterbox f32 patch + +Move the S3Tokenizer / VoiceEncoder monkey-patches from `chatterbox_backend.py:189-210` and `chatterbox_turbo_backend.py:193-214` into a shared `backends/chatterbox_patches.py` (or a function in `base.py`). Both files have identical code. + +--- + +## Phase 3: Generation Service + +The three generation closures in `main.py` (`_run_generation:782`, `_run_retry:923`, `_run_regenerate:1018`) share ~80% of their logic. Extract into a service module. + +### Create `services/generation.py` + +Single orchestration function with mode parameter: + +```python +async def run_generation( + generation_id: str, + profile_id: str, + text: str, + language: str, + engine: str, + model_size: str, + seed: Optional[int], + normalize: bool, + effects_chain: Optional[list], + instruct_text: Optional[str], + mode: Literal["generate", "retry", "regenerate"], + version_label: Optional[str] = None, +): +``` + +Differences between modes are small and can be handled with conditionals: +- `retry`: reuses same seed, skips effects/versions +- `regenerate`: seed=None, creates a new version with auto-label +- `generate`: full pipeline including effects version + +### Move background queue management + +Move `_generation_queue`, `_generation_worker`, `_enqueue_generation`, `_background_tasks`, and `_create_background_task` (currently `main.py:63-92`) into the service module or a dedicated `services/task_queue.py`. + +--- + +## Phase 4: Route Extraction + +Split `main.py` (72 routes) into domain-specific routers. After Phase 3, the route handlers should be thin — just validation, delegation, and response formatting. + +### Target structure + +``` +backend/ + app.py # FastAPI app creation, middleware, startup/shutdown + routes/ + __init__.py + health.py # GET /, /health, /health/filesystem, /shutdown, /watchdog/disable (5 routes) + profiles.py # All /profiles/* routes (17 routes) + channels.py # All /channels/* routes (7 routes) + generations.py # /generate, /generate/stream, /generate/*/retry, regenerate, status (5 routes) + history.py # All /history/* routes (8 routes) + stories.py # All /stories/* routes (15 routes) + effects.py # All /effects/* routes + /generations/*/versions/* (11 routes) + audio.py # /audio/*, /samples/* (2 routes) + models.py # All /models/* routes (11 routes) + tasks.py # /tasks/*, /cache/* (3 routes) + cuda.py # /backend/cuda-* (4 routes) + services/ + generation.py # TTS orchestration (from Phase 3) + model_status.py # HF cache inspection logic (currently inline at main.py:2251-2431) +``` + +`main.py` becomes a thin entry point that imports the app from `app.py` and runs uvicorn (preserving backward compat for `python -m backend.main`). + +### Model status extraction + +The `get_model_status` endpoint (`main.py:2251-2431`) is 180 lines of HuggingFace cache inspection that duplicates logic from `_is_model_cached` in the backends. Extract to `services/model_status.py` and reuse the shared `is_model_cached` from Phase 2 where possible. + +--- + +## Phase 5: Database Cleanup + +### Split `database.py` (487 lines) + +- `database/models.py` — ORM model definitions (11 models, ~140 lines) +- `database/migrations.py` — migration logic (`_run_migrations`, ~200 lines) +- `database/seed.py` — `_backfill_generation_versions` + `_seed_builtin_presets` +- `database/session.py` — engine creation, `init_db()`, `get_db()` + +### Fix async-over-sync CRUD modules + +`channels.py`, `history.py`, `stories.py`, `effects.py`, `versions.py`, `profiles.py` all declare `async def` but never `await`. They run synchronous SQLAlchemy queries directly, blocking the event loop. Two options: + +- **Option A**: Drop `async` keyword, wrap calls in `asyncio.to_thread()` at the route layer +- **Option B**: Switch to async SQLAlchemy (`create_async_engine` + `AsyncSession`) + +Option A is simpler and non-disruptive. Option B is cleaner long-term but touches every query. + +--- + +## Phase 6: Polish + +- Consolidate hardcoded constants (`24000` sample rate, `100MB`/`50MB` max file sizes, `HSA_OVERRIDE_GFX_VERSION`, CORS origins) into `config.py` or a `constants.py` +- Fix `hf_offline_patch.py` side-effect-on-import (runs patching twice — once on import, once explicitly in `mlx_backend.py`) +- Standardize error handling across routes (currently three different patterns) +- Rename `effects.py` (preset CRUD) to avoid confusion with `utils/effects.py` (DSP engine) — either rename to `effect_presets.py` or fold into routes +- Clean up test suite — the 4 manual integration scripts in `tests/` should either be converted to pytest or moved to a `scripts/` dir + +--- + +## Notes + +- Each phase is independently shippable and testable +- Phase 1 is zero-risk deletion +- Phase 2 is self-contained within `backends/` +- Phase 3 sets up the extraction pattern needed for Phase 4 +- Phase 4 is the largest change but should be mostly mechanical after Phase 3 +- Phase 5 can run in parallel with Phase 4 since it touches different files diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index f120e6ec..e51cd0b3 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -1,10 +1,12 @@ """ Backend abstraction layer for TTS and STT. -Provides a unified interface for MLX and PyTorch backends. +Provides a unified interface for MLX and PyTorch backends, +and a model config registry that eliminates per-engine dispatch maps. """ import threading +from dataclasses import dataclass, field from typing import Protocol, Optional, Tuple, List from typing_extensions import runtime_checkable import numpy as np @@ -12,14 +14,31 @@ import numpy as np from ..platform_detect import get_backend_type +@dataclass +class ModelConfig: + """Declarative config for a downloadable model variant.""" + model_name: str # e.g. "luxtts", "chatterbox-tts" + display_name: str # e.g. "LuxTTS (Fast, CPU-friendly)" + engine: str # e.g. "luxtts", "chatterbox" + hf_repo_id: str # e.g. "YatharthS/LuxTTS" + model_size: str = "default" + size_mb: int = 0 + needs_trim: bool = False + supports_instruct: bool = False + languages: list[str] = field(default_factory=lambda: ["en"]) + + @runtime_checkable class TTSBackend(Protocol): """Protocol for TTS backend implementations.""" - + + # Each backend class should define MODEL_CONFIGS as a class variable: + # MODEL_CONFIGS: list[ModelConfig] + async def load_model(self, model_size: str) -> None: """Load TTS model.""" ... - + async def create_voice_prompt( self, audio_path: str, @@ -28,12 +47,12 @@ class TTSBackend(Protocol): ) -> Tuple[dict, bool]: """ Create voice prompt from reference audio. - + Returns: Tuple of (voice_prompt_dict, was_cached) """ ... - + async def combine_voice_prompts( self, audio_paths: List[str], @@ -41,12 +60,12 @@ class TTSBackend(Protocol): ) -> Tuple[np.ndarray, str]: """ Combine multiple voice prompts. - + Returns: Tuple of (combined_audio_array, combined_text) """ ... - + async def generate( self, text: str, @@ -57,24 +76,24 @@ class TTSBackend(Protocol): ) -> Tuple[np.ndarray, int]: """ Generate audio from text. - + Returns: Tuple of (audio_array, sample_rate) """ ... - + def unload_model(self) -> None: """Unload model to free memory.""" ... - + def is_loaded(self) -> bool: """Check if model is loaded.""" ... - + def _get_model_path(self, model_size: str) -> str: """ Get model path for a given size. - + Returns: Model path or HuggingFace Hub ID """ @@ -84,11 +103,11 @@ class TTSBackend(Protocol): @runtime_checkable class STTBackend(Protocol): """Protocol for STT (Speech-to-Text) backend implementations.""" - + async def load_model(self, model_size: str) -> None: """Load STT model.""" ... - + async def transcribe( self, audio_path: str, @@ -96,16 +115,16 @@ class STTBackend(Protocol): ) -> str: """ Transcribe audio to text. - + Returns: Transcribed text """ ... - + def unload_model(self) -> None: """Unload model to free memory.""" ... - + def is_loaded(self) -> bool: """Check if model is loaded.""" ... @@ -117,7 +136,8 @@ _tts_backends: dict[str, TTSBackend] = {} _tts_backends_lock = threading.Lock() _stt_backend: Optional[STTBackend] = None -# Supported TTS engines +# Supported TTS engines — keyed by engine name, value is the backend class import path. +# The factory function uses this for the if/elif chain; the model configs live on the backend classes. TTS_ENGINES = { "qwen": "Qwen TTS", "luxtts": "LuxTTS", @@ -126,10 +146,233 @@ TTS_ENGINES = { } +# --------------------------------------------------------------------------- +# Model config registry +# --------------------------------------------------------------------------- + +def _get_qwen_model_configs() -> list[ModelConfig]: + """Return Qwen model configs with backend-aware HF repo IDs.""" + backend_type = get_backend_type() + if backend_type == "mlx": + repo_1_7b = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" + repo_0_6b = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" # 0.6B not available in MLX, falls back + else: + repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base" + repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" + + return [ + ModelConfig( + model_name="qwen-tts-1.7B", + display_name="Qwen TTS 1.7B", + engine="qwen", + hf_repo_id=repo_1_7b, + model_size="1.7B", + size_mb=3500, + supports_instruct=False, # Base model drops instruct silently + languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"], + ), + ModelConfig( + model_name="qwen-tts-0.6B", + display_name="Qwen TTS 0.6B", + engine="qwen", + hf_repo_id=repo_0_6b, + model_size="0.6B", + size_mb=1200, + supports_instruct=False, + languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"], + ), + ] + + +def _get_non_qwen_tts_configs() -> list[ModelConfig]: + """Return model configs for non-Qwen TTS engines. + + These are static — no backend-type branching needed. + """ + return [ + ModelConfig( + model_name="luxtts", + display_name="LuxTTS (Fast, CPU-friendly)", + engine="luxtts", + hf_repo_id="YatharthS/LuxTTS", + size_mb=300, + languages=["en"], + ), + ModelConfig( + model_name="chatterbox-tts", + display_name="Chatterbox TTS (Multilingual)", + engine="chatterbox", + hf_repo_id="ResembleAI/chatterbox", + size_mb=3200, + needs_trim=True, + languages=[ + "zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it", + "he", "ar", "da", "el", "fi", "hi", "ms", "nl", "no", "pl", + "sv", "sw", "tr", + ], + ), + ModelConfig( + model_name="chatterbox-turbo", + display_name="Chatterbox Turbo (English, Tags)", + engine="chatterbox_turbo", + hf_repo_id="ResembleAI/chatterbox-turbo", + size_mb=1500, + needs_trim=True, + languages=["en"], + ), + ] + + +def _get_whisper_configs() -> list[ModelConfig]: + """Return Whisper STT model configs.""" + return [ + ModelConfig(model_name="whisper-base", display_name="Whisper Base", engine="whisper", hf_repo_id="openai/whisper-base", model_size="base"), + ModelConfig(model_name="whisper-small", display_name="Whisper Small", engine="whisper", hf_repo_id="openai/whisper-small", model_size="small"), + ModelConfig(model_name="whisper-medium", display_name="Whisper Medium", engine="whisper", hf_repo_id="openai/whisper-medium", model_size="medium"), + ModelConfig(model_name="whisper-large", display_name="Whisper Large", engine="whisper", hf_repo_id="openai/whisper-large-v3", model_size="large"), + ModelConfig(model_name="whisper-turbo", display_name="Whisper Turbo", engine="whisper", hf_repo_id="openai/whisper-large-v3-turbo", model_size="turbo"), + ] + + +def get_all_model_configs() -> list[ModelConfig]: + """Return the full list of model configs (TTS + STT).""" + return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs() + + +def get_tts_model_configs() -> list[ModelConfig]: + """Return only TTS model configs.""" + return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + + +# Lookup helpers — these replace the if/elif chains in main.py + +def get_model_config(model_name: str) -> Optional[ModelConfig]: + """Look up a model config by model_name.""" + for cfg in get_all_model_configs(): + if cfg.model_name == model_name: + return cfg + return None + + +def engine_needs_trim(engine: str) -> bool: + """Whether this engine's output should be run through trim_tts_output.""" + for cfg in get_tts_model_configs(): + if cfg.engine == engine: + return cfg.needs_trim + return False + + +def engine_has_model_sizes(engine: str) -> bool: + """Whether this engine supports multiple model sizes (only Qwen currently).""" + configs = [c for c in get_tts_model_configs() if c.engine == engine] + return len(configs) > 1 + + +async def load_engine_model(engine: str, model_size: str = "default") -> None: + """Load a model for the given engine, handling the Qwen model_size special case.""" + backend = get_tts_backend_for_engine(engine) + if engine == "qwen": + await backend.load_model_async(model_size) + else: + await backend.load_model() + + +async def ensure_model_cached_or_raise(engine: str, model_size: str = "default") -> None: + """Check if a model is cached, raise HTTPException if not. Used by streaming endpoint.""" + from fastapi import HTTPException + backend = get_tts_backend_for_engine(engine) + cfg = None + for c in get_tts_model_configs(): + if c.engine == engine and c.model_size == model_size: + cfg = c + break + + if engine == "qwen": + if not backend._is_model_cached(model_size): + raise HTTPException( + status_code=400, + detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.", + ) + else: + if not backend._is_model_cached(): + display = cfg.display_name if cfg else engine + raise HTTPException( + status_code=400, + detail=f"{display} model is not downloaded yet. Use /generate to trigger a download.", + ) + + +def unload_model_by_config(config: ModelConfig) -> bool: + """Unload a model given its config. Returns True if it was loaded, False otherwise.""" + from . import get_tts_backend_for_engine + from .. import tts, transcribe + + if config.engine == "whisper": + whisper_model = transcribe.get_whisper_model() + if whisper_model.is_loaded() and whisper_model.model_size == config.model_size: + transcribe.unload_whisper_model() + return True + return False + + if config.engine == "qwen": + tts_model = tts.get_tts_model() + loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None) + if tts_model.is_loaded() and loaded_size == config.model_size: + tts.unload_tts_model() + return True + return False + + # All other TTS engines + backend = get_tts_backend_for_engine(config.engine) + if backend.is_loaded(): + backend.unload_model() + return True + return False + + +def check_model_loaded(config: ModelConfig) -> bool: + """Check if a model is currently loaded.""" + from . import get_tts_backend_for_engine + from .. import tts, transcribe + + try: + if config.engine == "whisper": + whisper_model = transcribe.get_whisper_model() + return whisper_model.is_loaded() and getattr(whisper_model, 'model_size', None) == config.model_size + + if config.engine == "qwen": + tts_model = tts.get_tts_model() + loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None) + return tts_model.is_loaded() and loaded_size == config.model_size + + backend = get_tts_backend_for_engine(config.engine) + return backend.is_loaded() + except Exception: + return False + + +def get_model_load_func(config: ModelConfig): + """Return a callable that loads/downloads the model.""" + from . import get_tts_backend_for_engine + from .. import tts, transcribe + + if config.engine == "whisper": + return lambda: transcribe.get_whisper_model().load_model(config.model_size) + + if config.engine == "qwen": + return lambda: tts.get_tts_model().load_model(config.model_size) + + return lambda: get_tts_backend_for_engine(config.engine).load_model() + + +# --------------------------------------------------------------------------- +# Backend factory +# --------------------------------------------------------------------------- + def get_tts_backend() -> TTSBackend: """ Get or create the default (Qwen) TTS backend instance based on platform. - + Returns: TTS backend instance (MLX or PyTorch) """ @@ -139,25 +382,25 @@ def get_tts_backend() -> TTSBackend: def get_tts_backend_for_engine(engine: str) -> TTSBackend: """ Get or create a TTS backend for the given engine. - + Args: - engine: Engine name ("qwen" or "luxtts") - + engine: Engine name (e.g. "qwen", "luxtts", "chatterbox", "chatterbox_turbo") + Returns: TTS backend instance """ global _tts_backends - + # Fast path: check without lock if engine in _tts_backends: return _tts_backends[engine] - + # Slow path: create with lock to avoid duplicate instantiation with _tts_backends_lock: # Double-check after acquiring lock if engine in _tts_backends: return _tts_backends[engine] - + if engine == "qwen": backend_type = get_backend_type() if backend_type == "mlx": @@ -177,7 +420,7 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: backend = ChatterboxTurboTTSBackend() else: raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}") - + _tts_backends[engine] = backend return backend @@ -185,22 +428,22 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: def get_stt_backend() -> STTBackend: """ Get or create STT backend instance based on platform. - + Returns: STT backend instance (MLX or PyTorch) """ global _stt_backend - + if _stt_backend is None: backend_type = get_backend_type() - + if backend_type == "mlx": from .mlx_backend import MLXSTTBackend _stt_backend = MLXSTTBackend() else: from .pytorch_backend import PyTorchSTTBackend _stt_backend = PyTorchSTTBackend() - + return _stt_backend diff --git a/backend/main.py b/backend/main.py index 9d238884..5c9fca44 100644 --- a/backend/main.py +++ b/backend/main.py @@ -233,11 +233,9 @@ async def health(): model_downloaded = None try: # Check if the default model (1.7B) is cached - # Use different model IDs based on backend - if backend_type == "mlx": - default_model_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" - else: - default_model_id = "Qwen/Qwen3-TTS-12Hz-1.7B-Base" + from .backends import get_model_config + default_config = get_model_config("qwen-tts-1.7B") + default_model_id = default_config.hf_repo_id if default_config else "Qwen/Qwen3-TTS-12Hz-1.7B-Base" # Method 1: Try scan_cache_dir if available try: @@ -738,7 +736,7 @@ async def generate_speech( if not profile: raise HTTPException(status_code=404, detail="Profile not found") - from .backends import get_tts_backend_for_engine + from .backends import get_tts_backend_for_engine, engine_has_model_sizes engine = data.engine or "qwen" tts_model = get_tts_backend_for_engine(engine) model_size = data.model_size or "1.7B" @@ -756,7 +754,7 @@ async def generate_speech( generation_id=generation_id, status="generating", engine=engine, - model_size=model_size if engine == "qwen" else None, + model_size=model_size if engine_has_model_sizes(engine) else None, ) # Track in task manager @@ -785,10 +783,8 @@ async def generate_speech( bg_db = next(get_db()) try: # Load model - if engine == "qwen": - await tts_model.load_model_async(model_size) - else: - await tts_model.load_model() + from .backends import load_engine_model, engine_needs_trim + await load_engine_model(engine, model_size) # Create voice prompt voice_prompt = await profiles.create_voice_prompt_for_profile( @@ -801,7 +797,7 @@ async def generate_speech( from .utils.chunked_tts import generate_chunked trim_fn = None - if engine in ("chatterbox", "chatterbox_turbo"): + if engine_needs_trim(engine): from .utils.audio import trim_tts_output trim_fn = trim_tts_output @@ -927,10 +923,8 @@ async def retry_generation(generation_id: str, db: Session = Depends(get_db)): async def _run_retry(): bg_db = next(get_db()) try: - if retry_engine == "qwen": - await tts_model.load_model_async(retry_model_size) - else: - await tts_model.load_model() + from .backends import load_engine_model, engine_needs_trim + await load_engine_model(retry_engine, retry_model_size) voice_prompt = await profiles.create_voice_prompt_for_profile( gen.profile_id, @@ -942,7 +936,7 @@ async def retry_generation(generation_id: str, db: Session = Depends(get_db)): from .utils.chunked_tts import generate_chunked trim_fn = None - if retry_engine in ("chatterbox", "chatterbox_turbo"): + if engine_needs_trim(retry_engine): from .utils.audio import trim_tts_output trim_fn = trim_tts_output @@ -1024,10 +1018,8 @@ async def regenerate_generation(generation_id: str, db: Session = Depends(get_db async def _run_regenerate(): bg_db = next(get_db()) try: - if regen_engine == "qwen": - await tts_model.load_model_async(regen_model_size) - else: - await tts_model.load_model() + from .backends import load_engine_model, engine_needs_trim + await load_engine_model(regen_engine, regen_model_size) voice_prompt = await profiles.create_voice_prompt_for_profile( gen.profile_id, @@ -1039,7 +1031,7 @@ async def regenerate_generation(generation_id: str, db: Session = Depends(get_db from .utils.chunked_tts import generate_chunked trim_fn = None - if regen_engine in ("chatterbox", "chatterbox_turbo"): + if engine_needs_trim(regen_engine): from .utils.audio import trim_tts_output trim_fn = trim_tts_output @@ -1162,34 +1154,9 @@ async def stream_speech( tts_model = get_tts_backend_for_engine(engine) model_size = data.model_size or "1.7B" - if engine == "qwen": - if not tts_model._is_model_cached(model_size): - raise HTTPException( - status_code=400, - detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.", - ) - await tts_model.load_model_async(model_size) - elif engine == "luxtts": - if not tts_model._is_model_cached(): - raise HTTPException( - status_code=400, - detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.", - ) - await tts_model.load_model() - elif engine == "chatterbox": - if not tts_model._is_model_cached(): - raise HTTPException( - status_code=400, - detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.", - ) - await tts_model.load_model() - elif engine == "chatterbox_turbo": - if not tts_model._is_model_cached(): - raise HTTPException( - status_code=400, - detail="Chatterbox Turbo model is not downloaded yet. Use /generate to trigger a download.", - ) - await tts_model.load_model() + from .backends import ensure_model_cached_or_raise, load_engine_model, engine_needs_trim + await ensure_model_cached_or_raise(engine, model_size) + await load_engine_model(engine, model_size) voice_prompt = await profiles.create_voice_prompt_for_profile( data.profile_id, db, engine=engine, @@ -1198,7 +1165,7 @@ async def stream_speech( from .utils.chunked_tts import generate_chunked trim_fn = None - if engine in ("chatterbox", "chatterbox_turbo"): + if engine_needs_trim(engine): from .utils.audio import trim_tts_output trim_fn = trim_tts_output @@ -2108,63 +2075,16 @@ async def unload_model(): @app.post("/models/{model_name}/unload") async def unload_model_by_name(model_name: str): """Unload a specific model from memory without deleting it from disk.""" - # Map of model_name -> (model_type, model_size) - model_types = { - "qwen-tts-1.7B": ("tts", "1.7B"), - "qwen-tts-0.6B": ("tts", "0.6B"), - "luxtts": ("luxtts", "default"), - "chatterbox-tts": ("chatterbox", "default"), - "chatterbox-turbo": ("chatterbox_turbo", "default"), - "whisper-base": ("whisper", "base"), - "whisper-small": ("whisper", "small"), - "whisper-medium": ("whisper", "medium"), - "whisper-large": ("whisper", "large"), - "whisper-turbo": ("whisper", "turbo"), - } + from .backends import get_model_config, unload_model_by_config - if model_name not in model_types: + config = get_model_config(model_name) + if not config: raise HTTPException(status_code=400, detail=f"Unknown model: {model_name}") - model_type, model_size = model_types[model_name] - try: - if model_type == "tts": - tts_model = tts.get_tts_model() - loaded_size = getattr( - tts_model, "_current_model_size", None - ) or getattr(tts_model, "model_size", None) - if tts_model.is_loaded() and loaded_size == model_size: - tts.unload_tts_model() - else: - return {"message": f"Model {model_name} is not loaded"} - elif model_type == "luxtts": - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("luxtts") - if backend.is_loaded(): - backend.unload_model() - else: - return {"message": f"Model {model_name} is not loaded"} - elif model_type == "chatterbox": - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("chatterbox") - if backend.is_loaded(): - backend.unload_model() - else: - return {"message": f"Model {model_name} is not loaded"} - elif model_type == "chatterbox_turbo": - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("chatterbox_turbo") - if backend.is_loaded(): - backend.unload_model() - else: - return {"message": f"Model {model_name} is not loaded"} - elif model_type == "whisper": - whisper_model = transcribe.get_whisper_model() - if whisper_model.is_loaded() and whisper_model.model_size == model_size: - transcribe.unload_whisper_model() - else: - return {"message": f"Model {model_name} is not loaded"} - + was_loaded = unload_model_by_config(config) + if not was_loaded: + return {"message": f"Model {model_name} is not loaded"} return {"message": f"Model {model_name} unloaded successfully"} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e @@ -2347,140 +2267,18 @@ async def get_model_status(): except ImportError: use_scan_cache = False - def check_tts_loaded(model_size: str): - """Check if TTS model is loaded with specific size.""" - try: - tts_model = tts.get_tts_model() - loaded_size = getattr( - tts_model, "_current_model_size", None - ) or getattr(tts_model, "model_size", None) - return tts_model.is_loaded() and loaded_size == model_size - except Exception: - return False - - def check_whisper_loaded(model_size: str): - """Check if Whisper model is loaded with specific size.""" - try: - whisper_model = transcribe.get_whisper_model() - return whisper_model.is_loaded() and getattr(whisper_model, 'model_size', None) == model_size - except Exception: - return False - - # Use backend-specific model IDs - if backend_type == "mlx": - tts_1_7b_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" - tts_0_6b_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" # Fallback to 1.7B - # MLX backend uses openai/whisper-* models, not mlx-community - whisper_base_id = "openai/whisper-base" - whisper_small_id = "openai/whisper-small" - whisper_medium_id = "openai/whisper-medium" - whisper_large_id = "openai/whisper-large-v3" - else: - tts_1_7b_id = "Qwen/Qwen3-TTS-12Hz-1.7B-Base" - tts_0_6b_id = "Qwen/Qwen3-TTS-12Hz-0.6B-Base" - whisper_base_id = "openai/whisper-base" - whisper_small_id = "openai/whisper-small" - whisper_medium_id = "openai/whisper-medium" - whisper_large_id = "openai/whisper-large-v3" - - # Check if LuxTTS backend is loaded - def check_luxtts_loaded(): - try: - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("luxtts") - return backend.is_loaded() - except Exception: - return False - - # Check if Chatterbox backend is loaded - def check_chatterbox_loaded(): - try: - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("chatterbox") - return backend.is_loaded() - except Exception: - return False - - # Check if Chatterbox Turbo backend is loaded - def check_chatterbox_turbo_loaded(): - try: - from .backends import get_tts_backend_for_engine - backend = get_tts_backend_for_engine("chatterbox_turbo") - return backend.is_loaded() - except Exception: - return False + from .backends import get_all_model_configs, check_model_loaded + registry_configs = get_all_model_configs() model_configs = [ { - "model_name": "qwen-tts-1.7B", - "display_name": "Qwen TTS 1.7B", - "hf_repo_id": tts_1_7b_id, - "model_size": "1.7B", - "check_loaded": lambda: check_tts_loaded("1.7B"), - }, - { - "model_name": "qwen-tts-0.6B", - "display_name": "Qwen TTS 0.6B", - "hf_repo_id": tts_0_6b_id, - "model_size": "0.6B", - "check_loaded": lambda: check_tts_loaded("0.6B"), - }, - { - "model_name": "luxtts", - "display_name": "LuxTTS (Fast, CPU-friendly)", - "hf_repo_id": "YatharthS/LuxTTS", - "model_size": "default", - "check_loaded": check_luxtts_loaded, - }, - { - "model_name": "chatterbox-tts", - "display_name": "Chatterbox TTS (Multilingual)", - "hf_repo_id": "ResembleAI/chatterbox", - "model_size": "default", - "check_loaded": check_chatterbox_loaded, - }, - { - "model_name": "chatterbox-turbo", - "display_name": "Chatterbox Turbo (English, Tags)", - "hf_repo_id": "ResembleAI/chatterbox-turbo", - "model_size": "default", - "check_loaded": check_chatterbox_turbo_loaded, - }, - { - "model_name": "whisper-base", - "display_name": "Whisper Base", - "hf_repo_id": whisper_base_id, - "model_size": "base", - "check_loaded": lambda: check_whisper_loaded("base"), - }, - { - "model_name": "whisper-small", - "display_name": "Whisper Small", - "hf_repo_id": whisper_small_id, - "model_size": "small", - "check_loaded": lambda: check_whisper_loaded("small"), - }, - { - "model_name": "whisper-medium", - "display_name": "Whisper Medium", - "hf_repo_id": whisper_medium_id, - "model_size": "medium", - "check_loaded": lambda: check_whisper_loaded("medium"), - }, - { - "model_name": "whisper-large", - "display_name": "Whisper Large", - "hf_repo_id": whisper_large_id, - "model_size": "large", - "check_loaded": lambda: check_whisper_loaded("large"), - }, - { - "model_name": "whisper-turbo", - "display_name": "Whisper Turbo", - "hf_repo_id": "openai/whisper-large-v3-turbo", - "model_size": "turbo", - "check_loaded": lambda: check_whisper_loaded("turbo"), - }, + "model_name": cfg.model_name, + "display_name": cfg.display_name, + "hf_repo_id": cfg.hf_repo_id, + "model_size": cfg.model_size, + "check_loaded": lambda c=cfg: check_model_loaded(c), + } + for cfg in registry_configs ] # Build a mapping of model_name -> hf_repo_id so we can check if shared repos are downloading @@ -2637,64 +2435,22 @@ async def get_model_status(): async def trigger_model_download(request: models.ModelDownloadRequest): """Trigger download of a specific model.""" import asyncio - from .backends import get_tts_backend_for_engine - + from .backends import get_model_config, get_model_load_func + task_manager = get_task_manager() progress_manager = get_progress_manager() - - model_configs = { - "qwen-tts-1.7B": { - "model_size": "1.7B", - "load_func": lambda: tts.get_tts_model().load_model("1.7B"), - }, - "qwen-tts-0.6B": { - "model_size": "0.6B", - "load_func": lambda: tts.get_tts_model().load_model("0.6B"), - }, - "luxtts": { - "model_size": "default", - "load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(), - }, - "chatterbox-tts": { - "model_size": "default", - "load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(), - }, - "chatterbox-turbo": { - "model_size": "default", - "load_func": lambda: get_tts_backend_for_engine("chatterbox_turbo").load_model(), - }, - "whisper-base": { - "model_size": "base", - "load_func": lambda: transcribe.get_whisper_model().load_model("base"), - }, - "whisper-small": { - "model_size": "small", - "load_func": lambda: transcribe.get_whisper_model().load_model("small"), - }, - "whisper-medium": { - "model_size": "medium", - "load_func": lambda: transcribe.get_whisper_model().load_model("medium"), - }, - "whisper-large": { - "model_size": "large", - "load_func": lambda: transcribe.get_whisper_model().load_model("large"), - }, - "whisper-turbo": { - "model_size": "turbo", - "load_func": lambda: transcribe.get_whisper_model().load_model("turbo"), - }, - } - - if request.model_name not in model_configs: + + config = get_model_config(request.model_name) + if not config: raise HTTPException(status_code=400, detail=f"Unknown model: {request.model_name}") - - config = model_configs[request.model_name] + + load_func = get_model_load_func(config) async def download_in_background(): """Download model in background without blocking the HTTP request.""" try: # Call the load function (which may be async) - result = config["load_func"]() + result = load_func() # If it's a coroutine, await it if asyncio.iscoroutine(result): await result @@ -2767,94 +2523,17 @@ async def delete_model(model_name: str): import os from huggingface_hub import constants as hf_constants - # Map model names to HuggingFace repo IDs - model_configs = { - "qwen-tts-1.7B": { - "hf_repo_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", - "model_size": "1.7B", - "model_type": "tts", - }, - "qwen-tts-0.6B": { - "hf_repo_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base", - "model_size": "0.6B", - "model_type": "tts", - }, - "luxtts": { - "hf_repo_id": "YatharthS/LuxTTS", - "model_size": "default", - "model_type": "luxtts", - }, - "chatterbox-tts": { - "hf_repo_id": "ResembleAI/chatterbox", - "model_size": "default", - "model_type": "chatterbox", - }, - "chatterbox-turbo": { - "hf_repo_id": "ResembleAI/chatterbox-turbo", - "model_size": "default", - "model_type": "chatterbox_turbo", - }, - "whisper-base": { - "hf_repo_id": "openai/whisper-base", - "model_size": "base", - "model_type": "whisper", - }, - "whisper-small": { - "hf_repo_id": "openai/whisper-small", - "model_size": "small", - "model_type": "whisper", - }, - "whisper-medium": { - "hf_repo_id": "openai/whisper-medium", - "model_size": "medium", - "model_type": "whisper", - }, - "whisper-large": { - "hf_repo_id": "openai/whisper-large-v3", - "model_size": "large", - "model_type": "whisper", - }, - "whisper-turbo": { - "hf_repo_id": "openai/whisper-large-v3-turbo", - "model_size": "turbo", - "model_type": "whisper", - }, - } + from .backends import get_model_config, unload_model_by_config - if model_name not in model_configs: + config = get_model_config(model_name) + if not config: raise HTTPException(status_code=400, detail=f"Unknown model: {model_name}") - - config = model_configs[model_name] - hf_repo_id = config["hf_repo_id"] - + + hf_repo_id = config.hf_repo_id + try: - # Check if model is loaded and unload it first - if config["model_type"] == "tts": - tts_model = tts.get_tts_model() - loaded_size = getattr( - tts_model, "_current_model_size", None - ) or getattr(tts_model, "model_size", None) - if tts_model.is_loaded() and loaded_size == config["model_size"]: - tts.unload_tts_model() - elif config["model_type"] == "luxtts": - from .backends import get_tts_backend_for_engine - luxtts = get_tts_backend_for_engine("luxtts") - if luxtts.is_loaded(): - luxtts.unload_model() - elif config["model_type"] == "chatterbox": - from .backends import get_tts_backend_for_engine - chatterbox = get_tts_backend_for_engine("chatterbox") - if chatterbox.is_loaded(): - chatterbox.unload_model() - elif config["model_type"] == "chatterbox_turbo": - from .backends import get_tts_backend_for_engine - turbo = get_tts_backend_for_engine("chatterbox_turbo") - if turbo.is_loaded(): - turbo.unload_model() - elif config["model_type"] == "whisper": - whisper_model = transcribe.get_whisper_model() - if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]: - transcribe.unload_whisper_model() + # Unload model if currently loaded + unload_model_by_config(config) # Find and delete the cache directory (using HuggingFace's OS-specific cache location) cache_dir = hf_constants.HF_HUB_CACHE diff --git a/docs/plans/ADDING_TTS_ENGINES.md b/docs/plans/ADDING_TTS_ENGINES.md new file mode 100644 index 00000000..b759bf62 --- /dev/null +++ b/docs/plans/ADDING_TTS_ENGINES.md @@ -0,0 +1,363 @@ +# Adding a TTS Engine to Voicebox + +Guide for adding new TTS model backends. Based on the implementation of LuxTTS (#254), Chatterbox Multilingual (#257), Chatterbox Turbo (#258), and the PyInstaller fixes in v0.2.3. + +--- + +## Overview + +Adding an engine touches ~12 files across 4 layers (down from ~19 after the model config registry refactor). The backend protocol work is straightforward — the real time sink is dependency hell, upstream library bugs, and PyInstaller bundling. + +--- + +## Phase 1: Backend Implementation + +### 1.1 Create the backend file + +`backend/backends/_backend.py` (~200-300 lines) + +Implement the `TTSBackend` protocol from `backend/backends/__init__.py`: + +```python +class YourBackend: + """Must satisfy the TTSBackend protocol.""" + + async def load_model(self, model_size: str = "default") -> None: ... + async def create_voice_prompt(self, audio_path: str, reference_text: str, use_cache: bool = True) -> tuple[dict, bool]: ... + async def combine_voice_prompts(self, audio_paths: list[str], ref_texts: list[str]) -> tuple[np.ndarray, str]: ... + async def generate(self, text: str, voice_prompt: dict, language: str = "en", seed: int | None = None, instruct: str | None = None) -> tuple[np.ndarray, int]: ... + def unload_model(self) -> None: ... + def is_loaded(self) -> bool: ... + def _get_model_path(self, model_size: str) -> str: ... +``` + +Key decisions per engine: + +| Decision | Options | Examples | +|----------|---------|---------| +| **Voice prompt storage** | Pre-computed tensors vs deferred file paths | Qwen PyTorch stores tensor dicts; Chatterbox stores `{"ref_audio": path, "ref_text": text}` | +| **Caching** | Use voice prompt cache or skip it | LuxTTS caches with `luxtts_` prefix; Chatterbox skips caching entirely | +| **Device selection** | CUDA / MPS / CPU | Chatterbox forces CPU on macOS (MPS tensor bugs); LuxTTS supports MPS | +| **Model download** | Library handles it vs manual `snapshot_download` | Turbo uses manual download to bypass upstream `token=True` bug | +| **Sample rate** | Engine-specific | LuxTTS outputs 48kHz, everything else is 24kHz | + +### 1.2 Voice prompt patterns + +There are three patterns in use. Pick the one that fits your model: + +**Pattern A: Pre-computed tensors** (Qwen PyTorch, LuxTTS) +```python +# create_voice_prompt returns opaque dict of tensors +# Cached via torch.save(), reused across generations +encoded = model.encode_prompt(audio_path) +return encoded, False # (prompt_dict, was_cached) +``` + +**Pattern B: Deferred file paths** (Chatterbox, MLX) +```python +# Just store paths, process at generation time +return {"ref_audio": audio_path, "ref_text": reference_text}, False +``` + +**Pattern C: Hybrid** (possible for new engines) +```python +# Pre-compute speaker embeddings, store alongside paths +embedding = model.extract_speaker(audio_path) +return {"embedding": embedding, "ref_audio": audio_path}, False +``` + +If caching, prefix your cache keys to avoid collisions with other engines using the same reference audio: +```python +cache_key = "yourengine_" + get_cache_key(audio_path, reference_text) +``` + +### 1.3 Register the engine + +In `backend/backends/__init__.py`, three things: + +**1. Add a `ModelConfig` entry** in `_get_non_qwen_tts_configs()`: + +```python +ModelConfig( + model_name="your-engine", + display_name="Your Engine", + engine="your_engine", + hf_repo_id="org/model-repo", + size_mb=3200, + needs_trim=False, # set True if output needs trim_tts_output() + languages=["en", "fr", "de"], +), +``` + +This single entry replaces what used to be 6+ scattered dicts in `main.py`. The registry helpers (`get_model_config()`, `check_model_loaded()`, `engine_needs_trim()`, etc.) all derive from this config automatically. + +**2. Add to `TTS_ENGINES` dict:** + +```python +TTS_ENGINES = { + ... + "your_engine": "Your Engine", +} +``` + +**3. Add an elif branch in `get_tts_backend_for_engine()`:** + +```python +elif engine == "your_engine": + from .your_backend import YourBackend + backend = YourBackend() +``` + +The import is deferred so platform-specific deps aren't loaded until the engine is first requested. + +### 1.4 Update request models + +In `backend/models.py`: + +- Add engine name to `GenerationRequest.engine` regex pattern +- Add any new language codes to the language regex on both `GenerationRequest` and `VoiceProfileCreate` + +--- + +## Phase 2: API Integration (`main.py`) + +With the model config registry, `main.py` has **zero per-engine dispatch points**. All endpoints use registry helpers like `get_model_config()`, `load_engine_model()`, `engine_needs_trim()`, `check_model_loaded()`, etc. + +**You don't need to touch `main.py` at all** unless your engine needs custom behavior in the generate endpoint (e.g. a new post-processing step beyond `trim_tts_output`). + +### 2.1 What the registry handles automatically + +| Endpoint | Registry function used | +|----------|----------------------| +| `POST /generate` | `load_engine_model(engine, size)` + `engine_needs_trim(engine)` | +| `POST /generate/stream` | `ensure_model_cached_or_raise(engine, size)` + `load_engine_model()` | +| `GET /models/status` | `get_all_model_configs()` + `check_model_loaded(config)` | +| `POST /models/download` | `get_model_config(name)` + `get_model_load_func(config)` | +| `POST /models/{name}/unload` | `get_model_config(name)` + `unload_model_by_config(config)` | +| `DELETE /models/{name}` | `get_model_config(name)` + `unload_model_by_config(config)` | + +### 2.2 Post-processing + +If your model produces trailing silence or hallucinated audio, set `needs_trim=True` on your `ModelConfig`. The generate endpoint checks `engine_needs_trim(engine)` and applies `trim_tts_output()` automatically. + +--- + +## Phase 3: Frontend Integration + +### 3.1 TypeScript types + +In `app/src/lib/api/types.ts`: +- Add to the `engine` union type on `GenerationRequest` + +### 3.2 Language maps + +In `app/src/lib/constants/languages.ts`: +- Add entry to `ENGINE_LANGUAGES` record +- Add any new language codes to `ALL_LANGUAGES` if needed + +### 3.3 Engine/model selector (shared component) + +The model selector is a shared component — update one file: + +- `app/src/components/Generation/EngineModelSelector.tsx` + +Add an entry to `ENGINE_OPTIONS` and `ENGINE_DESCRIPTIONS`. If the engine is English-only, add it to `ENGLISH_ONLY_ENGINES`. The `handleEngineChange()` function handles language validation automatically (resets to first available language if the current one isn't supported). + +Both `GenerationForm.tsx` and `FloatingGenerateBox.tsx` use `` — no changes needed in either. + +Handle engine-specific UI conditionals in the form components if needed: +- Hide instruct field for engines that don't support it +- Show engine-specific controls (e.g. `ParalinguisticInput` for Turbo) + +### 3.4 Form hook + +In `app/src/lib/hooks/useGenerationForm.ts`: +- Add to Zod schema enum for `engine` +- Add engine-to-model-name mapping (e.g. `"your_engine"` → `"your-engine"`) +- Update payload construction to conditionally include engine-specific fields + +### 3.5 Model management + +In `app/src/components/ServerSettings/ModelManagement.tsx`: +- Add description to `MODEL_DESCRIPTIONS` record +- The model list auto-renders from `/models/status` data + +--- + +## Phase 4: Dependencies + +### 4.1 Python dependencies + +Add to `backend/requirements.txt`. Watch for: + +**Pinned dependency conflicts** — If the model package pins old versions of numpy, torch, or transformers, install with `--no-deps` and list sub-dependencies manually. This is what Chatterbox requires: +``` +# In justfile/Makefile (NOT requirements.txt): +pip install --no-deps chatterbox-tts + +# In requirements.txt — list the transitive deps: +conformer +diffusers +omegaconf +# ... etc +``` + +**Non-PyPI packages** — Some deps only exist as git repos: +``` +linacodec @ git+https://github.com/user/repo.git +Zipvoice @ git+https://github.com/user/repo.git +``` + +**Custom package indexes** — Some packages need `--find-links`: +``` +--find-links https://k2-fsa.github.io/icefall/piper_phonemize.html +``` + +### 4.2 Identifying hidden sub-dependencies + +When using `--no-deps`, you need to manually figure out what the package actually imports. There's no shortcut: + +1. Install the package normally in a throwaway venv +2. Run `pip show ` to get its `Requires:` list +3. Cross-reference against what's already in our requirements.txt +4. Test that the engine loads and generates without import errors + +--- + +## Phase 5: PyInstaller Bundling + +This is where most of the pain lives. If your model's Python package or its dependencies use any of the following at runtime, PyInstaller won't bundle them automatically: + +### 5.1 Common PyInstaller issues + +| Issue | Symptom | Fix | +|-------|---------|-----| +| **`inspect.getsource()` at import time** | "could not get source code" | `--collect-all ` (bundles `.py` source files, not just bytecode) | +| **Data files (yaml, .pth.tar, lang dicts)** | FileNotFoundError at runtime | `--collect-all ` or `--collect-data ` | +| **Native data paths (espeak-ng, etc.)** | Library looks at `/usr/share/...` | Set env var in frozen builds: `os.environ["ESPEAK_DATA_PATH"] = bundled_path` | +| **`importlib.metadata` lookups** | "No package metadata found" | `--copy-metadata ` | +| **Dynamic imports** | ModuleNotFoundError | `--hidden-import ` | +| **`typeguard` / `@typechecked`** | Calls `inspect.getsource()` on decorated functions | `--collect-all` for the decorated package | + +### 5.2 Testing frozen builds + +You can't skip this. Models that work in `python -m uvicorn` will break in the PyInstaller binary. The flow: + +1. Build the binary: `just build` or the PyInstaller spec +2. Run it and try to download + load + generate with the new engine +3. Check stderr for the actual error (macOS/Linux: stdout/stderr go to Tauri sidecar logs) +4. Fix, rebuild, repeat + +### 5.3 Real examples from v0.2.3 + +These were all models that worked perfectly in dev: + +- **LuxTTS**: `typeguard`'s `@typechecked` calls `inspect.getsource()` at import → needed `--collect-all inflect`. `piper_phonemize` bundles `espeak-ng-data/` → needed `--collect-all piper_phonemize` + `ESPEAK_DATA_PATH` env var +- **Chatterbox**: `resemble-perth` bundles a pretrained watermark model (`.pth.tar`, `hparams.yaml`) → needed `--collect-all perth` +- **Both**: `huggingface_hub` silently disables tqdm based on logger level → progress bars showed 0% in frozen builds until we force-enabled the internal counter + +--- + +## Phase 6: Common Upstream Workarounds + +Almost every model library has bugs you'll need to work around. Here's the catalog: + +### 6.1 torch.load device mismatch + +If model weights were saved on CUDA but you're loading on CPU/MPS: +```python +_original_torch_load = torch.load +def _patched_torch_load(*args, **kwargs): + kwargs.setdefault("map_location", "cpu") + return _original_torch_load(*args, **kwargs) +torch.load = _patched_torch_load +``` +Used by both Chatterbox backends. Use a threading lock if patching globally. + +### 6.2 Float64/Float32 dtype mismatch + +`librosa` returns float64, model weights are float32. Patch the offending methods: +```python +original_fn = SomeClass.some_method +def patched_fn(self, *args, **kwargs): + result = original_fn(self, *args, **kwargs) + return result.float() # float64 → float32 +SomeClass.some_method = patched_fn +``` +Used by Chatterbox for `S3Tokenizer.log_mel_spectrogram` and `VoiceEncoder.forward`. + +### 6.3 Transformers attention implementation + +If the model uses `output_attentions=True` with transformers >= 4.36: +```python +for module in model.modules(): + if hasattr(module, '_attn_implementation'): + module._attn_implementation = "eager" +``` +SDPA (the new default) doesn't support `output_attentions`. Force eager attention. + +### 6.4 HuggingFace token bug + +Some models' `from_pretrained()` passes `token=True` which requires a stored HF token even for public repos: +```python +from huggingface_hub import snapshot_download +local_path = snapshot_download(repo_id=REPO, token=None) +model = ModelClass.from_local(local_path, device=device) +``` +Used by Chatterbox Turbo. + +### 6.5 MPS tensor issues + +MPS (Apple Silicon GPU) has incomplete operator coverage. If generation crashes on MPS: +```python +def _get_device(self): + if torch.cuda.is_available(): + return "cuda" + return "cpu" # Skip MPS entirely +``` +Used by both Chatterbox backends. LuxTTS works fine on MPS. + +### 6.6 HuggingFace progress tracking + +To get download progress bars in the UI, wrap model loading with `HFProgressTracker`: +```python +from backend.utils.hf_progress import HFProgressTracker +tracker = HFProgressTracker(model_name, progress_manager) +with tracker.patch_download(): + model = ModelClass.from_pretrained(repo_id) +``` +The tracker monkey-patches tqdm to intercept HuggingFace's internal progress bars. Must be set up BEFORE importing the model library if it imports HF at module level. + +--- + +## Checklist + +### Backend +- [ ] `backend/backends/_backend.py` — implements TTSBackend protocol +- [ ] `backend/backends/__init__.py` — `ModelConfig` entry + `TTS_ENGINES` + `get_tts_backend_for_engine()` elif +- [ ] `backend/models.py` — engine name in regex, any new language codes +- [ ] `backend/requirements.txt` — dependencies added (check for `--no-deps` needs) +- [ ] `justfile` / `Makefile` — `--no-deps` install step if needed + +### API (`backend/main.py`) +No changes needed — the model config registry handles all dispatch automatically. + +### Frontend +- [ ] `app/src/lib/api/types.ts` — engine union type +- [ ] `app/src/lib/constants/languages.ts` — `ENGINE_LANGUAGES` entry +- [ ] `app/src/components/Generation/EngineModelSelector.tsx` — `ENGINE_OPTIONS` + `ENGINE_DESCRIPTIONS` + `ENGLISH_ONLY_ENGINES` +- [ ] `app/src/lib/hooks/useGenerationForm.ts` — Zod schema + model mapping +- [ ] `app/src/components/ServerSettings/ModelManagement.tsx` — model description + +### Production +- [ ] PyInstaller spec — `--collect-all`, `--hidden-import`, `--copy-metadata` as needed +- [ ] Test in frozen binary — download, load, generate all work +- [ ] Download progress — `HFProgressTracker` wired up, progress shows in UI + +### Upstream workarounds (check which apply) +- [ ] torch.load device mapping (CUDA weights on CPU) +- [ ] Float64→Float32 patches (librosa interaction) +- [ ] Eager attention forcing (transformers >= 4.36) +- [ ] HF token bypass (snapshot_download + from_local) +- [ ] MPS skip (if operators not supported) +- [ ] espeak-ng / native data path env vars diff --git a/docs/plans/PROJECT_STATUS.md b/docs/plans/PROJECT_STATUS.md index b8d6ff53..628dfa0a 100644 --- a/docs/plans/PROJECT_STATUS.md +++ b/docs/plans/PROJECT_STATUS.md @@ -50,9 +50,10 @@ | Layer | File | Purpose | |-------|------|---------| -| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2100 lines) | -| TTS protocol | `backend/backends/__init__.py:14-81` | `TTSBackend` Protocol definition | -| TTS factory | `backend/backends/__init__.py:138-178` | Thread-safe engine registry (double-checked locking) | +| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2850 lines) | +| TTS protocol | `backend/backends/__init__.py:32-101` | `TTSBackend` Protocol definition | +| Model registry | `backend/backends/__init__.py:17-29,153-366` | `ModelConfig` dataclass + registry helpers | +| TTS factory | `backend/backends/__init__.py:382-426` | Thread-safe engine registry (double-checked locking) | | PyTorch TTS | `backend/backends/pytorch_backend.py` | Qwen3-TTS via `qwen_tts` package | | MLX TTS | `backend/backends/mlx_backend.py` | Qwen3-TTS via `mlx_audio.tts` | | LuxTTS | `backend/backends/luxtts_backend.py` | LuxTTS — fast, CPU-friendly | @@ -64,6 +65,7 @@ | Audio utils | `backend/utils/audio.py` | `trim_tts_output()`, normalize, load/save audio | | Frontend API | `app/src/lib/api/client.ts` | Hand-written fetch wrapper | | Frontend types | `app/src/lib/api/types.ts` | TypeScript API types | +| Engine selector | `app/src/components/Generation/EngineModelSelector.tsx` | Shared engine/model dropdown | | Generation form | `app/src/components/Generation/GenerationForm.tsx` | TTS generation UI | | Floating gen box | `app/src/components/Generation/FloatingGenerateBox.tsx` | Compact generation UI | | Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status/progress UI | @@ -101,8 +103,10 @@ POST /generate - Multi-engine TTS architecture with thread-safe backend registry (PR #254) - LuxTTS integration — fast, CPU-friendly English TTS (PR #254) - Chatterbox Multilingual TTS — 23 languages including Hebrew (PR #257) -- Delivery instructions (instruct parameter, Qwen only) +- Instruct parameter UI exists but is non-functional across all backends (see #224, Known Limitations) - Single flat model dropdown (Qwen 1.7B, Qwen 0.6B, LuxTTS, Chatterbox, Chatterbox Turbo) +- Centralized model config registry (`ModelConfig` dataclass) — no per-engine dispatch maps in `main.py` +- Shared `EngineModelSelector` component — engine/model dropdown defined once, used in both generation forms **Infrastructure:** - CUDA backend swap via binary download and restart (PR #252) @@ -125,13 +129,13 @@ POST /generate ### TTS Engine Comparison -| Engine | Model Name | Languages | Size | Key Features | -|--------|-----------|-----------|------|-------------| -| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Instruct mode, highest quality | -| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster | -| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast | -| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual | -| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency | +| Engine | Model Name | Languages | Size | Key Features | Instruct Support | +|--------|-----------|-----------|------|-------------|-----------------| +| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Highest quality, voice cloning | None (Base model has no instruct path) | +| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster | None | +| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast | None | +| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual | Partial — `exaggeration` float (0-1) for expressiveness | +| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency | Partial — inline tags only, no separate instruct param | ### Multi-Engine Architecture (Shipped) @@ -149,6 +153,7 @@ The singleton TTS backend blocker described in the previous version of this doc - **HF XET progress**: Large files downloaded via `hf-xet` (HuggingFace's new transfer backend) report `n=0` in tqdm updates. Progress bars may appear stuck for large `.safetensors` files even though the download is proceeding. This is a known upstream limitation. - **Chatterbox Turbo upstream token bug**: `from_pretrained()` passes `token=os.getenv("HF_TOKEN") or True` which fails without a stored HF token. Our backend works around this by calling `snapshot_download(token=None)` + `from_local()`. - **chatterbox-tts must install with `--no-deps`**: It pins `numpy<1.26`, `torch==2.6.0`, `transformers==4.46.3` — all incompatible with our stack (Python 3.12, torch 2.10, transformers 4.57.3). Sub-deps listed explicitly in `requirements.txt`. +- **Instruct parameter is non-functional** (#224): The UI exposes an instruct text field, but it's silently dropped by every backend. The Qwen3-TTS Base model we ship only supports voice cloning — instruct requires the separate CustomVoice model variant (`Qwen3-TTS-12Hz-1.7B-CustomVoice`), which uses predefined speakers instead of ref audio. The instruct UI should be hidden until a backend with real support is integrated. - **Streaming generation** only works for Qwen on MLX. Other engines use the non-streaming `/generate` endpoint. - **dicta-onnx** (Hebrew diacritization) not included — upstream Chatterbox bug requires `model_path` arg but calls `Dicta()` with none. Hebrew works fine without it. @@ -323,41 +328,43 @@ Notable requests: ### Models Worth Supporting (2026 SOTA — updated March 13) -| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Status | -|-------|---------|-------|-------------|-----------|------|-----------------|--------| -| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | **Shipped** | v0.1.13 | -| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | **Shipped** | PR #254 | -| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | **Shipped** | PR #257 | -| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | **PR #258** | In review | -| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Needs vetting | MIT, 700s+ coherent, synced transcript output | -| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | Needs vetting | Apache 2.0, multi-speaker dialogue, text-to-voice design (no ref audio) | -| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion, LoRA-friendly | -| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | Needs vetting | MIT, Kyutai Labs, no GPU required | -| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Ready | Apache 2.0, multi-engine arch in place | -| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Ready | Multi-engine arch in place | -| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Ready | Multi-engine arch in place | -| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Ready | Multi-engine arch in place | +| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Instruct Support | Integration Ease | Status | +|-------|---------|-------|-------------|-----------|------|-----------------|-----------------|--------| +| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | None (Base); Yes (CustomVoice variant, predefined speakers only) | **Shipped** | v0.1.13 | +| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | None | **Shipped** | PR #254 | +| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | Partial — `exaggeration` float | **Shipped** | PR #257 | +| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | Partial — inline tags only | **PR #258** | In review | +| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** — `inference_instruct2()`, works with cloning | Ready | Best instruct candidate | +| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — inline text descriptions, word-level control | Ready | Multi-engine arch in place | +| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | **Yes** — text prompts for style + timbre design | Needs vetting | Apache 2.0, multi-speaker dialogue | +| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Partial — automatic prosody from text context | Needs vetting | MIT, 700s+ coherent, synced transcript output | +| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Partial — automatic context-aware prosody | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion | +| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Partial — automatic style inference | Ready | Apache 2.0, multi-engine arch in place | +| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Partial — style transfer from ref audio only | Ready | Multi-engine arch in place | +| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | None | Needs vetting | MIT, Kyutai Labs, no GPU required | #### Notes on New Candidates (March 2026) -- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada) -- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions, no ref audio). Unique UX for Stories voice design. [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS) -- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Context-aware prosody/emotion, real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM) -- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts) +- **CosyVoice2-0.5B** — Best candidate for instruct support. `inference_instruct2()` accepts a text instruct parameter for emotions, speed, volume, dialects — and it works alongside voice cloning. This is the closest match to what users expect from our instruct UI. [HF: FunAudioLLM/CosyVoice2-0.5B](https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B) +- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. Prosody/emotion is automatic from text context, not user-controllable. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada) +- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions). VoiceGenerator unifies timbre design and style control via text prompts, usable as a layer for downstream TTS including cloning. [HF: OpenMOSS-Team/MOSS-VoiceGenerator](https://huggingface.co/OpenMOSS-Team/MOSS-VoiceGenerator) | [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS) +- **Fish Speech** — Word-level fine-grained control using plain language descriptions inline in the script. Works with cloning. Note: Fish Audio S2 has a restrictive research license (commercial use requires approval), but the open-source Fish Speech model may differ. Needs license clarification. [fish.audio blog](https://fish.audio/blog/fish-audio-s2-fine-grained-ai-voice-control-at-the-word-level) +- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Prosody/emotion is context-aware but automatic, not explicitly controllable via text prompt. Real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM) +- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. No style control. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts) - **Watch list:** MioTTS-2.6B (fast LLM-based EN/JP, vLLM compatible), Oolel-Voices (Soynade Research, expressive modular control) -- **Skipped:** Fish Audio S2 — restrictive research license (commercial use requires approval), despite strong features ### Adding a New Engine (Now Straightforward) -With the multi-engine architecture shipped, adding a new TTS engine requires: +With the model config registry and shared `EngineModelSelector` component, adding a new TTS engine requires: 1. **Create `backend/backends/_backend.py`** — implement `TTSBackend` protocol (~200-300 lines) -2. **Register in `backend/backends/__init__.py`** — add to `TTS_ENGINES` dict + factory function +2. **Register in `backend/backends/__init__.py`** — add `ModelConfig` entry + `TTS_ENGINES` entry + factory elif 3. **Update `backend/models.py`** — add engine name to regex -4. **Update `backend/main.py`** — add engine cases in generate, stream, model-status, download, delete (5 dispatch points) -5. **Update frontend** — add to engine union type, form schema, model dropdown, language map (5-6 files) +4. **Update frontend** — add to engine union type, `EngineModelSelector` options, form schema, language map (4 files) -Total effort: **~1 day** for a well-documented model with a PyPI package. +`main.py` requires **zero changes** — the registry handles all dispatch automatically. + +Total effort: **~1 day** for a well-documented model with a PyPI package. See `docs/plans/ADDING_TTS_ENGINES.md` for the full guide. --- @@ -367,13 +374,13 @@ Total effort: **~1 day** for a well-documented model with a PyPI package. The singleton TTS backend was replaced with a thread-safe per-engine registry in PR #254. Multiple engines can now be loaded simultaneously. -### 2. `main.py` is 2100+ Lines +### ~~2. `main.py` Dispatch Point Duplication~~ — RESOLVED -All API routes, all model configs, all business logic in one file. Five separate dispatch points for each engine. Any new engine touches this file in 5 places. A model config registry pattern would reduce duplication. +Previously, each engine required updates to 6+ hardcoded dispatch maps across `main.py` (~320 lines of if/elif chains). A model config registry in `backend/backends/__init__.py` now centralizes all model metadata (`ModelConfig` dataclass) with helper functions (`load_engine_model()`, `check_model_loaded()`, `engine_needs_trim()`, etc.). Adding a new engine requires zero changes to `main.py`. -### 3. Model Config is Scattered (Improved) +### ~~3. Model Config is Scattered~~ — RESOLVED -Model identifiers are still duplicated across `main.py` (3 dicts), backend files, frontend components, and the languages constant. However, the pattern is now consistent and well-understood. A centralized model registry would help but isn't blocking. +Model identifiers, HF repo IDs, display names, and engine metadata are now consolidated in the `ModelConfig` registry. Backend-aware branching (e.g. MLX vs PyTorch Qwen repo IDs) happens inside the registry. Frontend model options are centralized in `EngineModelSelector.tsx`. ### 4. Voice Prompt Cache Assumes PyTorch Tensors @@ -410,7 +417,7 @@ The generation form now uses a flat model dropdown with engine-based routing. Pe | 1 | **#253** — 48kHz speech tokenizer | Quality improvement | Medium | | 2 | **#161** — Docker deployment | Server/headless users | Medium | | 3 | **#154** — Audiobook tab | Long-form users | Medium | -| 4 | **Model config registry** | Reduce 5-dispatch-point duplication in main.py | Medium | +| 4 | ~~**Model config registry**~~ | ~~Reduce dispatch duplication in main.py~~ | **Done** | | 5 | **#225** — Custom HuggingFace models | User-supplied models | High (needs rework for multi-engine) | ### Tier 3 — Future (v0.3.0+) @@ -421,7 +428,7 @@ The generation form now uses a flat model dropdown with engine-based routing. Pe | 2 | **Pocket TTS** (Kyutai) | CPU-first 100M model, broadens hardware support. Kyutai ships clean code. Needs API vetting. | | 3 | **MOSS-TTS** | Text-to-voice design (no ref audio) is unique. Multi-speaker dialogue for Stories. Needs thorough API vetting. | | 4 | **Kokoro-82M** | 82M params, CPU realtime, Apache 2.0. Easy win. | -| 5 | **Model config registry refactor** | Reduce 5-dispatch-point duplication in main.py — do before adding 3+ more engines | +| 5 | ~~**Model config registry refactor**~~ | **Done** — consolidated in `backend/backends/__init__.py` + `EngineModelSelector.tsx` | | 6 | XTTS-v2 / Fish Speech / CosyVoice | Multi-engine arch is ready; just needs backend implementation | | 7 | **VoxCPM 1.5** | Tokenizer-free streaming, interesting but uncertain integration surface | | 8 | OpenAI-compatible API (plan doc exists) | Low effort once API is stable |