mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 04:40:40 -07:00
refactor start
This commit is contained in:
Jamie Pine
@@ -0,0 +1,96 @@
|
||||
import type { UseFormReturn } from 'react-hook-form';
|
||||
import { FormControl } from '@/components/ui/form';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { getLanguageOptionsForEngine } from '@/lib/constants/languages';
|
||||
import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm';
|
||||
|
||||
/**
|
||||
* Engine/model options and their display metadata.
|
||||
* Adding a new engine means adding one entry here.
|
||||
*/
|
||||
const ENGINE_OPTIONS = [
|
||||
{ value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B' },
|
||||
{ value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B' },
|
||||
{ value: 'luxtts', label: 'LuxTTS' },
|
||||
{ value: 'chatterbox', label: 'Chatterbox' },
|
||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
|
||||
] as const;
|
||||
|
||||
const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
||||
qwen: 'Multi-language, two sizes',
|
||||
luxtts: 'Fast, English-focused',
|
||||
chatterbox: '23 languages, incl. Hebrew',
|
||||
chatterbox_turbo: 'English, [laugh] [cough] tags',
|
||||
};
|
||||
|
||||
/** Engines that only support English and should force language to 'en' on select. */
|
||||
const ENGLISH_ONLY_ENGINES = new Set(['luxtts', 'chatterbox_turbo']);
|
||||
|
||||
function getSelectValue(engine: string, modelSize?: string): string {
|
||||
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
|
||||
return engine;
|
||||
}
|
||||
|
||||
function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) {
|
||||
if (value.startsWith('qwen:')) {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
} else {
|
||||
form.setValue('engine', value as GenerationFormValues['engine']);
|
||||
if (ENGLISH_ONLY_ENGINES.has(value)) {
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
// If current language isn't supported by the new engine, reset to first available
|
||||
const currentLang = form.getValues('language');
|
||||
const available = getLanguageOptionsForEngine(value);
|
||||
if (!available.some((l) => l.value === currentLang)) {
|
||||
form.setValue('language', available[0]?.value ?? 'en');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface EngineModelSelectorProps {
|
||||
form: UseFormReturn<GenerationFormValues>;
|
||||
compact?: boolean;
|
||||
}
|
||||
|
||||
export function EngineModelSelector({ form, compact }: EngineModelSelectorProps) {
|
||||
const engine = form.watch('engine') || 'qwen';
|
||||
const modelSize = form.watch('modelSize');
|
||||
const selectValue = getSelectValue(engine, modelSize);
|
||||
|
||||
const itemClass = compact ? 'text-xs text-muted-foreground' : undefined;
|
||||
const triggerClass = compact
|
||||
? 'h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all'
|
||||
: undefined;
|
||||
|
||||
return (
|
||||
<Select value={selectValue} onValueChange={(v) => handleEngineChange(form, v)}>
|
||||
<FormControl>
|
||||
<SelectTrigger className={triggerClass}>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{ENGINE_OPTIONS.map((opt) => (
|
||||
<SelectItem key={opt.value} value={opt.value} className={itemClass}>
|
||||
{opt.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns a human-readable description for the currently selected engine. */
|
||||
export function getEngineDescription(engine: string): string {
|
||||
return ENGINE_DESCRIPTIONS[engine] ?? '';
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { cn } from '@/lib/utils/cn';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
import { EngineModelSelector } from './EngineModelSelector';
|
||||
import { ParalinguisticInput } from './ParalinguisticInput';
|
||||
|
||||
interface FloatingGenerateBoxProps {
|
||||
@@ -455,58 +456,7 @@ export function FloatingGenerateBox({
|
||||
/>
|
||||
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
}
|
||||
}}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="qwen:1.7B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 1.7B
|
||||
</SelectItem>
|
||||
<SelectItem value="qwen:0.6B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 0.6B
|
||||
</SelectItem>
|
||||
<SelectItem value="luxtts" className="text-xs text-muted-foreground">
|
||||
LuxTTS
|
||||
</SelectItem>
|
||||
<SelectItem value="chatterbox" className="text-xs text-muted-foreground">
|
||||
Chatterbox
|
||||
</SelectItem>
|
||||
<SelectItem
|
||||
value="chatterbox_turbo"
|
||||
className="text-xs text-muted-foreground"
|
||||
>
|
||||
Chatterbox Turbo
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<EngineModelSelector form={form} compact />
|
||||
</FormItem>
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
@@ -23,6 +23,7 @@ import { getLanguageOptionsForEngine } from '@/lib/constants/languages';
|
||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||
import { useProfile } from '@/lib/hooks/useProfiles';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
import { EngineModelSelector, getEngineDescription } from './EngineModelSelector';
|
||||
import { ParalinguisticInput } from './ParalinguisticInput';
|
||||
|
||||
export function GenerationForm() {
|
||||
@@ -117,53 +118,9 @@ export function GenerationForm() {
|
||||
<div className="grid gap-4 md:grid-cols-3">
|
||||
<FormItem>
|
||||
<FormLabel>Model</FormLabel>
|
||||
<Select
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
}
|
||||
}}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="qwen:1.7B">Qwen3-TTS 1.7B</SelectItem>
|
||||
<SelectItem value="qwen:0.6B">Qwen3-TTS 0.6B</SelectItem>
|
||||
<SelectItem value="luxtts">LuxTTS</SelectItem>
|
||||
<SelectItem value="chatterbox">Chatterbox</SelectItem>
|
||||
<SelectItem value="chatterbox_turbo">Chatterbox Turbo</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<EngineModelSelector form={form} />
|
||||
<FormDescription>
|
||||
{form.watch('engine') === 'luxtts'
|
||||
? 'Fast, English-focused'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? '23 languages, incl. Hebrew'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'English, [laugh] [cough] tags'
|
||||
: 'Multi-language, two sizes'}
|
||||
{getEngineDescription(form.watch('engine') || 'qwen')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { Link, useMatchRoute } from '@tanstack/react-router';
|
||||
import { AudioLines, Box, Mic, Server, Speaker, Volume2, Wand2 } from 'lucide-react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import voiceboxLogo from '@/assets/voicebox-logo.png';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import type { UpdateStatus } from '@/platform/types';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { version } from '../../package.json';
|
||||
|
||||
@@ -22,6 +25,10 @@ const tabs = [
|
||||
export function Sidebar({ isMacOS }: SidebarProps) {
|
||||
const matchRoute = useMatchRoute();
|
||||
const isPlayerOpen = !!usePlayerStore((s) => s.audioUrl);
|
||||
const platform = usePlatform();
|
||||
|
||||
const [updateStatus, setUpdateStatus] = useState<UpdateStatus>(platform.updater.getStatus());
|
||||
useEffect(() => platform.updater.subscribe(setUpdateStatus), [platform.updater]);
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -85,10 +92,18 @@ export function Sidebar({ isMacOS }: SidebarProps) {
|
||||
|
||||
{/* Version */}
|
||||
<div
|
||||
className="mt-auto text-[10px] text-muted-foreground/50 transition-all duration-300"
|
||||
className="mt-auto flex flex-col items-center gap-1.5 transition-all duration-300"
|
||||
style={{ paddingBottom: isPlayerOpen ? '7rem' : undefined }}
|
||||
>
|
||||
v{version}
|
||||
<span className="text-[10px] text-muted-foreground/50">v{version}</span>
|
||||
{updateStatus.available && (
|
||||
<Link
|
||||
to="/server"
|
||||
className="text-[9px] font-semibold tracking-wide uppercase px-2 py-0.5 rounded-full bg-accent/15 text-accent hover:bg-accent/25 transition-colors"
|
||||
>
|
||||
Update
|
||||
</Link>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# Backend Refactor Plan
|
||||
|
||||
## Current State
|
||||
|
||||
2,856-line god file (`main.py`), ~500 lines of copy-pasted backend methods, 3x duplicated generation orchestration, dead modules, fake async, scattered constants. 72 routes all registered in one file. Works fine, but will fight us on every new feature.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Dead Code & Low-Hanging Fruit
|
||||
|
||||
Remove noise so the real structure is easier to see.
|
||||
|
||||
- Delete `studio.py` (66 lines, every method raises `NotImplementedError`, never imported)
|
||||
- Delete `migrate_add_instruct.py` (48 lines, superseded by `database.py` migrations)
|
||||
- Delete `utils/validation.py` (66 lines, none of the 3 functions are called anywhere)
|
||||
- Remove duplicate `_profile_to_response()` in `main.py:1983-2005`, use the one from `profiles.py`
|
||||
- Remove duplicate `import asyncio` in `main.py`
|
||||
- Remove pointless one-line wrappers (`_get_profiles_dir()`, `_get_generations_dir()`) in `profiles.py`, `history.py`, `export_import.py` — call `config.*` directly
|
||||
- Deduplicate `LANGUAGE_CODE_TO_NAME` (defined in both `pytorch_backend.py:18` and `mlx_backend.py:24`) — move to `backends/__init__.py`
|
||||
- Deduplicate `WHISPER_HF_REPOS` (defined in both `pytorch_backend.py:379` and `mlx_backend.py:416`) — move to `backends/__init__.py`
|
||||
- Update `README.md` to reflect actual file structure (it still references `studio.py` and the old two-backend layout)
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Backend Deduplication
|
||||
|
||||
The backends have 5-7 copies of identical or near-identical methods. This is the highest-value structural change because it removes ~500 lines and makes adding new engines trivial.
|
||||
|
||||
### Extract shared methods
|
||||
|
||||
Create `backends/base.py` with:
|
||||
|
||||
- **`is_model_cached(hf_repo, hf_revision)`** — the HuggingFace cache directory check. Currently copy-pasted in `pytorch_backend.py:81`, `mlx_backend.py:68`, `chatterbox_backend.py:66`, `chatterbox_turbo_backend.py:66`, `luxtts_backend.py:57`, and both STT backends. One function, parameterized by repo/revision.
|
||||
|
||||
- **`combine_voice_prompts(samples, sample_rate, backend_type)`** — load audio, normalize, concatenate, join texts. Identical in all 5 TTS backends (`pytorch:301`, `mlx:291`, `chatterbox:266`, `chatterbox_turbo:269`, `luxtts:208`). The only variation is which audio loading function is used (torchaudio vs mlx_audio) — pass the loader as a parameter or detect from backend type.
|
||||
|
||||
- **`get_device(backend_type)`** — device detection. Currently 5 slightly different implementations. Parameterize the differences:
|
||||
- PyTorch: checks CUDA > XPU > DirectML > MPS > CPU
|
||||
- Chatterbox/Chatterbox Turbo: forces CPU on macOS, otherwise CUDA > CPU
|
||||
- LuxTTS: checks MPS > CUDA > CPU
|
||||
|
||||
- **`model_load_wrapper(load_fn, ...)`** — the progress tracking boilerplate shared by all 7 `_load_model_sync` implementations. Every backend does the same setup/teardown dance with `progress_manager`, `task_manager`, `HFProgressTracker`, and tqdm patching. Extract the wrapper, backends just supply the actual model loading callable.
|
||||
|
||||
### Extract Chatterbox f32 patch
|
||||
|
||||
Move the S3Tokenizer / VoiceEncoder monkey-patches from `chatterbox_backend.py:189-210` and `chatterbox_turbo_backend.py:193-214` into a shared `backends/chatterbox_patches.py` (or a function in `base.py`). Both files have identical code.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Generation Service
|
||||
|
||||
The three generation closures in `main.py` (`_run_generation:782`, `_run_retry:923`, `_run_regenerate:1018`) share ~80% of their logic. Extract into a service module.
|
||||
|
||||
### Create `services/generation.py`
|
||||
|
||||
Single orchestration function with mode parameter:
|
||||
|
||||
```python
|
||||
async def run_generation(
|
||||
generation_id: str,
|
||||
profile_id: str,
|
||||
text: str,
|
||||
language: str,
|
||||
engine: str,
|
||||
model_size: str,
|
||||
seed: Optional[int],
|
||||
normalize: bool,
|
||||
effects_chain: Optional[list],
|
||||
instruct_text: Optional[str],
|
||||
mode: Literal["generate", "retry", "regenerate"],
|
||||
version_label: Optional[str] = None,
|
||||
):
|
||||
```
|
||||
|
||||
Differences between modes are small and can be handled with conditionals:
|
||||
- `retry`: reuses same seed, skips effects/versions
|
||||
- `regenerate`: seed=None, creates a new version with auto-label
|
||||
- `generate`: full pipeline including effects version
|
||||
|
||||
### Move background queue management
|
||||
|
||||
Move `_generation_queue`, `_generation_worker`, `_enqueue_generation`, `_background_tasks`, and `_create_background_task` (currently `main.py:63-92`) into the service module or a dedicated `services/task_queue.py`.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Route Extraction
|
||||
|
||||
Split `main.py` (72 routes) into domain-specific routers. After Phase 3, the route handlers should be thin — just validation, delegation, and response formatting.
|
||||
|
||||
### Target structure
|
||||
|
||||
```
|
||||
backend/
|
||||
app.py # FastAPI app creation, middleware, startup/shutdown
|
||||
routes/
|
||||
__init__.py
|
||||
health.py # GET /, /health, /health/filesystem, /shutdown, /watchdog/disable (5 routes)
|
||||
profiles.py # All /profiles/* routes (17 routes)
|
||||
channels.py # All /channels/* routes (7 routes)
|
||||
generations.py # /generate, /generate/stream, /generate/*/retry, regenerate, status (5 routes)
|
||||
history.py # All /history/* routes (8 routes)
|
||||
stories.py # All /stories/* routes (15 routes)
|
||||
effects.py # All /effects/* routes + /generations/*/versions/* (11 routes)
|
||||
audio.py # /audio/*, /samples/* (2 routes)
|
||||
models.py # All /models/* routes (11 routes)
|
||||
tasks.py # /tasks/*, /cache/* (3 routes)
|
||||
cuda.py # /backend/cuda-* (4 routes)
|
||||
services/
|
||||
generation.py # TTS orchestration (from Phase 3)
|
||||
model_status.py # HF cache inspection logic (currently inline at main.py:2251-2431)
|
||||
```
|
||||
|
||||
`main.py` becomes a thin entry point that imports the app from `app.py` and runs uvicorn (preserving backward compat for `python -m backend.main`).
|
||||
|
||||
### Model status extraction
|
||||
|
||||
The `get_model_status` endpoint (`main.py:2251-2431`) is 180 lines of HuggingFace cache inspection that duplicates logic from `_is_model_cached` in the backends. Extract to `services/model_status.py` and reuse the shared `is_model_cached` from Phase 2 where possible.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Database Cleanup
|
||||
|
||||
### Split `database.py` (487 lines)
|
||||
|
||||
- `database/models.py` — ORM model definitions (11 models, ~140 lines)
|
||||
- `database/migrations.py` — migration logic (`_run_migrations`, ~200 lines)
|
||||
- `database/seed.py` — `_backfill_generation_versions` + `_seed_builtin_presets`
|
||||
- `database/session.py` — engine creation, `init_db()`, `get_db()`
|
||||
|
||||
### Fix async-over-sync CRUD modules
|
||||
|
||||
`channels.py`, `history.py`, `stories.py`, `effects.py`, `versions.py`, `profiles.py` all declare `async def` but never `await`. They run synchronous SQLAlchemy queries directly, blocking the event loop. Two options:
|
||||
|
||||
- **Option A**: Drop `async` keyword, wrap calls in `asyncio.to_thread()` at the route layer
|
||||
- **Option B**: Switch to async SQLAlchemy (`create_async_engine` + `AsyncSession`)
|
||||
|
||||
Option A is simpler and non-disruptive. Option B is cleaner long-term but touches every query.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Polish
|
||||
|
||||
- Consolidate hardcoded constants (`24000` sample rate, `100MB`/`50MB` max file sizes, `HSA_OVERRIDE_GFX_VERSION`, CORS origins) into `config.py` or a `constants.py`
|
||||
- Fix `hf_offline_patch.py` side-effect-on-import (runs patching twice — once on import, once explicitly in `mlx_backend.py`)
|
||||
- Standardize error handling across routes (currently three different patterns)
|
||||
- Rename `effects.py` (preset CRUD) to avoid confusion with `utils/effects.py` (DSP engine) — either rename to `effect_presets.py` or fold into routes
|
||||
- Clean up test suite — the 4 manual integration scripts in `tests/` should either be converted to pytest or moved to a `scripts/` dir
|
||||
|
||||
---
|
||||
|
||||
## Notes
|
||||
|
||||
- Each phase is independently shippable and testable
|
||||
- Phase 1 is zero-risk deletion
|
||||
- Phase 2 is self-contained within `backends/`
|
||||
- Phase 3 sets up the extraction pattern needed for Phase 4
|
||||
- Phase 4 is the largest change but should be mostly mechanical after Phase 3
|
||||
- Phase 5 can run in parallel with Phase 4 since it touches different files
|
||||
+273
-30
@@ -1,10 +1,12 @@
|
||||
"""
|
||||
Backend abstraction layer for TTS and STT.
|
||||
|
||||
Provides a unified interface for MLX and PyTorch backends.
|
||||
Provides a unified interface for MLX and PyTorch backends,
|
||||
and a model config registry that eliminates per-engine dispatch maps.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol, Optional, Tuple, List
|
||||
from typing_extensions import runtime_checkable
|
||||
import numpy as np
|
||||
@@ -12,14 +14,31 @@ import numpy as np
|
||||
from ..platform_detect import get_backend_type
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
"""Declarative config for a downloadable model variant."""
|
||||
model_name: str # e.g. "luxtts", "chatterbox-tts"
|
||||
display_name: str # e.g. "LuxTTS (Fast, CPU-friendly)"
|
||||
engine: str # e.g. "luxtts", "chatterbox"
|
||||
hf_repo_id: str # e.g. "YatharthS/LuxTTS"
|
||||
model_size: str = "default"
|
||||
size_mb: int = 0
|
||||
needs_trim: bool = False
|
||||
supports_instruct: bool = False
|
||||
languages: list[str] = field(default_factory=lambda: ["en"])
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TTSBackend(Protocol):
|
||||
"""Protocol for TTS backend implementations."""
|
||||
|
||||
|
||||
# Each backend class should define MODEL_CONFIGS as a class variable:
|
||||
# MODEL_CONFIGS: list[ModelConfig]
|
||||
|
||||
async def load_model(self, model_size: str) -> None:
|
||||
"""Load TTS model."""
|
||||
...
|
||||
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
@@ -28,12 +47,12 @@ class TTSBackend(Protocol):
|
||||
) -> Tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt from reference audio.
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (voice_prompt_dict, was_cached)
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: List[str],
|
||||
@@ -41,12 +60,12 @@ class TTSBackend(Protocol):
|
||||
) -> Tuple[np.ndarray, str]:
|
||||
"""
|
||||
Combine multiple voice prompts.
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (combined_audio_array, combined_text)
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
@@ -57,24 +76,24 @@ class TTSBackend(Protocol):
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio from text.
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
...
|
||||
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
"""Check if model is loaded."""
|
||||
...
|
||||
|
||||
|
||||
def _get_model_path(self, model_size: str) -> str:
|
||||
"""
|
||||
Get model path for a given size.
|
||||
|
||||
|
||||
Returns:
|
||||
Model path or HuggingFace Hub ID
|
||||
"""
|
||||
@@ -84,11 +103,11 @@ class TTSBackend(Protocol):
|
||||
@runtime_checkable
|
||||
class STTBackend(Protocol):
|
||||
"""Protocol for STT (Speech-to-Text) backend implementations."""
|
||||
|
||||
|
||||
async def load_model(self, model_size: str) -> None:
|
||||
"""Load STT model."""
|
||||
...
|
||||
|
||||
|
||||
async def transcribe(
|
||||
self,
|
||||
audio_path: str,
|
||||
@@ -96,16 +115,16 @@ class STTBackend(Protocol):
|
||||
) -> str:
|
||||
"""
|
||||
Transcribe audio to text.
|
||||
|
||||
|
||||
Returns:
|
||||
Transcribed text
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
...
|
||||
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
"""Check if model is loaded."""
|
||||
...
|
||||
@@ -117,7 +136,8 @@ _tts_backends: dict[str, TTSBackend] = {}
|
||||
_tts_backends_lock = threading.Lock()
|
||||
_stt_backend: Optional[STTBackend] = None
|
||||
|
||||
# Supported TTS engines
|
||||
# Supported TTS engines — keyed by engine name, value is the backend class import path.
|
||||
# The factory function uses this for the if/elif chain; the model configs live on the backend classes.
|
||||
TTS_ENGINES = {
|
||||
"qwen": "Qwen TTS",
|
||||
"luxtts": "LuxTTS",
|
||||
@@ -126,10 +146,233 @@ TTS_ENGINES = {
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model config registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _get_qwen_model_configs() -> list[ModelConfig]:
|
||||
"""Return Qwen model configs with backend-aware HF repo IDs."""
|
||||
backend_type = get_backend_type()
|
||||
if backend_type == "mlx":
|
||||
repo_1_7b = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
||||
repo_0_6b = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" # 0.6B not available in MLX, falls back
|
||||
else:
|
||||
repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
|
||||
return [
|
||||
ModelConfig(
|
||||
model_name="qwen-tts-1.7B",
|
||||
display_name="Qwen TTS 1.7B",
|
||||
engine="qwen",
|
||||
hf_repo_id=repo_1_7b,
|
||||
model_size="1.7B",
|
||||
size_mb=3500,
|
||||
supports_instruct=False, # Base model drops instruct silently
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
ModelConfig(
|
||||
model_name="qwen-tts-0.6B",
|
||||
display_name="Qwen TTS 0.6B",
|
||||
engine="qwen",
|
||||
hf_repo_id=repo_0_6b,
|
||||
model_size="0.6B",
|
||||
size_mb=1200,
|
||||
supports_instruct=False,
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _get_non_qwen_tts_configs() -> list[ModelConfig]:
|
||||
"""Return model configs for non-Qwen TTS engines.
|
||||
|
||||
These are static — no backend-type branching needed.
|
||||
"""
|
||||
return [
|
||||
ModelConfig(
|
||||
model_name="luxtts",
|
||||
display_name="LuxTTS (Fast, CPU-friendly)",
|
||||
engine="luxtts",
|
||||
hf_repo_id="YatharthS/LuxTTS",
|
||||
size_mb=300,
|
||||
languages=["en"],
|
||||
),
|
||||
ModelConfig(
|
||||
model_name="chatterbox-tts",
|
||||
display_name="Chatterbox TTS (Multilingual)",
|
||||
engine="chatterbox",
|
||||
hf_repo_id="ResembleAI/chatterbox",
|
||||
size_mb=3200,
|
||||
needs_trim=True,
|
||||
languages=[
|
||||
"zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it",
|
||||
"he", "ar", "da", "el", "fi", "hi", "ms", "nl", "no", "pl",
|
||||
"sv", "sw", "tr",
|
||||
],
|
||||
),
|
||||
ModelConfig(
|
||||
model_name="chatterbox-turbo",
|
||||
display_name="Chatterbox Turbo (English, Tags)",
|
||||
engine="chatterbox_turbo",
|
||||
hf_repo_id="ResembleAI/chatterbox-turbo",
|
||||
size_mb=1500,
|
||||
needs_trim=True,
|
||||
languages=["en"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _get_whisper_configs() -> list[ModelConfig]:
|
||||
"""Return Whisper STT model configs."""
|
||||
return [
|
||||
ModelConfig(model_name="whisper-base", display_name="Whisper Base", engine="whisper", hf_repo_id="openai/whisper-base", model_size="base"),
|
||||
ModelConfig(model_name="whisper-small", display_name="Whisper Small", engine="whisper", hf_repo_id="openai/whisper-small", model_size="small"),
|
||||
ModelConfig(model_name="whisper-medium", display_name="Whisper Medium", engine="whisper", hf_repo_id="openai/whisper-medium", model_size="medium"),
|
||||
ModelConfig(model_name="whisper-large", display_name="Whisper Large", engine="whisper", hf_repo_id="openai/whisper-large-v3", model_size="large"),
|
||||
ModelConfig(model_name="whisper-turbo", display_name="Whisper Turbo", engine="whisper", hf_repo_id="openai/whisper-large-v3-turbo", model_size="turbo"),
|
||||
]
|
||||
|
||||
|
||||
def get_all_model_configs() -> list[ModelConfig]:
|
||||
"""Return the full list of model configs (TTS + STT)."""
|
||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
|
||||
|
||||
|
||||
def get_tts_model_configs() -> list[ModelConfig]:
|
||||
"""Return only TTS model configs."""
|
||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs()
|
||||
|
||||
|
||||
# Lookup helpers — these replace the if/elif chains in main.py
|
||||
|
||||
def get_model_config(model_name: str) -> Optional[ModelConfig]:
|
||||
"""Look up a model config by model_name."""
|
||||
for cfg in get_all_model_configs():
|
||||
if cfg.model_name == model_name:
|
||||
return cfg
|
||||
return None
|
||||
|
||||
|
||||
def engine_needs_trim(engine: str) -> bool:
|
||||
"""Whether this engine's output should be run through trim_tts_output."""
|
||||
for cfg in get_tts_model_configs():
|
||||
if cfg.engine == engine:
|
||||
return cfg.needs_trim
|
||||
return False
|
||||
|
||||
|
||||
def engine_has_model_sizes(engine: str) -> bool:
|
||||
"""Whether this engine supports multiple model sizes (only Qwen currently)."""
|
||||
configs = [c for c in get_tts_model_configs() if c.engine == engine]
|
||||
return len(configs) > 1
|
||||
|
||||
|
||||
async def load_engine_model(engine: str, model_size: str = "default") -> None:
|
||||
"""Load a model for the given engine, handling the Qwen model_size special case."""
|
||||
backend = get_tts_backend_for_engine(engine)
|
||||
if engine == "qwen":
|
||||
await backend.load_model_async(model_size)
|
||||
else:
|
||||
await backend.load_model()
|
||||
|
||||
|
||||
async def ensure_model_cached_or_raise(engine: str, model_size: str = "default") -> None:
|
||||
"""Check if a model is cached, raise HTTPException if not. Used by streaming endpoint."""
|
||||
from fastapi import HTTPException
|
||||
backend = get_tts_backend_for_engine(engine)
|
||||
cfg = None
|
||||
for c in get_tts_model_configs():
|
||||
if c.engine == engine and c.model_size == model_size:
|
||||
cfg = c
|
||||
break
|
||||
|
||||
if engine == "qwen":
|
||||
if not backend._is_model_cached(model_size):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
else:
|
||||
if not backend._is_model_cached():
|
||||
display = cfg.display_name if cfg else engine
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"{display} model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
|
||||
|
||||
def unload_model_by_config(config: ModelConfig) -> bool:
|
||||
"""Unload a model given its config. Returns True if it was loaded, False otherwise."""
|
||||
from . import get_tts_backend_for_engine
|
||||
from .. import tts, transcribe
|
||||
|
||||
if config.engine == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == config.model_size:
|
||||
transcribe.unload_whisper_model()
|
||||
return True
|
||||
return False
|
||||
|
||||
if config.engine == "qwen":
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
|
||||
if tts_model.is_loaded() and loaded_size == config.model_size:
|
||||
tts.unload_tts_model()
|
||||
return True
|
||||
return False
|
||||
|
||||
# All other TTS engines
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_model_loaded(config: ModelConfig) -> bool:
|
||||
"""Check if a model is currently loaded."""
|
||||
from . import get_tts_backend_for_engine
|
||||
from .. import tts, transcribe
|
||||
|
||||
try:
|
||||
if config.engine == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
return whisper_model.is_loaded() and getattr(whisper_model, 'model_size', None) == config.model_size
|
||||
|
||||
if config.engine == "qwen":
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
|
||||
return tts_model.is_loaded() and loaded_size == config.model_size
|
||||
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_model_load_func(config: ModelConfig):
|
||||
"""Return a callable that loads/downloads the model."""
|
||||
from . import get_tts_backend_for_engine
|
||||
from .. import tts, transcribe
|
||||
|
||||
if config.engine == "whisper":
|
||||
return lambda: transcribe.get_whisper_model().load_model(config.model_size)
|
||||
|
||||
if config.engine == "qwen":
|
||||
return lambda: tts.get_tts_model().load_model(config.model_size)
|
||||
|
||||
return lambda: get_tts_backend_for_engine(config.engine).load_model()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backend factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_tts_backend() -> TTSBackend:
|
||||
"""
|
||||
Get or create the default (Qwen) TTS backend instance based on platform.
|
||||
|
||||
|
||||
Returns:
|
||||
TTS backend instance (MLX or PyTorch)
|
||||
"""
|
||||
@@ -139,25 +382,25 @@ def get_tts_backend() -> TTSBackend:
|
||||
def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
"""
|
||||
Get or create a TTS backend for the given engine.
|
||||
|
||||
|
||||
Args:
|
||||
engine: Engine name ("qwen" or "luxtts")
|
||||
|
||||
engine: Engine name (e.g. "qwen", "luxtts", "chatterbox", "chatterbox_turbo")
|
||||
|
||||
Returns:
|
||||
TTS backend instance
|
||||
"""
|
||||
global _tts_backends
|
||||
|
||||
|
||||
# Fast path: check without lock
|
||||
if engine in _tts_backends:
|
||||
return _tts_backends[engine]
|
||||
|
||||
|
||||
# Slow path: create with lock to avoid duplicate instantiation
|
||||
with _tts_backends_lock:
|
||||
# Double-check after acquiring lock
|
||||
if engine in _tts_backends:
|
||||
return _tts_backends[engine]
|
||||
|
||||
|
||||
if engine == "qwen":
|
||||
backend_type = get_backend_type()
|
||||
if backend_type == "mlx":
|
||||
@@ -177,7 +420,7 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
backend = ChatterboxTurboTTSBackend()
|
||||
else:
|
||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||
|
||||
|
||||
_tts_backends[engine] = backend
|
||||
return backend
|
||||
|
||||
@@ -185,22 +428,22 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
def get_stt_backend() -> STTBackend:
|
||||
"""
|
||||
Get or create STT backend instance based on platform.
|
||||
|
||||
|
||||
Returns:
|
||||
STT backend instance (MLX or PyTorch)
|
||||
"""
|
||||
global _stt_backend
|
||||
|
||||
|
||||
if _stt_backend is None:
|
||||
backend_type = get_backend_type()
|
||||
|
||||
|
||||
if backend_type == "mlx":
|
||||
from .mlx_backend import MLXSTTBackend
|
||||
_stt_backend = MLXSTTBackend()
|
||||
else:
|
||||
from .pytorch_backend import PyTorchSTTBackend
|
||||
_stt_backend = PyTorchSTTBackend()
|
||||
|
||||
|
||||
return _stt_backend
|
||||
|
||||
|
||||
|
||||
+49
-370
@@ -233,11 +233,9 @@ async def health():
|
||||
model_downloaded = None
|
||||
try:
|
||||
# Check if the default model (1.7B) is cached
|
||||
# Use different model IDs based on backend
|
||||
if backend_type == "mlx":
|
||||
default_model_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
||||
else:
|
||||
default_model_id = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
from .backends import get_model_config
|
||||
default_config = get_model_config("qwen-tts-1.7B")
|
||||
default_model_id = default_config.hf_repo_id if default_config else "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
|
||||
# Method 1: Try scan_cache_dir if available
|
||||
try:
|
||||
@@ -738,7 +736,7 @@ async def generate_speech(
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
|
||||
from .backends import get_tts_backend_for_engine
|
||||
from .backends import get_tts_backend_for_engine, engine_has_model_sizes
|
||||
engine = data.engine or "qwen"
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
@@ -756,7 +754,7 @@ async def generate_speech(
|
||||
generation_id=generation_id,
|
||||
status="generating",
|
||||
engine=engine,
|
||||
model_size=model_size if engine == "qwen" else None,
|
||||
model_size=model_size if engine_has_model_sizes(engine) else None,
|
||||
)
|
||||
|
||||
# Track in task manager
|
||||
@@ -785,10 +783,8 @@ async def generate_speech(
|
||||
bg_db = next(get_db())
|
||||
try:
|
||||
# Load model
|
||||
if engine == "qwen":
|
||||
await tts_model.load_model_async(model_size)
|
||||
else:
|
||||
await tts_model.load_model()
|
||||
from .backends import load_engine_model, engine_needs_trim
|
||||
await load_engine_model(engine, model_size)
|
||||
|
||||
# Create voice prompt
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
@@ -801,7 +797,7 @@ async def generate_speech(
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
if engine_needs_trim(engine):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
@@ -927,10 +923,8 @@ async def retry_generation(generation_id: str, db: Session = Depends(get_db)):
|
||||
async def _run_retry():
|
||||
bg_db = next(get_db())
|
||||
try:
|
||||
if retry_engine == "qwen":
|
||||
await tts_model.load_model_async(retry_model_size)
|
||||
else:
|
||||
await tts_model.load_model()
|
||||
from .backends import load_engine_model, engine_needs_trim
|
||||
await load_engine_model(retry_engine, retry_model_size)
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
gen.profile_id,
|
||||
@@ -942,7 +936,7 @@ async def retry_generation(generation_id: str, db: Session = Depends(get_db)):
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if retry_engine in ("chatterbox", "chatterbox_turbo"):
|
||||
if engine_needs_trim(retry_engine):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
@@ -1024,10 +1018,8 @@ async def regenerate_generation(generation_id: str, db: Session = Depends(get_db
|
||||
async def _run_regenerate():
|
||||
bg_db = next(get_db())
|
||||
try:
|
||||
if regen_engine == "qwen":
|
||||
await tts_model.load_model_async(regen_model_size)
|
||||
else:
|
||||
await tts_model.load_model()
|
||||
from .backends import load_engine_model, engine_needs_trim
|
||||
await load_engine_model(regen_engine, regen_model_size)
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
gen.profile_id,
|
||||
@@ -1039,7 +1031,7 @@ async def regenerate_generation(generation_id: str, db: Session = Depends(get_db
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if regen_engine in ("chatterbox", "chatterbox_turbo"):
|
||||
if engine_needs_trim(regen_engine):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
@@ -1162,34 +1154,9 @@ async def stream_speech(
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
|
||||
if engine == "qwen":
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model_async(model_size)
|
||||
elif engine == "luxtts":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox_turbo":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Chatterbox Turbo model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
from .backends import ensure_model_cached_or_raise, load_engine_model, engine_needs_trim
|
||||
await ensure_model_cached_or_raise(engine, model_size)
|
||||
await load_engine_model(engine, model_size)
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id, db, engine=engine,
|
||||
@@ -1198,7 +1165,7 @@ async def stream_speech(
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
if engine_needs_trim(engine):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
@@ -2108,63 +2075,16 @@ async def unload_model():
|
||||
@app.post("/models/{model_name}/unload")
|
||||
async def unload_model_by_name(model_name: str):
|
||||
"""Unload a specific model from memory without deleting it from disk."""
|
||||
# Map of model_name -> (model_type, model_size)
|
||||
model_types = {
|
||||
"qwen-tts-1.7B": ("tts", "1.7B"),
|
||||
"qwen-tts-0.6B": ("tts", "0.6B"),
|
||||
"luxtts": ("luxtts", "default"),
|
||||
"chatterbox-tts": ("chatterbox", "default"),
|
||||
"chatterbox-turbo": ("chatterbox_turbo", "default"),
|
||||
"whisper-base": ("whisper", "base"),
|
||||
"whisper-small": ("whisper", "small"),
|
||||
"whisper-medium": ("whisper", "medium"),
|
||||
"whisper-large": ("whisper", "large"),
|
||||
"whisper-turbo": ("whisper", "turbo"),
|
||||
}
|
||||
from .backends import get_model_config, unload_model_by_config
|
||||
|
||||
if model_name not in model_types:
|
||||
config = get_model_config(model_name)
|
||||
if not config:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model_name}")
|
||||
|
||||
model_type, model_size = model_types[model_name]
|
||||
|
||||
try:
|
||||
if model_type == "tts":
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
if tts_model.is_loaded() and loaded_size == model_size:
|
||||
tts.unload_tts_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "luxtts":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("luxtts")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "chatterbox":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "chatterbox_turbo":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == model_size:
|
||||
transcribe.unload_whisper_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
|
||||
was_loaded = unload_model_by_config(config)
|
||||
if not was_loaded:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
return {"message": f"Model {model_name} unloaded successfully"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
@@ -2347,140 +2267,18 @@ async def get_model_status():
|
||||
except ImportError:
|
||||
use_scan_cache = False
|
||||
|
||||
def check_tts_loaded(model_size: str):
|
||||
"""Check if TTS model is loaded with specific size."""
|
||||
try:
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
return tts_model.is_loaded() and loaded_size == model_size
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def check_whisper_loaded(model_size: str):
|
||||
"""Check if Whisper model is loaded with specific size."""
|
||||
try:
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
return whisper_model.is_loaded() and getattr(whisper_model, 'model_size', None) == model_size
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Use backend-specific model IDs
|
||||
if backend_type == "mlx":
|
||||
tts_1_7b_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
||||
tts_0_6b_id = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" # Fallback to 1.7B
|
||||
# MLX backend uses openai/whisper-* models, not mlx-community
|
||||
whisper_base_id = "openai/whisper-base"
|
||||
whisper_small_id = "openai/whisper-small"
|
||||
whisper_medium_id = "openai/whisper-medium"
|
||||
whisper_large_id = "openai/whisper-large-v3"
|
||||
else:
|
||||
tts_1_7b_id = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
tts_0_6b_id = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
whisper_base_id = "openai/whisper-base"
|
||||
whisper_small_id = "openai/whisper-small"
|
||||
whisper_medium_id = "openai/whisper-medium"
|
||||
whisper_large_id = "openai/whisper-large-v3"
|
||||
|
||||
# Check if LuxTTS backend is loaded
|
||||
def check_luxtts_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("luxtts")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Check if Chatterbox backend is loaded
|
||||
def check_chatterbox_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Check if Chatterbox Turbo backend is loaded
|
||||
def check_chatterbox_turbo_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
from .backends import get_all_model_configs, check_model_loaded
|
||||
|
||||
registry_configs = get_all_model_configs()
|
||||
model_configs = [
|
||||
{
|
||||
"model_name": "qwen-tts-1.7B",
|
||||
"display_name": "Qwen TTS 1.7B",
|
||||
"hf_repo_id": tts_1_7b_id,
|
||||
"model_size": "1.7B",
|
||||
"check_loaded": lambda: check_tts_loaded("1.7B"),
|
||||
},
|
||||
{
|
||||
"model_name": "qwen-tts-0.6B",
|
||||
"display_name": "Qwen TTS 0.6B",
|
||||
"hf_repo_id": tts_0_6b_id,
|
||||
"model_size": "0.6B",
|
||||
"check_loaded": lambda: check_tts_loaded("0.6B"),
|
||||
},
|
||||
{
|
||||
"model_name": "luxtts",
|
||||
"display_name": "LuxTTS (Fast, CPU-friendly)",
|
||||
"hf_repo_id": "YatharthS/LuxTTS",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_luxtts_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "chatterbox-tts",
|
||||
"display_name": "Chatterbox TTS (Multilingual)",
|
||||
"hf_repo_id": "ResembleAI/chatterbox",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "chatterbox-turbo",
|
||||
"display_name": "Chatterbox Turbo (English, Tags)",
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_turbo_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-base",
|
||||
"display_name": "Whisper Base",
|
||||
"hf_repo_id": whisper_base_id,
|
||||
"model_size": "base",
|
||||
"check_loaded": lambda: check_whisper_loaded("base"),
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-small",
|
||||
"display_name": "Whisper Small",
|
||||
"hf_repo_id": whisper_small_id,
|
||||
"model_size": "small",
|
||||
"check_loaded": lambda: check_whisper_loaded("small"),
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-medium",
|
||||
"display_name": "Whisper Medium",
|
||||
"hf_repo_id": whisper_medium_id,
|
||||
"model_size": "medium",
|
||||
"check_loaded": lambda: check_whisper_loaded("medium"),
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-large",
|
||||
"display_name": "Whisper Large",
|
||||
"hf_repo_id": whisper_large_id,
|
||||
"model_size": "large",
|
||||
"check_loaded": lambda: check_whisper_loaded("large"),
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-turbo",
|
||||
"display_name": "Whisper Turbo",
|
||||
"hf_repo_id": "openai/whisper-large-v3-turbo",
|
||||
"model_size": "turbo",
|
||||
"check_loaded": lambda: check_whisper_loaded("turbo"),
|
||||
},
|
||||
"model_name": cfg.model_name,
|
||||
"display_name": cfg.display_name,
|
||||
"hf_repo_id": cfg.hf_repo_id,
|
||||
"model_size": cfg.model_size,
|
||||
"check_loaded": lambda c=cfg: check_model_loaded(c),
|
||||
}
|
||||
for cfg in registry_configs
|
||||
]
|
||||
|
||||
# Build a mapping of model_name -> hf_repo_id so we can check if shared repos are downloading
|
||||
@@ -2637,64 +2435,22 @@ async def get_model_status():
|
||||
async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"""Trigger download of a specific model."""
|
||||
import asyncio
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
from .backends import get_model_config, get_model_load_func
|
||||
|
||||
task_manager = get_task_manager()
|
||||
progress_manager = get_progress_manager()
|
||||
|
||||
model_configs = {
|
||||
"qwen-tts-1.7B": {
|
||||
"model_size": "1.7B",
|
||||
"load_func": lambda: tts.get_tts_model().load_model("1.7B"),
|
||||
},
|
||||
"qwen-tts-0.6B": {
|
||||
"model_size": "0.6B",
|
||||
"load_func": lambda: tts.get_tts_model().load_model("0.6B"),
|
||||
},
|
||||
"luxtts": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
|
||||
},
|
||||
"chatterbox-tts": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(),
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox_turbo").load_model(),
|
||||
},
|
||||
"whisper-base": {
|
||||
"model_size": "base",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
|
||||
},
|
||||
"whisper-small": {
|
||||
"model_size": "small",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("small"),
|
||||
},
|
||||
"whisper-medium": {
|
||||
"model_size": "medium",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("medium"),
|
||||
},
|
||||
"whisper-large": {
|
||||
"model_size": "large",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("large"),
|
||||
},
|
||||
"whisper-turbo": {
|
||||
"model_size": "turbo",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("turbo"),
|
||||
},
|
||||
}
|
||||
|
||||
if request.model_name not in model_configs:
|
||||
|
||||
config = get_model_config(request.model_name)
|
||||
if not config:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {request.model_name}")
|
||||
|
||||
config = model_configs[request.model_name]
|
||||
|
||||
load_func = get_model_load_func(config)
|
||||
|
||||
async def download_in_background():
|
||||
"""Download model in background without blocking the HTTP request."""
|
||||
try:
|
||||
# Call the load function (which may be async)
|
||||
result = config["load_func"]()
|
||||
result = load_func()
|
||||
# If it's a coroutine, await it
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
@@ -2767,94 +2523,17 @@ async def delete_model(model_name: str):
|
||||
import os
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
# Map model names to HuggingFace repo IDs
|
||||
model_configs = {
|
||||
"qwen-tts-1.7B": {
|
||||
"hf_repo_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
||||
"model_size": "1.7B",
|
||||
"model_type": "tts",
|
||||
},
|
||||
"qwen-tts-0.6B": {
|
||||
"hf_repo_id": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
|
||||
"model_size": "0.6B",
|
||||
"model_type": "tts",
|
||||
},
|
||||
"luxtts": {
|
||||
"hf_repo_id": "YatharthS/LuxTTS",
|
||||
"model_size": "default",
|
||||
"model_type": "luxtts",
|
||||
},
|
||||
"chatterbox-tts": {
|
||||
"hf_repo_id": "ResembleAI/chatterbox",
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox",
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox_turbo",
|
||||
},
|
||||
"whisper-base": {
|
||||
"hf_repo_id": "openai/whisper-base",
|
||||
"model_size": "base",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
"whisper-small": {
|
||||
"hf_repo_id": "openai/whisper-small",
|
||||
"model_size": "small",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
"whisper-medium": {
|
||||
"hf_repo_id": "openai/whisper-medium",
|
||||
"model_size": "medium",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
"whisper-large": {
|
||||
"hf_repo_id": "openai/whisper-large-v3",
|
||||
"model_size": "large",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
"whisper-turbo": {
|
||||
"hf_repo_id": "openai/whisper-large-v3-turbo",
|
||||
"model_size": "turbo",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
}
|
||||
from .backends import get_model_config, unload_model_by_config
|
||||
|
||||
if model_name not in model_configs:
|
||||
config = get_model_config(model_name)
|
||||
if not config:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model_name}")
|
||||
|
||||
config = model_configs[model_name]
|
||||
hf_repo_id = config["hf_repo_id"]
|
||||
|
||||
|
||||
hf_repo_id = config.hf_repo_id
|
||||
|
||||
try:
|
||||
# Check if model is loaded and unload it first
|
||||
if config["model_type"] == "tts":
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
if tts_model.is_loaded() and loaded_size == config["model_size"]:
|
||||
tts.unload_tts_model()
|
||||
elif config["model_type"] == "luxtts":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
luxtts = get_tts_backend_for_engine("luxtts")
|
||||
if luxtts.is_loaded():
|
||||
luxtts.unload_model()
|
||||
elif config["model_type"] == "chatterbox":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
chatterbox = get_tts_backend_for_engine("chatterbox")
|
||||
if chatterbox.is_loaded():
|
||||
chatterbox.unload_model()
|
||||
elif config["model_type"] == "chatterbox_turbo":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
turbo = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
if turbo.is_loaded():
|
||||
turbo.unload_model()
|
||||
elif config["model_type"] == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:
|
||||
transcribe.unload_whisper_model()
|
||||
# Unload model if currently loaded
|
||||
unload_model_by_config(config)
|
||||
|
||||
# Find and delete the cache directory (using HuggingFace's OS-specific cache location)
|
||||
cache_dir = hf_constants.HF_HUB_CACHE
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
# Adding a TTS Engine to Voicebox
|
||||
|
||||
Guide for adding new TTS model backends. Based on the implementation of LuxTTS (#254), Chatterbox Multilingual (#257), Chatterbox Turbo (#258), and the PyInstaller fixes in v0.2.3.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Adding an engine touches ~12 files across 4 layers (down from ~19 after the model config registry refactor). The backend protocol work is straightforward — the real time sink is dependency hell, upstream library bugs, and PyInstaller bundling.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Backend Implementation
|
||||
|
||||
### 1.1 Create the backend file
|
||||
|
||||
`backend/backends/<engine>_backend.py` (~200-300 lines)
|
||||
|
||||
Implement the `TTSBackend` protocol from `backend/backends/__init__.py`:
|
||||
|
||||
```python
|
||||
class YourBackend:
|
||||
"""Must satisfy the TTSBackend protocol."""
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None: ...
|
||||
async def create_voice_prompt(self, audio_path: str, reference_text: str, use_cache: bool = True) -> tuple[dict, bool]: ...
|
||||
async def combine_voice_prompts(self, audio_paths: list[str], ref_texts: list[str]) -> tuple[np.ndarray, str]: ...
|
||||
async def generate(self, text: str, voice_prompt: dict, language: str = "en", seed: int | None = None, instruct: str | None = None) -> tuple[np.ndarray, int]: ...
|
||||
def unload_model(self) -> None: ...
|
||||
def is_loaded(self) -> bool: ...
|
||||
def _get_model_path(self, model_size: str) -> str: ...
|
||||
```
|
||||
|
||||
Key decisions per engine:
|
||||
|
||||
| Decision | Options | Examples |
|
||||
|----------|---------|---------|
|
||||
| **Voice prompt storage** | Pre-computed tensors vs deferred file paths | Qwen PyTorch stores tensor dicts; Chatterbox stores `{"ref_audio": path, "ref_text": text}` |
|
||||
| **Caching** | Use voice prompt cache or skip it | LuxTTS caches with `luxtts_` prefix; Chatterbox skips caching entirely |
|
||||
| **Device selection** | CUDA / MPS / CPU | Chatterbox forces CPU on macOS (MPS tensor bugs); LuxTTS supports MPS |
|
||||
| **Model download** | Library handles it vs manual `snapshot_download` | Turbo uses manual download to bypass upstream `token=True` bug |
|
||||
| **Sample rate** | Engine-specific | LuxTTS outputs 48kHz, everything else is 24kHz |
|
||||
|
||||
### 1.2 Voice prompt patterns
|
||||
|
||||
There are three patterns in use. Pick the one that fits your model:
|
||||
|
||||
**Pattern A: Pre-computed tensors** (Qwen PyTorch, LuxTTS)
|
||||
```python
|
||||
# create_voice_prompt returns opaque dict of tensors
|
||||
# Cached via torch.save(), reused across generations
|
||||
encoded = model.encode_prompt(audio_path)
|
||||
return encoded, False # (prompt_dict, was_cached)
|
||||
```
|
||||
|
||||
**Pattern B: Deferred file paths** (Chatterbox, MLX)
|
||||
```python
|
||||
# Just store paths, process at generation time
|
||||
return {"ref_audio": audio_path, "ref_text": reference_text}, False
|
||||
```
|
||||
|
||||
**Pattern C: Hybrid** (possible for new engines)
|
||||
```python
|
||||
# Pre-compute speaker embeddings, store alongside paths
|
||||
embedding = model.extract_speaker(audio_path)
|
||||
return {"embedding": embedding, "ref_audio": audio_path}, False
|
||||
```
|
||||
|
||||
If caching, prefix your cache keys to avoid collisions with other engines using the same reference audio:
|
||||
```python
|
||||
cache_key = "yourengine_" + get_cache_key(audio_path, reference_text)
|
||||
```
|
||||
|
||||
### 1.3 Register the engine
|
||||
|
||||
In `backend/backends/__init__.py`, three things:
|
||||
|
||||
**1. Add a `ModelConfig` entry** in `_get_non_qwen_tts_configs()`:
|
||||
|
||||
```python
|
||||
ModelConfig(
|
||||
model_name="your-engine",
|
||||
display_name="Your Engine",
|
||||
engine="your_engine",
|
||||
hf_repo_id="org/model-repo",
|
||||
size_mb=3200,
|
||||
needs_trim=False, # set True if output needs trim_tts_output()
|
||||
languages=["en", "fr", "de"],
|
||||
),
|
||||
```
|
||||
|
||||
This single entry replaces what used to be 6+ scattered dicts in `main.py`. The registry helpers (`get_model_config()`, `check_model_loaded()`, `engine_needs_trim()`, etc.) all derive from this config automatically.
|
||||
|
||||
**2. Add to `TTS_ENGINES` dict:**
|
||||
|
||||
```python
|
||||
TTS_ENGINES = {
|
||||
...
|
||||
"your_engine": "Your Engine",
|
||||
}
|
||||
```
|
||||
|
||||
**3. Add an elif branch in `get_tts_backend_for_engine()`:**
|
||||
|
||||
```python
|
||||
elif engine == "your_engine":
|
||||
from .your_backend import YourBackend
|
||||
backend = YourBackend()
|
||||
```
|
||||
|
||||
The import is deferred so platform-specific deps aren't loaded until the engine is first requested.
|
||||
|
||||
### 1.4 Update request models
|
||||
|
||||
In `backend/models.py`:
|
||||
|
||||
- Add engine name to `GenerationRequest.engine` regex pattern
|
||||
- Add any new language codes to the language regex on both `GenerationRequest` and `VoiceProfileCreate`
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: API Integration (`main.py`)
|
||||
|
||||
With the model config registry, `main.py` has **zero per-engine dispatch points**. All endpoints use registry helpers like `get_model_config()`, `load_engine_model()`, `engine_needs_trim()`, `check_model_loaded()`, etc.
|
||||
|
||||
**You don't need to touch `main.py` at all** unless your engine needs custom behavior in the generate endpoint (e.g. a new post-processing step beyond `trim_tts_output`).
|
||||
|
||||
### 2.1 What the registry handles automatically
|
||||
|
||||
| Endpoint | Registry function used |
|
||||
|----------|----------------------|
|
||||
| `POST /generate` | `load_engine_model(engine, size)` + `engine_needs_trim(engine)` |
|
||||
| `POST /generate/stream` | `ensure_model_cached_or_raise(engine, size)` + `load_engine_model()` |
|
||||
| `GET /models/status` | `get_all_model_configs()` + `check_model_loaded(config)` |
|
||||
| `POST /models/download` | `get_model_config(name)` + `get_model_load_func(config)` |
|
||||
| `POST /models/{name}/unload` | `get_model_config(name)` + `unload_model_by_config(config)` |
|
||||
| `DELETE /models/{name}` | `get_model_config(name)` + `unload_model_by_config(config)` |
|
||||
|
||||
### 2.2 Post-processing
|
||||
|
||||
If your model produces trailing silence or hallucinated audio, set `needs_trim=True` on your `ModelConfig`. The generate endpoint checks `engine_needs_trim(engine)` and applies `trim_tts_output()` automatically.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Frontend Integration
|
||||
|
||||
### 3.1 TypeScript types
|
||||
|
||||
In `app/src/lib/api/types.ts`:
|
||||
- Add to the `engine` union type on `GenerationRequest`
|
||||
|
||||
### 3.2 Language maps
|
||||
|
||||
In `app/src/lib/constants/languages.ts`:
|
||||
- Add entry to `ENGINE_LANGUAGES` record
|
||||
- Add any new language codes to `ALL_LANGUAGES` if needed
|
||||
|
||||
### 3.3 Engine/model selector (shared component)
|
||||
|
||||
The model selector is a shared component — update one file:
|
||||
|
||||
- `app/src/components/Generation/EngineModelSelector.tsx`
|
||||
|
||||
Add an entry to `ENGINE_OPTIONS` and `ENGINE_DESCRIPTIONS`. If the engine is English-only, add it to `ENGLISH_ONLY_ENGINES`. The `handleEngineChange()` function handles language validation automatically (resets to first available language if the current one isn't supported).
|
||||
|
||||
Both `GenerationForm.tsx` and `FloatingGenerateBox.tsx` use `<EngineModelSelector>` — no changes needed in either.
|
||||
|
||||
Handle engine-specific UI conditionals in the form components if needed:
|
||||
- Hide instruct field for engines that don't support it
|
||||
- Show engine-specific controls (e.g. `ParalinguisticInput` for Turbo)
|
||||
|
||||
### 3.4 Form hook
|
||||
|
||||
In `app/src/lib/hooks/useGenerationForm.ts`:
|
||||
- Add to Zod schema enum for `engine`
|
||||
- Add engine-to-model-name mapping (e.g. `"your_engine"` → `"your-engine"`)
|
||||
- Update payload construction to conditionally include engine-specific fields
|
||||
|
||||
### 3.5 Model management
|
||||
|
||||
In `app/src/components/ServerSettings/ModelManagement.tsx`:
|
||||
- Add description to `MODEL_DESCRIPTIONS` record
|
||||
- The model list auto-renders from `/models/status` data
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Dependencies
|
||||
|
||||
### 4.1 Python dependencies
|
||||
|
||||
Add to `backend/requirements.txt`. Watch for:
|
||||
|
||||
**Pinned dependency conflicts** — If the model package pins old versions of numpy, torch, or transformers, install with `--no-deps` and list sub-dependencies manually. This is what Chatterbox requires:
|
||||
```
|
||||
# In justfile/Makefile (NOT requirements.txt):
|
||||
pip install --no-deps chatterbox-tts
|
||||
|
||||
# In requirements.txt — list the transitive deps:
|
||||
conformer
|
||||
diffusers
|
||||
omegaconf
|
||||
# ... etc
|
||||
```
|
||||
|
||||
**Non-PyPI packages** — Some deps only exist as git repos:
|
||||
```
|
||||
linacodec @ git+https://github.com/user/repo.git
|
||||
Zipvoice @ git+https://github.com/user/repo.git
|
||||
```
|
||||
|
||||
**Custom package indexes** — Some packages need `--find-links`:
|
||||
```
|
||||
--find-links https://k2-fsa.github.io/icefall/piper_phonemize.html
|
||||
```
|
||||
|
||||
### 4.2 Identifying hidden sub-dependencies
|
||||
|
||||
When using `--no-deps`, you need to manually figure out what the package actually imports. There's no shortcut:
|
||||
|
||||
1. Install the package normally in a throwaway venv
|
||||
2. Run `pip show <package>` to get its `Requires:` list
|
||||
3. Cross-reference against what's already in our requirements.txt
|
||||
4. Test that the engine loads and generates without import errors
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: PyInstaller Bundling
|
||||
|
||||
This is where most of the pain lives. If your model's Python package or its dependencies use any of the following at runtime, PyInstaller won't bundle them automatically:
|
||||
|
||||
### 5.1 Common PyInstaller issues
|
||||
|
||||
| Issue | Symptom | Fix |
|
||||
|-------|---------|-----|
|
||||
| **`inspect.getsource()` at import time** | "could not get source code" | `--collect-all <package>` (bundles `.py` source files, not just bytecode) |
|
||||
| **Data files (yaml, .pth.tar, lang dicts)** | FileNotFoundError at runtime | `--collect-all <package>` or `--collect-data <package>` |
|
||||
| **Native data paths (espeak-ng, etc.)** | Library looks at `/usr/share/...` | Set env var in frozen builds: `os.environ["ESPEAK_DATA_PATH"] = bundled_path` |
|
||||
| **`importlib.metadata` lookups** | "No package metadata found" | `--copy-metadata <package>` |
|
||||
| **Dynamic imports** | ModuleNotFoundError | `--hidden-import <module>` |
|
||||
| **`typeguard` / `@typechecked`** | Calls `inspect.getsource()` on decorated functions | `--collect-all` for the decorated package |
|
||||
|
||||
### 5.2 Testing frozen builds
|
||||
|
||||
You can't skip this. Models that work in `python -m uvicorn` will break in the PyInstaller binary. The flow:
|
||||
|
||||
1. Build the binary: `just build` or the PyInstaller spec
|
||||
2. Run it and try to download + load + generate with the new engine
|
||||
3. Check stderr for the actual error (macOS/Linux: stdout/stderr go to Tauri sidecar logs)
|
||||
4. Fix, rebuild, repeat
|
||||
|
||||
### 5.3 Real examples from v0.2.3
|
||||
|
||||
These were all models that worked perfectly in dev:
|
||||
|
||||
- **LuxTTS**: `typeguard`'s `@typechecked` calls `inspect.getsource()` at import → needed `--collect-all inflect`. `piper_phonemize` bundles `espeak-ng-data/` → needed `--collect-all piper_phonemize` + `ESPEAK_DATA_PATH` env var
|
||||
- **Chatterbox**: `resemble-perth` bundles a pretrained watermark model (`.pth.tar`, `hparams.yaml`) → needed `--collect-all perth`
|
||||
- **Both**: `huggingface_hub` silently disables tqdm based on logger level → progress bars showed 0% in frozen builds until we force-enabled the internal counter
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Common Upstream Workarounds
|
||||
|
||||
Almost every model library has bugs you'll need to work around. Here's the catalog:
|
||||
|
||||
### 6.1 torch.load device mismatch
|
||||
|
||||
If model weights were saved on CUDA but you're loading on CPU/MPS:
|
||||
```python
|
||||
_original_torch_load = torch.load
|
||||
def _patched_torch_load(*args, **kwargs):
|
||||
kwargs.setdefault("map_location", "cpu")
|
||||
return _original_torch_load(*args, **kwargs)
|
||||
torch.load = _patched_torch_load
|
||||
```
|
||||
Used by both Chatterbox backends. Use a threading lock if patching globally.
|
||||
|
||||
### 6.2 Float64/Float32 dtype mismatch
|
||||
|
||||
`librosa` returns float64, model weights are float32. Patch the offending methods:
|
||||
```python
|
||||
original_fn = SomeClass.some_method
|
||||
def patched_fn(self, *args, **kwargs):
|
||||
result = original_fn(self, *args, **kwargs)
|
||||
return result.float() # float64 → float32
|
||||
SomeClass.some_method = patched_fn
|
||||
```
|
||||
Used by Chatterbox for `S3Tokenizer.log_mel_spectrogram` and `VoiceEncoder.forward`.
|
||||
|
||||
### 6.3 Transformers attention implementation
|
||||
|
||||
If the model uses `output_attentions=True` with transformers >= 4.36:
|
||||
```python
|
||||
for module in model.modules():
|
||||
if hasattr(module, '_attn_implementation'):
|
||||
module._attn_implementation = "eager"
|
||||
```
|
||||
SDPA (the new default) doesn't support `output_attentions`. Force eager attention.
|
||||
|
||||
### 6.4 HuggingFace token bug
|
||||
|
||||
Some models' `from_pretrained()` passes `token=True` which requires a stored HF token even for public repos:
|
||||
```python
|
||||
from huggingface_hub import snapshot_download
|
||||
local_path = snapshot_download(repo_id=REPO, token=None)
|
||||
model = ModelClass.from_local(local_path, device=device)
|
||||
```
|
||||
Used by Chatterbox Turbo.
|
||||
|
||||
### 6.5 MPS tensor issues
|
||||
|
||||
MPS (Apple Silicon GPU) has incomplete operator coverage. If generation crashes on MPS:
|
||||
```python
|
||||
def _get_device(self):
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
return "cpu" # Skip MPS entirely
|
||||
```
|
||||
Used by both Chatterbox backends. LuxTTS works fine on MPS.
|
||||
|
||||
### 6.6 HuggingFace progress tracking
|
||||
|
||||
To get download progress bars in the UI, wrap model loading with `HFProgressTracker`:
|
||||
```python
|
||||
from backend.utils.hf_progress import HFProgressTracker
|
||||
tracker = HFProgressTracker(model_name, progress_manager)
|
||||
with tracker.patch_download():
|
||||
model = ModelClass.from_pretrained(repo_id)
|
||||
```
|
||||
The tracker monkey-patches tqdm to intercept HuggingFace's internal progress bars. Must be set up BEFORE importing the model library if it imports HF at module level.
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### Backend
|
||||
- [ ] `backend/backends/<engine>_backend.py` — implements TTSBackend protocol
|
||||
- [ ] `backend/backends/__init__.py` — `ModelConfig` entry + `TTS_ENGINES` + `get_tts_backend_for_engine()` elif
|
||||
- [ ] `backend/models.py` — engine name in regex, any new language codes
|
||||
- [ ] `backend/requirements.txt` — dependencies added (check for `--no-deps` needs)
|
||||
- [ ] `justfile` / `Makefile` — `--no-deps` install step if needed
|
||||
|
||||
### API (`backend/main.py`)
|
||||
No changes needed — the model config registry handles all dispatch automatically.
|
||||
|
||||
### Frontend
|
||||
- [ ] `app/src/lib/api/types.ts` — engine union type
|
||||
- [ ] `app/src/lib/constants/languages.ts` — `ENGINE_LANGUAGES` entry
|
||||
- [ ] `app/src/components/Generation/EngineModelSelector.tsx` — `ENGINE_OPTIONS` + `ENGINE_DESCRIPTIONS` + `ENGLISH_ONLY_ENGINES`
|
||||
- [ ] `app/src/lib/hooks/useGenerationForm.ts` — Zod schema + model mapping
|
||||
- [ ] `app/src/components/ServerSettings/ModelManagement.tsx` — model description
|
||||
|
||||
### Production
|
||||
- [ ] PyInstaller spec — `--collect-all`, `--hidden-import`, `--copy-metadata` as needed
|
||||
- [ ] Test in frozen binary — download, load, generate all work
|
||||
- [ ] Download progress — `HFProgressTracker` wired up, progress shows in UI
|
||||
|
||||
### Upstream workarounds (check which apply)
|
||||
- [ ] torch.load device mapping (CUDA weights on CPU)
|
||||
- [ ] Float64→Float32 patches (librosa interaction)
|
||||
- [ ] Eager attention forcing (transformers >= 4.36)
|
||||
- [ ] HF token bypass (snapshot_download + from_local)
|
||||
- [ ] MPS skip (if operators not supported)
|
||||
- [ ] espeak-ng / native data path env vars
|
||||
@@ -50,9 +50,10 @@
|
||||
|
||||
| Layer | File | Purpose |
|
||||
|-------|------|---------|
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2100 lines) |
|
||||
| TTS protocol | `backend/backends/__init__.py:14-81` | `TTSBackend` Protocol definition |
|
||||
| TTS factory | `backend/backends/__init__.py:138-178` | Thread-safe engine registry (double-checked locking) |
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2850 lines) |
|
||||
| TTS protocol | `backend/backends/__init__.py:32-101` | `TTSBackend` Protocol definition |
|
||||
| Model registry | `backend/backends/__init__.py:17-29,153-366` | `ModelConfig` dataclass + registry helpers |
|
||||
| TTS factory | `backend/backends/__init__.py:382-426` | Thread-safe engine registry (double-checked locking) |
|
||||
| PyTorch TTS | `backend/backends/pytorch_backend.py` | Qwen3-TTS via `qwen_tts` package |
|
||||
| MLX TTS | `backend/backends/mlx_backend.py` | Qwen3-TTS via `mlx_audio.tts` |
|
||||
| LuxTTS | `backend/backends/luxtts_backend.py` | LuxTTS — fast, CPU-friendly |
|
||||
@@ -64,6 +65,7 @@
|
||||
| Audio utils | `backend/utils/audio.py` | `trim_tts_output()`, normalize, load/save audio |
|
||||
| Frontend API | `app/src/lib/api/client.ts` | Hand-written fetch wrapper |
|
||||
| Frontend types | `app/src/lib/api/types.ts` | TypeScript API types |
|
||||
| Engine selector | `app/src/components/Generation/EngineModelSelector.tsx` | Shared engine/model dropdown |
|
||||
| Generation form | `app/src/components/Generation/GenerationForm.tsx` | TTS generation UI |
|
||||
| Floating gen box | `app/src/components/Generation/FloatingGenerateBox.tsx` | Compact generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status/progress UI |
|
||||
@@ -101,8 +103,10 @@ POST /generate
|
||||
- Multi-engine TTS architecture with thread-safe backend registry (PR #254)
|
||||
- LuxTTS integration — fast, CPU-friendly English TTS (PR #254)
|
||||
- Chatterbox Multilingual TTS — 23 languages including Hebrew (PR #257)
|
||||
- Delivery instructions (instruct parameter, Qwen only)
|
||||
- Instruct parameter UI exists but is non-functional across all backends (see #224, Known Limitations)
|
||||
- Single flat model dropdown (Qwen 1.7B, Qwen 0.6B, LuxTTS, Chatterbox, Chatterbox Turbo)
|
||||
- Centralized model config registry (`ModelConfig` dataclass) — no per-engine dispatch maps in `main.py`
|
||||
- Shared `EngineModelSelector` component — engine/model dropdown defined once, used in both generation forms
|
||||
|
||||
**Infrastructure:**
|
||||
- CUDA backend swap via binary download and restart (PR #252)
|
||||
@@ -125,13 +129,13 @@ POST /generate
|
||||
|
||||
### TTS Engine Comparison
|
||||
|
||||
| Engine | Model Name | Languages | Size | Key Features |
|
||||
|--------|-----------|-----------|------|-------------|
|
||||
| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Instruct mode, highest quality |
|
||||
| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster |
|
||||
| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast |
|
||||
| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual |
|
||||
| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency |
|
||||
| Engine | Model Name | Languages | Size | Key Features | Instruct Support |
|
||||
|--------|-----------|-----------|------|-------------|-----------------|
|
||||
| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Highest quality, voice cloning | None (Base model has no instruct path) |
|
||||
| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster | None |
|
||||
| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast | None |
|
||||
| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual | Partial — `exaggeration` float (0-1) for expressiveness |
|
||||
| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency | Partial — inline tags only, no separate instruct param |
|
||||
|
||||
### Multi-Engine Architecture (Shipped)
|
||||
|
||||
@@ -149,6 +153,7 @@ The singleton TTS backend blocker described in the previous version of this doc
|
||||
- **HF XET progress**: Large files downloaded via `hf-xet` (HuggingFace's new transfer backend) report `n=0` in tqdm updates. Progress bars may appear stuck for large `.safetensors` files even though the download is proceeding. This is a known upstream limitation.
|
||||
- **Chatterbox Turbo upstream token bug**: `from_pretrained()` passes `token=os.getenv("HF_TOKEN") or True` which fails without a stored HF token. Our backend works around this by calling `snapshot_download(token=None)` + `from_local()`.
|
||||
- **chatterbox-tts must install with `--no-deps`**: It pins `numpy<1.26`, `torch==2.6.0`, `transformers==4.46.3` — all incompatible with our stack (Python 3.12, torch 2.10, transformers 4.57.3). Sub-deps listed explicitly in `requirements.txt`.
|
||||
- **Instruct parameter is non-functional** (#224): The UI exposes an instruct text field, but it's silently dropped by every backend. The Qwen3-TTS Base model we ship only supports voice cloning — instruct requires the separate CustomVoice model variant (`Qwen3-TTS-12Hz-1.7B-CustomVoice`), which uses predefined speakers instead of ref audio. The instruct UI should be hidden until a backend with real support is integrated.
|
||||
- **Streaming generation** only works for Qwen on MLX. Other engines use the non-streaming `/generate` endpoint.
|
||||
- **dicta-onnx** (Hebrew diacritization) not included — upstream Chatterbox bug requires `model_path` arg but calls `Dicta()` with none. Hebrew works fine without it.
|
||||
|
||||
@@ -323,41 +328,43 @@ Notable requests:
|
||||
|
||||
### Models Worth Supporting (2026 SOTA — updated March 13)
|
||||
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Status |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|--------|
|
||||
| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | **Shipped** | v0.1.13 |
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | **Shipped** | PR #254 |
|
||||
| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | **Shipped** | PR #257 |
|
||||
| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | **PR #258** | In review |
|
||||
| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Needs vetting | MIT, 700s+ coherent, synced transcript output |
|
||||
| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | Needs vetting | Apache 2.0, multi-speaker dialogue, text-to-voice design (no ref audio) |
|
||||
| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion, LoRA-friendly |
|
||||
| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | Needs vetting | MIT, Kyutai Labs, no GPU required |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Ready | Apache 2.0, multi-engine arch in place |
|
||||
| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Ready | Multi-engine arch in place |
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Instruct Support | Integration Ease | Status |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|-----------------|--------|
|
||||
| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | None (Base); Yes (CustomVoice variant, predefined speakers only) | **Shipped** | v0.1.13 |
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | None | **Shipped** | PR #254 |
|
||||
| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | Partial — `exaggeration` float | **Shipped** | PR #257 |
|
||||
| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | Partial — inline tags only | **PR #258** | In review |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** — `inference_instruct2()`, works with cloning | Ready | Best instruct candidate |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — inline text descriptions, word-level control | Ready | Multi-engine arch in place |
|
||||
| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | **Yes** — text prompts for style + timbre design | Needs vetting | Apache 2.0, multi-speaker dialogue |
|
||||
| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Partial — automatic prosody from text context | Needs vetting | MIT, 700s+ coherent, synced transcript output |
|
||||
| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Partial — automatic context-aware prosody | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Partial — automatic style inference | Ready | Apache 2.0, multi-engine arch in place |
|
||||
| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Partial — style transfer from ref audio only | Ready | Multi-engine arch in place |
|
||||
| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | None | Needs vetting | MIT, Kyutai Labs, no GPU required |
|
||||
|
||||
#### Notes on New Candidates (March 2026)
|
||||
|
||||
- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada)
|
||||
- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions, no ref audio). Unique UX for Stories voice design. [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS)
|
||||
- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Context-aware prosody/emotion, real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM)
|
||||
- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts)
|
||||
- **CosyVoice2-0.5B** — Best candidate for instruct support. `inference_instruct2()` accepts a text instruct parameter for emotions, speed, volume, dialects — and it works alongside voice cloning. This is the closest match to what users expect from our instruct UI. [HF: FunAudioLLM/CosyVoice2-0.5B](https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B)
|
||||
- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. Prosody/emotion is automatic from text context, not user-controllable. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada)
|
||||
- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions). VoiceGenerator unifies timbre design and style control via text prompts, usable as a layer for downstream TTS including cloning. [HF: OpenMOSS-Team/MOSS-VoiceGenerator](https://huggingface.co/OpenMOSS-Team/MOSS-VoiceGenerator) | [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS)
|
||||
- **Fish Speech** — Word-level fine-grained control using plain language descriptions inline in the script. Works with cloning. Note: Fish Audio S2 has a restrictive research license (commercial use requires approval), but the open-source Fish Speech model may differ. Needs license clarification. [fish.audio blog](https://fish.audio/blog/fish-audio-s2-fine-grained-ai-voice-control-at-the-word-level)
|
||||
- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Prosody/emotion is context-aware but automatic, not explicitly controllable via text prompt. Real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM)
|
||||
- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. No style control. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts)
|
||||
- **Watch list:** MioTTS-2.6B (fast LLM-based EN/JP, vLLM compatible), Oolel-Voices (Soynade Research, expressive modular control)
|
||||
- **Skipped:** Fish Audio S2 — restrictive research license (commercial use requires approval), despite strong features
|
||||
|
||||
### Adding a New Engine (Now Straightforward)
|
||||
|
||||
With the multi-engine architecture shipped, adding a new TTS engine requires:
|
||||
With the model config registry and shared `EngineModelSelector` component, adding a new TTS engine requires:
|
||||
|
||||
1. **Create `backend/backends/<engine>_backend.py`** — implement `TTSBackend` protocol (~200-300 lines)
|
||||
2. **Register in `backend/backends/__init__.py`** — add to `TTS_ENGINES` dict + factory function
|
||||
2. **Register in `backend/backends/__init__.py`** — add `ModelConfig` entry + `TTS_ENGINES` entry + factory elif
|
||||
3. **Update `backend/models.py`** — add engine name to regex
|
||||
4. **Update `backend/main.py`** — add engine cases in generate, stream, model-status, download, delete (5 dispatch points)
|
||||
5. **Update frontend** — add to engine union type, form schema, model dropdown, language map (5-6 files)
|
||||
4. **Update frontend** — add to engine union type, `EngineModelSelector` options, form schema, language map (4 files)
|
||||
|
||||
Total effort: **~1 day** for a well-documented model with a PyPI package.
|
||||
`main.py` requires **zero changes** — the registry handles all dispatch automatically.
|
||||
|
||||
Total effort: **~1 day** for a well-documented model with a PyPI package. See `docs/plans/ADDING_TTS_ENGINES.md` for the full guide.
|
||||
|
||||
---
|
||||
|
||||
@@ -367,13 +374,13 @@ Total effort: **~1 day** for a well-documented model with a PyPI package.
|
||||
|
||||
The singleton TTS backend was replaced with a thread-safe per-engine registry in PR #254. Multiple engines can now be loaded simultaneously.
|
||||
|
||||
### 2. `main.py` is 2100+ Lines
|
||||
### ~~2. `main.py` Dispatch Point Duplication~~ — RESOLVED
|
||||
|
||||
All API routes, all model configs, all business logic in one file. Five separate dispatch points for each engine. Any new engine touches this file in 5 places. A model config registry pattern would reduce duplication.
|
||||
Previously, each engine required updates to 6+ hardcoded dispatch maps across `main.py` (~320 lines of if/elif chains). A model config registry in `backend/backends/__init__.py` now centralizes all model metadata (`ModelConfig` dataclass) with helper functions (`load_engine_model()`, `check_model_loaded()`, `engine_needs_trim()`, etc.). Adding a new engine requires zero changes to `main.py`.
|
||||
|
||||
### 3. Model Config is Scattered (Improved)
|
||||
### ~~3. Model Config is Scattered~~ — RESOLVED
|
||||
|
||||
Model identifiers are still duplicated across `main.py` (3 dicts), backend files, frontend components, and the languages constant. However, the pattern is now consistent and well-understood. A centralized model registry would help but isn't blocking.
|
||||
Model identifiers, HF repo IDs, display names, and engine metadata are now consolidated in the `ModelConfig` registry. Backend-aware branching (e.g. MLX vs PyTorch Qwen repo IDs) happens inside the registry. Frontend model options are centralized in `EngineModelSelector.tsx`.
|
||||
|
||||
### 4. Voice Prompt Cache Assumes PyTorch Tensors
|
||||
|
||||
@@ -410,7 +417,7 @@ The generation form now uses a flat model dropdown with engine-based routing. Pe
|
||||
| 1 | **#253** — 48kHz speech tokenizer | Quality improvement | Medium |
|
||||
| 2 | **#161** — Docker deployment | Server/headless users | Medium |
|
||||
| 3 | **#154** — Audiobook tab | Long-form users | Medium |
|
||||
| 4 | **Model config registry** | Reduce 5-dispatch-point duplication in main.py | Medium |
|
||||
| 4 | ~~**Model config registry**~~ | ~~Reduce dispatch duplication in main.py~~ | **Done** |
|
||||
| 5 | **#225** — Custom HuggingFace models | User-supplied models | High (needs rework for multi-engine) |
|
||||
|
||||
### Tier 3 — Future (v0.3.0+)
|
||||
@@ -421,7 +428,7 @@ The generation form now uses a flat model dropdown with engine-based routing. Pe
|
||||
| 2 | **Pocket TTS** (Kyutai) | CPU-first 100M model, broadens hardware support. Kyutai ships clean code. Needs API vetting. |
|
||||
| 3 | **MOSS-TTS** | Text-to-voice design (no ref audio) is unique. Multi-speaker dialogue for Stories. Needs thorough API vetting. |
|
||||
| 4 | **Kokoro-82M** | 82M params, CPU realtime, Apache 2.0. Easy win. |
|
||||
| 5 | **Model config registry refactor** | Reduce 5-dispatch-point duplication in main.py — do before adding 3+ more engines |
|
||||
| 5 | ~~**Model config registry refactor**~~ | **Done** — consolidated in `backend/backends/__init__.py` + `EngineModelSelector.tsx` |
|
||||
| 6 | XTTS-v2 / Fish Speech / CosyVoice | Multi-engine arch is ready; just needs backend implementation |
|
||||
| 7 | **VoxCPM 1.5** | Tokenizer-free streaming, interesting but uncertain integration surface |
|
||||
| 8 | OpenAI-compatible API (plan doc exists) | Low effort once API is stable |
|
||||
|
||||
Reference in New Issue
Block a user