mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 04:40:40 -07:00
enable Edit on GitHub and last updated on all doc pages
This commit is contained in:
James Pine
@@ -1,15 +1,10 @@
|
||||
import { getPageImage, source } from '@/lib/source';
|
||||
import {
|
||||
DocsBody,
|
||||
DocsDescription,
|
||||
DocsPage,
|
||||
DocsTitle,
|
||||
} from 'fumadocs-ui/page';
|
||||
import { notFound } from 'next/navigation';
|
||||
import { getMDXComponents } from '@/mdx-components';
|
||||
import type { Metadata } from 'next';
|
||||
import { createRelativeLink } from 'fumadocs-ui/mdx';
|
||||
import { DocsBody, DocsDescription, DocsPage, DocsTitle } from 'fumadocs-ui/page';
|
||||
import type { Metadata } from 'next';
|
||||
import { notFound } from 'next/navigation';
|
||||
import { APIPage } from '@/components/api-page';
|
||||
import { getPageImage, source } from '@/lib/source';
|
||||
import { getMDXComponents } from '@/mdx-components';
|
||||
|
||||
export default async function Page(props: PageProps<'/docs/[[...slug]]'>) {
|
||||
const params = await props.params;
|
||||
@@ -19,7 +14,17 @@ export default async function Page(props: PageProps<'/docs/[[...slug]]'>) {
|
||||
const MDX = page.data.body;
|
||||
|
||||
return (
|
||||
<DocsPage toc={page.data.toc} full={page.data.full}>
|
||||
<DocsPage
|
||||
toc={page.data.toc}
|
||||
full={page.data.full}
|
||||
editOnGithub={{
|
||||
owner: 'jamiepine',
|
||||
repo: 'voicebox',
|
||||
sha: 'main',
|
||||
path: `docs/content/docs/${page.path}`,
|
||||
}}
|
||||
lastUpdate={page.data.lastModified}
|
||||
>
|
||||
<DocsTitle>{page.data.title}</DocsTitle>
|
||||
<DocsDescription>{page.data.description}</DocsDescription>
|
||||
<DocsBody>
|
||||
@@ -38,9 +43,7 @@ export async function generateStaticParams() {
|
||||
return source.generateParams();
|
||||
}
|
||||
|
||||
export async function generateMetadata(
|
||||
props: PageProps<'/docs/[[...slug]]'>,
|
||||
): Promise<Metadata> {
|
||||
export async function generateMetadata(props: PageProps<'/docs/[[...slug]]'>): Promise<Metadata> {
|
||||
const params = await props.params;
|
||||
const page = source.getPage(params.slug);
|
||||
if (!page) notFound();
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
# Adding a TTS Engine to Voicebox
|
||||
|
||||
Guide for adding new TTS model backends. Based on the implementation of LuxTTS (#254), Chatterbox Multilingual (#257), Chatterbox Turbo (#258), and the PyInstaller fixes in v0.2.3.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Adding an engine touches ~10 files across 4 layers. The backend protocol work is straightforward — the real time sink is dependency hell, upstream library bugs, and PyInstaller bundling.
|
||||
|
||||
The backend is split into layers: `routes/` (thin HTTP handlers), `services/` (business logic), `backends/` (engine implementations), and `utils/` (shared utilities). New engines only need to touch `backends/` and `models.py` on the backend side — the route and service layers use a model config registry that handles dispatch automatically.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Backend Implementation
|
||||
|
||||
### 1.1 Create the backend file
|
||||
|
||||
`backend/backends/<engine>_backend.py` (~200-300 lines)
|
||||
|
||||
Implement the `TTSBackend` protocol from `backend/backends/__init__.py`:
|
||||
|
||||
```python
|
||||
class YourBackend:
|
||||
"""Must satisfy the TTSBackend protocol."""
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None: ...
|
||||
async def create_voice_prompt(self, audio_path: str, reference_text: str, use_cache: bool = True) -> tuple[dict, bool]: ...
|
||||
async def combine_voice_prompts(self, audio_paths: list[str], ref_texts: list[str]) -> tuple[np.ndarray, str]: ...
|
||||
async def generate(self, text: str, voice_prompt: dict, language: str = "en", seed: int | None = None, instruct: str | None = None) -> tuple[np.ndarray, int]: ...
|
||||
def unload_model(self) -> None: ...
|
||||
def is_loaded(self) -> bool: ...
|
||||
def _get_model_path(self, model_size: str) -> str: ...
|
||||
```
|
||||
|
||||
Key decisions per engine:
|
||||
|
||||
| Decision | Options | Examples |
|
||||
|----------|---------|---------|
|
||||
| **Voice prompt storage** | Pre-computed tensors vs deferred file paths | Qwen PyTorch stores tensor dicts; Chatterbox stores `{"ref_audio": path, "ref_text": text}` |
|
||||
| **Caching** | Use voice prompt cache or skip it | LuxTTS caches with `luxtts_` prefix; Chatterbox skips caching entirely |
|
||||
| **Device selection** | CUDA / MPS / CPU | Chatterbox forces CPU on macOS (MPS tensor bugs); LuxTTS supports MPS |
|
||||
| **Model download** | Library handles it vs manual `snapshot_download` | Turbo uses manual download to bypass upstream `token=True` bug |
|
||||
| **Sample rate** | Engine-specific | LuxTTS outputs 48kHz, everything else is 24kHz |
|
||||
|
||||
### 1.2 Voice prompt patterns
|
||||
|
||||
There are three patterns in use. Pick the one that fits your model:
|
||||
|
||||
**Pattern A: Pre-computed tensors** (Qwen PyTorch, LuxTTS)
|
||||
```python
|
||||
# create_voice_prompt returns opaque dict of tensors
|
||||
# Cached via torch.save(), reused across generations
|
||||
encoded = model.encode_prompt(audio_path)
|
||||
return encoded, False # (prompt_dict, was_cached)
|
||||
```
|
||||
|
||||
**Pattern B: Deferred file paths** (Chatterbox, MLX)
|
||||
```python
|
||||
# Just store paths, process at generation time
|
||||
return {"ref_audio": audio_path, "ref_text": reference_text}, False
|
||||
```
|
||||
|
||||
**Pattern C: Hybrid** (possible for new engines)
|
||||
```python
|
||||
# Pre-compute speaker embeddings, store alongside paths
|
||||
embedding = model.extract_speaker(audio_path)
|
||||
return {"embedding": embedding, "ref_audio": audio_path}, False
|
||||
```
|
||||
|
||||
If caching, prefix your cache keys to avoid collisions with other engines using the same reference audio:
|
||||
```python
|
||||
cache_key = "yourengine_" + get_cache_key(audio_path, reference_text)
|
||||
```
|
||||
|
||||
### 1.3 Register the engine
|
||||
|
||||
In `backend/backends/__init__.py`, three things:
|
||||
|
||||
**1. Add a `ModelConfig` entry** in `_get_non_qwen_tts_configs()`:
|
||||
|
||||
```python
|
||||
ModelConfig(
|
||||
model_name="your-engine",
|
||||
display_name="Your Engine",
|
||||
engine="your_engine",
|
||||
hf_repo_id="org/model-repo",
|
||||
size_mb=3200,
|
||||
needs_trim=False, # set True if output needs trim_tts_output()
|
||||
languages=["en", "fr", "de"],
|
||||
),
|
||||
```
|
||||
|
||||
This single entry replaces what used to be 6+ scattered dicts in `main.py`. The registry helpers (`get_model_config()`, `check_model_loaded()`, `engine_needs_trim()`, etc.) all derive from this config automatically.
|
||||
|
||||
**2. Add to `TTS_ENGINES` dict:**
|
||||
|
||||
```python
|
||||
TTS_ENGINES = {
|
||||
...
|
||||
"your_engine": "Your Engine",
|
||||
}
|
||||
```
|
||||
|
||||
**3. Add an elif branch in `get_tts_backend_for_engine()`:**
|
||||
|
||||
```python
|
||||
elif engine == "your_engine":
|
||||
from .your_backend import YourBackend
|
||||
backend = YourBackend()
|
||||
```
|
||||
|
||||
The import is deferred so platform-specific deps aren't loaded until the engine is first requested.
|
||||
|
||||
### 1.4 Update request models
|
||||
|
||||
In `backend/models.py`:
|
||||
|
||||
- Add engine name to `GenerationRequest.engine` regex pattern
|
||||
- Add any new language codes to the language regex on both `GenerationRequest` and `VoiceProfileCreate`
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Route and Service Integration
|
||||
|
||||
With the model config registry, the route and service layers have **zero per-engine dispatch points**. All endpoints use registry helpers like `get_model_config()`, `load_engine_model()`, `engine_needs_trim()`, `check_model_loaded()`, etc.
|
||||
|
||||
**You don't need to touch any route or service files** unless your engine needs custom behavior in the generate pipeline (e.g. a new post-processing step beyond `trim_tts_output`).
|
||||
|
||||
### 2.1 What the registry handles automatically
|
||||
|
||||
| Route file | Registry function used |
|
||||
|------------|----------------------|
|
||||
| `routes/generations.py` | `load_engine_model(engine, size)` + `engine_needs_trim(engine)` |
|
||||
| `routes/models.py` | `get_all_model_configs()` + `check_model_loaded(config)` |
|
||||
| `routes/models.py` | `get_model_config(name)` + `get_model_load_func(config)` |
|
||||
| `services/generation.py` | `get_tts_backend_for_engine()` + `ensure_model_cached_or_raise()` |
|
||||
|
||||
### 2.2 Post-processing
|
||||
|
||||
If your model produces trailing silence or hallucinated audio, set `needs_trim=True` on your `ModelConfig`. The generation service checks `engine_needs_trim(engine)` and applies `trim_tts_output()` automatically.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: Frontend Integration
|
||||
|
||||
### 3.1 TypeScript types
|
||||
|
||||
In `app/src/lib/api/types.ts`:
|
||||
- Add to the `engine` union type on `GenerationRequest`
|
||||
|
||||
### 3.2 Language maps
|
||||
|
||||
In `app/src/lib/constants/languages.ts`:
|
||||
- Add entry to `ENGINE_LANGUAGES` record
|
||||
- Add any new language codes to `ALL_LANGUAGES` if needed
|
||||
|
||||
### 3.3 Engine/model selector (shared component)
|
||||
|
||||
The model selector is a shared component — update one file:
|
||||
|
||||
- `app/src/components/Generation/EngineModelSelector.tsx`
|
||||
|
||||
Add an entry to `ENGINE_OPTIONS` and `ENGINE_DESCRIPTIONS`. If the engine is English-only, add it to `ENGLISH_ONLY_ENGINES`. The `handleEngineChange()` function handles language validation automatically (resets to first available language if the current one isn't supported).
|
||||
|
||||
Both `GenerationForm.tsx` and `FloatingGenerateBox.tsx` use `<EngineModelSelector>` — no changes needed in either.
|
||||
|
||||
Handle engine-specific UI conditionals in the form components if needed:
|
||||
- Hide instruct field for engines that don't support it
|
||||
- Show engine-specific controls (e.g. `ParalinguisticInput` for Turbo)
|
||||
|
||||
### 3.4 Form hook
|
||||
|
||||
In `app/src/lib/hooks/useGenerationForm.ts`:
|
||||
- Add to Zod schema enum for `engine`
|
||||
- Add engine-to-model-name mapping (e.g. `"your_engine"` → `"your-engine"`)
|
||||
- Update payload construction to conditionally include engine-specific fields
|
||||
|
||||
### 3.5 Model management
|
||||
|
||||
In `app/src/components/ServerSettings/ModelManagement.tsx`:
|
||||
- Add description to `MODEL_DESCRIPTIONS` record
|
||||
- The model list auto-renders from `/models/status` data
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Dependencies
|
||||
|
||||
### 4.1 Python dependencies
|
||||
|
||||
Add to `backend/requirements.txt`. Watch for:
|
||||
|
||||
**Pinned dependency conflicts** — If the model package pins old versions of numpy, torch, or transformers, install with `--no-deps` and list sub-dependencies manually. This is what Chatterbox requires:
|
||||
```
|
||||
# In justfile (NOT requirements.txt):
|
||||
pip install --no-deps chatterbox-tts
|
||||
|
||||
# In requirements.txt — list the transitive deps:
|
||||
conformer
|
||||
diffusers
|
||||
omegaconf
|
||||
# ... etc
|
||||
```
|
||||
|
||||
**Non-PyPI packages** — Some deps only exist as git repos:
|
||||
```
|
||||
linacodec @ git+https://github.com/user/repo.git
|
||||
Zipvoice @ git+https://github.com/user/repo.git
|
||||
```
|
||||
|
||||
**Custom package indexes** — Some packages need `--find-links`:
|
||||
```
|
||||
--find-links https://k2-fsa.github.io/icefall/piper_phonemize.html
|
||||
```
|
||||
|
||||
### 4.2 Identifying hidden sub-dependencies
|
||||
|
||||
When using `--no-deps`, you need to manually figure out what the package actually imports. There's no shortcut:
|
||||
|
||||
1. Install the package normally in a throwaway venv
|
||||
2. Run `pip show <package>` to get its `Requires:` list
|
||||
3. Cross-reference against what's already in our requirements.txt
|
||||
4. Test that the engine loads and generates without import errors
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: PyInstaller Bundling
|
||||
|
||||
This is where most of the pain lives. If your model's Python package or its dependencies use any of the following at runtime, PyInstaller won't bundle them automatically:
|
||||
|
||||
### 5.1 Common PyInstaller issues
|
||||
|
||||
| Issue | Symptom | Fix |
|
||||
|-------|---------|-----|
|
||||
| **`inspect.getsource()` at import time** | "could not get source code" | `--collect-all <package>` (bundles `.py` source files, not just bytecode) |
|
||||
| **Data files (yaml, .pth.tar, lang dicts)** | FileNotFoundError at runtime | `--collect-all <package>` or `--collect-data <package>` |
|
||||
| **Native data paths (espeak-ng, etc.)** | Library looks at `/usr/share/...` | Set env var in frozen builds: `os.environ["ESPEAK_DATA_PATH"] = bundled_path` |
|
||||
| **`importlib.metadata` lookups** | "No package metadata found" | `--copy-metadata <package>` |
|
||||
| **Dynamic imports** | ModuleNotFoundError | `--hidden-import <module>` |
|
||||
| **`typeguard` / `@typechecked`** | Calls `inspect.getsource()` on decorated functions | `--collect-all` for the decorated package |
|
||||
|
||||
### 5.2 Testing frozen builds
|
||||
|
||||
You can't skip this. Models that work in `python -m uvicorn` will break in the PyInstaller binary. The flow:
|
||||
|
||||
1. Build the binary: `just build` or the PyInstaller spec
|
||||
2. Run it and try to download + load + generate with the new engine
|
||||
3. Check stderr for the actual error (macOS/Linux: stdout/stderr go to Tauri sidecar logs)
|
||||
4. Fix, rebuild, repeat
|
||||
|
||||
### 5.3 Real examples from v0.2.3
|
||||
|
||||
These were all models that worked perfectly in dev:
|
||||
|
||||
- **LuxTTS**: `typeguard`'s `@typechecked` calls `inspect.getsource()` at import → needed `--collect-all inflect`. `piper_phonemize` bundles `espeak-ng-data/` → needed `--collect-all piper_phonemize` + `ESPEAK_DATA_PATH` env var
|
||||
- **Chatterbox**: `resemble-perth` bundles a pretrained watermark model (`.pth.tar`, `hparams.yaml`) → needed `--collect-all perth`
|
||||
- **Both**: `huggingface_hub` silently disables tqdm based on logger level → progress bars showed 0% in frozen builds until we force-enabled the internal counter
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Common Upstream Workarounds
|
||||
|
||||
Almost every model library has bugs you'll need to work around. Here's the catalog:
|
||||
|
||||
### 6.1 torch.load device mismatch
|
||||
|
||||
If model weights were saved on CUDA but you're loading on CPU/MPS:
|
||||
```python
|
||||
_original_torch_load = torch.load
|
||||
def _patched_torch_load(*args, **kwargs):
|
||||
kwargs.setdefault("map_location", "cpu")
|
||||
return _original_torch_load(*args, **kwargs)
|
||||
torch.load = _patched_torch_load
|
||||
```
|
||||
Used by both Chatterbox backends. Use a threading lock if patching globally.
|
||||
|
||||
### 6.2 Float64/Float32 dtype mismatch
|
||||
|
||||
`librosa` returns float64, model weights are float32. Patch the offending methods:
|
||||
```python
|
||||
original_fn = SomeClass.some_method
|
||||
def patched_fn(self, *args, **kwargs):
|
||||
result = original_fn(self, *args, **kwargs)
|
||||
return result.float() # float64 → float32
|
||||
SomeClass.some_method = patched_fn
|
||||
```
|
||||
Used by Chatterbox for `S3Tokenizer.log_mel_spectrogram` and `VoiceEncoder.forward`.
|
||||
|
||||
### 6.3 Transformers attention implementation
|
||||
|
||||
If the model uses `output_attentions=True` with transformers >= 4.36:
|
||||
```python
|
||||
for module in model.modules():
|
||||
if hasattr(module, '_attn_implementation'):
|
||||
module._attn_implementation = "eager"
|
||||
```
|
||||
SDPA (the new default) doesn't support `output_attentions`. Force eager attention.
|
||||
|
||||
### 6.4 HuggingFace token bug
|
||||
|
||||
Some models' `from_pretrained()` passes `token=True` which requires a stored HF token even for public repos:
|
||||
```python
|
||||
from huggingface_hub import snapshot_download
|
||||
local_path = snapshot_download(repo_id=REPO, token=None)
|
||||
model = ModelClass.from_local(local_path, device=device)
|
||||
```
|
||||
Used by Chatterbox Turbo.
|
||||
|
||||
### 6.5 MPS tensor issues
|
||||
|
||||
MPS (Apple Silicon GPU) has incomplete operator coverage. If generation crashes on MPS:
|
||||
```python
|
||||
def _get_device(self):
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
return "cpu" # Skip MPS entirely
|
||||
```
|
||||
Used by both Chatterbox backends. LuxTTS works fine on MPS.
|
||||
|
||||
### 6.6 HuggingFace progress tracking
|
||||
|
||||
To get download progress bars in the UI, wrap model loading with `HFProgressTracker`:
|
||||
```python
|
||||
from ..utils.hf_progress import HFProgressTracker
|
||||
tracker = HFProgressTracker(model_name, progress_manager)
|
||||
with tracker.patch_download():
|
||||
model = ModelClass.from_pretrained(repo_id)
|
||||
```
|
||||
The tracker monkey-patches tqdm to intercept HuggingFace's internal progress bars. Must be set up BEFORE importing the model library if it imports HF at module level.
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
### Backend
|
||||
- [ ] `backend/backends/<engine>_backend.py` — implements TTSBackend protocol
|
||||
- [ ] `backend/backends/__init__.py` — `ModelConfig` entry + `TTS_ENGINES` + `get_tts_backend_for_engine()` elif
|
||||
- [ ] `backend/models.py` — engine name in regex, any new language codes
|
||||
- [ ] `backend/requirements.txt` — dependencies added (check for `--no-deps` needs)
|
||||
- [ ] `justfile` — `--no-deps` install step if needed
|
||||
|
||||
### Routes and services
|
||||
No changes needed — the model config registry handles all dispatch automatically.
|
||||
|
||||
### Frontend
|
||||
- [ ] `app/src/lib/api/types.ts` — engine union type
|
||||
- [ ] `app/src/lib/constants/languages.ts` — `ENGINE_LANGUAGES` entry
|
||||
- [ ] `app/src/components/Generation/EngineModelSelector.tsx` — `ENGINE_OPTIONS` + `ENGINE_DESCRIPTIONS` + `ENGLISH_ONLY_ENGINES`
|
||||
- [ ] `app/src/lib/hooks/useGenerationForm.ts` — Zod schema + model mapping
|
||||
- [ ] `app/src/components/ServerSettings/ModelManagement.tsx` — model description
|
||||
|
||||
### Production
|
||||
- [ ] PyInstaller spec — `--collect-all`, `--hidden-import`, `--copy-metadata` as needed
|
||||
- [ ] Test in frozen binary — download, load, generate all work
|
||||
- [ ] Download progress — `HFProgressTracker` wired up, progress shows in UI
|
||||
|
||||
### Upstream workarounds (check which apply)
|
||||
- [ ] torch.load device mapping (CUDA weights on CPU)
|
||||
- [ ] Float64→Float32 patches (librosa interaction)
|
||||
- [ ] Eager attention forcing (transformers >= 4.36)
|
||||
- [ ] HF token bypass (snapshot_download + from_local)
|
||||
- [ ] MPS skip (if operators not supported)
|
||||
- [ ] espeak-ng / native data path env vars
|
||||
@@ -0,0 +1,581 @@
|
||||
# CUDA Backend Swap via Binary Replacement
|
||||
|
||||
> Status: Plan | Target: v0.2.0 | Created: 2026-03-12
|
||||
|
||||
## Problem
|
||||
|
||||
The CUDA PyTorch backend binary is ~2.4 GB. GitHub Releases has a 2 GB asset limit. The current release ships CPU-only PyTorch on Windows and Intel Mac — NVIDIA GPU users get no acceleration from official releases. This is the #1 reported issue category (19 open issues).
|
||||
|
||||
Users who want GPU today must clone the repo and run from source. That's not acceptable for a desktop app targeting non-technical users.
|
||||
|
||||
## Solution
|
||||
|
||||
Ship two backend binaries: a default CPU build (~150 MB) bundled with the app, and a downloadable CUDA build (~2.4 GB) hosted externally. When the user downloads the CUDA build, the app kills the current backend process, swaps in the CUDA binary, and relaunches — a backend-only restart. The frontend stays running, all UI state is preserved.
|
||||
|
||||
No subprocesses. No HTTP protocol between processes. No port allocation. No provider manager. The backend is still one monolithic process — just a different binary.
|
||||
|
||||
## Architecture
|
||||
|
||||
### What Exists Today
|
||||
|
||||
```
|
||||
Tauri App
|
||||
├── React Frontend (in-process webview)
|
||||
└── voicebox-server (sidecar subprocess on :17493)
|
||||
└── One PyInstaller binary: CPU PyTorch or MLX
|
||||
```
|
||||
|
||||
**Sidecar lifecycle** (`tauri/src-tauri/src/main.rs`):
|
||||
- `start_server` command spawns `voicebox-server` sidecar (line 181)
|
||||
- Binary located at `tauri/src-tauri/binaries/voicebox-server-{platform-triple}`
|
||||
- Tauri resolves the sidecar name via `externalBin` in `tauri.conf.json` (line 16)
|
||||
- Waits up to 120s for "Uvicorn running" in stdout/stderr (line 286)
|
||||
- `stop_server` kills the process tree (line 466)
|
||||
|
||||
**Frontend reconnection** (`app/src/lib/hooks/useServer.ts`):
|
||||
- Health check polls `GET /health` every 30 seconds
|
||||
- React Query cache retains data for 10 minutes after disconnect
|
||||
- All UI state (Zustand stores, form data, open tabs) survives disconnection
|
||||
- No active reconnect logic — just keeps polling until server responds
|
||||
|
||||
This means a backend restart is mostly invisible to the frontend: it sees a few seconds of failed health checks, then the server comes back. The only risk is in-flight operations (generation, transcription) failing mid-request.
|
||||
|
||||
### What Changes
|
||||
|
||||
```
|
||||
Tauri App
|
||||
├── React Frontend (in-process webview)
|
||||
└── voicebox-server (sidecar subprocess on :17493)
|
||||
└── One of:
|
||||
├── voicebox-server-cpu (bundled, ~150 MB)
|
||||
└── voicebox-server-cuda (downloaded, ~2.4 GB)
|
||||
```
|
||||
|
||||
The CUDA binary is functionally identical to the CPU binary. Same FastAPI app, same endpoints, same code. The only difference is PyTorch is compiled with CUDA 12.1 support and the binary includes CUDA runtime libraries.
|
||||
|
||||
The user downloads it once. On every subsequent app launch, Tauri checks which binary variant exists and spawns the appropriate one.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Build Infrastructure
|
||||
|
||||
Build the CUDA binary in CI separately from the main release.
|
||||
|
||||
#### 1a. CUDA PyInstaller Build
|
||||
|
||||
Add a `build_binary_cuda.py` or parameterize the existing `build_binary.py`:
|
||||
|
||||
```python
|
||||
# backend/build_binary.py — add flag
|
||||
def build_server(cuda=False):
|
||||
args = [
|
||||
'server.py',
|
||||
'--onefile',
|
||||
'--name', f'voicebox-server-{"cuda" if cuda else "cpu"}',
|
||||
]
|
||||
|
||||
if cuda:
|
||||
args.extend([
|
||||
'--hidden-import', 'torch.cuda',
|
||||
'--hidden-import', 'torch.backends.cudnn',
|
||||
])
|
||||
# ... rest of existing build
|
||||
```
|
||||
|
||||
The `--onefile` flag is already used, which produces a single executable. This is important — `--onedir` would complicate the swap (replacing a directory vs a file).
|
||||
|
||||
#### 1b. CI Workflow for CUDA Binary
|
||||
|
||||
New workflow: `.github/workflows/build-cuda.yml`
|
||||
|
||||
```yaml
|
||||
name: Build CUDA Provider
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
|
||||
jobs:
|
||||
build-cuda:
|
||||
runs-on: windows-latest # CUDA is Windows/Linux only
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with: { python-version: "3.12" }
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install pyinstaller
|
||||
pip install -r backend/requirements.txt
|
||||
pip install torch --index-url https://download.pytorch.org/whl/cu121 --force-reinstall
|
||||
- name: Build CUDA binary
|
||||
run: python backend/build_binary.py --cuda
|
||||
- name: Split binary for GitHub Releases
|
||||
run: |
|
||||
python scripts/split_binary.py backend/dist/voicebox-server-cuda.exe \
|
||||
--chunk-size 1900MB \
|
||||
--output release-assets/
|
||||
- name: Upload to R2
|
||||
# Full binary to R2 (no size limit)
|
||||
run: |
|
||||
aws s3 cp backend/dist/voicebox-server-cuda.exe \
|
||||
s3://voicebox-downloads/cuda/v${{ github.ref_name }}/voicebox-server-cuda.exe \
|
||||
--endpoint-url ${{ secrets.R2_ENDPOINT }}
|
||||
- name: Upload split parts to GitHub Release
|
||||
# Split parts as GitHub Release assets (each <2 GB)
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: release-assets/*
|
||||
```
|
||||
|
||||
Two distribution paths for redundancy:
|
||||
- **Cloudflare R2**: Full binary, direct download, no size limit.
|
||||
- **GitHub Releases**: Split into <2 GB chunks as fallback.
|
||||
|
||||
#### 1c. Binary Splitting Script
|
||||
|
||||
```python
|
||||
# scripts/split_binary.py
|
||||
"""Split a large binary into chunks for GitHub Releases."""
|
||||
import hashlib
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
def split(input_path: Path, chunk_size: int, output_dir: Path):
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
data = input_path.read_bytes()
|
||||
|
||||
# Write SHA-256 of the complete file
|
||||
sha256 = hashlib.sha256(data).hexdigest()
|
||||
(output_dir / f"{input_path.stem}.sha256").write_text(
|
||||
f"{sha256} {input_path.name}\n"
|
||||
)
|
||||
|
||||
# Split into chunks
|
||||
parts = []
|
||||
for i in range(0, len(data), chunk_size):
|
||||
part_name = f"{input_path.stem}.part{len(parts):02d}{input_path.suffix}"
|
||||
part_path = output_dir / part_name
|
||||
part_path.write_bytes(data[i:i + chunk_size])
|
||||
parts.append(part_name)
|
||||
|
||||
# Write manifest
|
||||
(output_dir / f"{input_path.stem}.manifest").write_text(
|
||||
"\n".join(parts) + "\n"
|
||||
)
|
||||
|
||||
print(f"Split into {len(parts)} parts, SHA-256: {sha256}")
|
||||
```
|
||||
|
||||
### Phase 2: Download & Assemble in App
|
||||
|
||||
#### 2a. Backend Download Endpoint
|
||||
|
||||
Add to `backend/main.py`:
|
||||
|
||||
```python
|
||||
@app.post("/backend/download-cuda")
|
||||
async def download_cuda_backend():
|
||||
"""Download the CUDA backend binary."""
|
||||
# Returns immediately, runs download in background
|
||||
task = asyncio.create_task(_download_cuda_binary())
|
||||
task.add_done_callback(lambda t: logger.error(f"CUDA download failed: {t.exception()}") if t.exception() else None)
|
||||
return {"status": "downloading"}
|
||||
|
||||
@app.get("/backend/cuda-status")
|
||||
async def cuda_status():
|
||||
"""Check if CUDA binary is available."""
|
||||
cuda_path = _get_cuda_binary_path()
|
||||
return {
|
||||
"available": cuda_path is not None and cuda_path.exists(),
|
||||
"active": _is_cuda_active(),
|
||||
"download_progress": progress_manager.get_progress("cuda-backend"),
|
||||
}
|
||||
```
|
||||
|
||||
#### 2b. Download + Assemble + Verify Logic
|
||||
|
||||
New file: `backend/cuda_download.py`
|
||||
|
||||
Core logic:
|
||||
|
||||
```python
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from backend.config import get_data_dir
|
||||
from backend.utils.progress import get_progress_manager
|
||||
|
||||
CUDA_DOWNLOAD_URL = "https://downloads.voicebox.sh/cuda/{version}/voicebox-server-cuda{ext}"
|
||||
CUDA_CHECKSUMS = {
|
||||
# Populated per release
|
||||
"0.2.0-windows": "sha256:abc123...",
|
||||
"0.2.0-linux": "sha256:def456...",
|
||||
}
|
||||
|
||||
def get_cuda_binary_dir() -> Path:
|
||||
"""Where CUDA binaries live. Inside the app's data directory."""
|
||||
return get_data_dir() / "backends"
|
||||
|
||||
def get_cuda_binary_path() -> Path | None:
|
||||
"""Return path to CUDA binary if it exists and is verified."""
|
||||
d = get_cuda_binary_dir()
|
||||
for name in ["voicebox-server-cuda.exe", "voicebox-server-cuda"]:
|
||||
p = d / name
|
||||
if p.exists():
|
||||
return p
|
||||
return None
|
||||
|
||||
async def download_cuda_binary(version: str):
|
||||
"""Download, assemble (if split), and verify the CUDA binary."""
|
||||
progress = get_progress_manager()
|
||||
dest_dir = get_cuda_binary_dir()
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ext = ".exe" if sys.platform == "win32" else ""
|
||||
url = CUDA_DOWNLOAD_URL.format(version=version, ext=ext)
|
||||
|
||||
# Download with progress tracking
|
||||
temp_path = dest_dir / f"voicebox-server-cuda{ext}.download"
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
total = int(response.headers.get("content-length", 0))
|
||||
downloaded = 0
|
||||
with open(temp_path, "wb") as f:
|
||||
async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
progress.update("cuda-backend", downloaded, total)
|
||||
|
||||
# Verify checksum
|
||||
sha256 = hashlib.sha256(temp_path.read_bytes()).hexdigest()
|
||||
expected = CUDA_CHECKSUMS.get(f"{version}-{sys.platform}")
|
||||
if expected and not expected.endswith(sha256):
|
||||
temp_path.unlink()
|
||||
raise ValueError(f"Checksum mismatch: expected {expected}, got sha256:{sha256}")
|
||||
|
||||
# Atomic move into place
|
||||
final_path = dest_dir / f"voicebox-server-cuda{ext}"
|
||||
temp_path.rename(final_path)
|
||||
|
||||
# Make executable on Unix
|
||||
if sys.platform != "win32":
|
||||
final_path.chmod(0o755)
|
||||
|
||||
progress.complete("cuda-backend")
|
||||
```
|
||||
|
||||
Key points:
|
||||
- Downloads to a `.download` temp file, verifies checksum, then atomically renames. No partial binaries left on crash.
|
||||
- Progress tracked via the existing `ProgressManager` so the frontend SSE system works unchanged.
|
||||
- CUDA binary lives in the **app data directory** (`data/backends/`), not alongside the app bundle. This avoids code-signing issues on macOS (though CUDA isn't relevant on macOS) and survives app updates.
|
||||
|
||||
#### 2c. Reassembly from Split Parts (GitHub Releases Fallback)
|
||||
|
||||
If the R2 download fails, fall back to downloading split parts from GitHub Releases:
|
||||
|
||||
```python
|
||||
async def download_cuda_from_github(version: str):
|
||||
"""Fallback: download split parts from GitHub Releases, reassemble."""
|
||||
base_url = f"https://github.com/jamiepine/voicebox/releases/download/v{version}"
|
||||
|
||||
# Get manifest
|
||||
manifest_url = f"{base_url}/voicebox-server-cuda.manifest"
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
manifest = (await client.get(manifest_url)).text
|
||||
parts = [p.strip() for p in manifest.strip().splitlines()]
|
||||
|
||||
# Download checksum
|
||||
sha256_url = f"{base_url}/voicebox-server-cuda.sha256"
|
||||
expected_sha = (await client.get(sha256_url)).text.split()[0]
|
||||
|
||||
# Download parts
|
||||
dest_dir = get_cuda_binary_dir()
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = dest_dir / "voicebox-server-cuda.exe.download"
|
||||
|
||||
total_downloaded = 0
|
||||
with open(temp_path, "wb") as f:
|
||||
for i, part_name in enumerate(parts):
|
||||
part_url = f"{base_url}/{part_name}"
|
||||
async with client.stream("GET", part_url) as response:
|
||||
async for chunk in response.aiter_bytes(chunk_size=1024 * 1024):
|
||||
f.write(chunk)
|
||||
total_downloaded += len(chunk)
|
||||
get_progress_manager().update(
|
||||
"cuda-backend", total_downloaded, None,
|
||||
message=f"Downloading part {i+1}/{len(parts)}"
|
||||
)
|
||||
|
||||
# Verify reassembled file
|
||||
sha256 = hashlib.sha256(temp_path.read_bytes()).hexdigest()
|
||||
if sha256 != expected_sha:
|
||||
temp_path.unlink()
|
||||
raise ValueError(f"Checksum mismatch after reassembly")
|
||||
|
||||
final_path = dest_dir / "voicebox-server-cuda.exe"
|
||||
temp_path.rename(final_path)
|
||||
get_progress_manager().complete("cuda-backend")
|
||||
```
|
||||
|
||||
### Phase 3: Backend Restart (The Swap)
|
||||
|
||||
This is the core of the feature: kill the CPU backend, launch the CUDA backend, frontend reconnects automatically.
|
||||
|
||||
#### 3a. New Tauri Command: `restart_server`
|
||||
|
||||
Add to `tauri/src-tauri/src/main.rs`:
|
||||
|
||||
```rust
|
||||
#[command]
|
||||
async fn restart_server(
|
||||
app: tauri::AppHandle,
|
||||
state: State<'_, ServerState>,
|
||||
use_cuda: Option<bool>,
|
||||
) -> Result<String, String> {
|
||||
println!("restart_server: use_cuda={:?}", use_cuda);
|
||||
|
||||
// 1. Stop the current server
|
||||
stop_server(state.clone()).await?;
|
||||
|
||||
// 2. Brief wait for port release
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// 3. Start with the appropriate binary
|
||||
// The start_server logic needs to check for CUDA binary
|
||||
start_server(app, state, None).await
|
||||
}
|
||||
```
|
||||
|
||||
#### 3b. Modify `start_server` to Prefer CUDA Binary
|
||||
|
||||
The existing `start_server` uses `app.shell().sidecar("voicebox-server")` which resolves via Tauri's `externalBin` config. For the CUDA binary (which lives in the data directory, not the app bundle), we need an alternative launch path.
|
||||
|
||||
Modify `start_server` in `main.rs`:
|
||||
|
||||
```rust
|
||||
// After the existing sidecar logic, before spawning:
|
||||
|
||||
// Check for CUDA binary in data directory
|
||||
let cuda_binary = data_dir.join("backends")
|
||||
.join(if cfg!(windows) { "voicebox-server-cuda.exe" } else { "voicebox-server-cuda" });
|
||||
|
||||
let (mut rx, child) = if cuda_binary.exists() {
|
||||
println!("Found CUDA backend binary at {:?}", cuda_binary);
|
||||
|
||||
// Launch CUDA binary directly (not as Tauri sidecar)
|
||||
let mut cmd = app.shell().command(cuda_binary.to_str().unwrap());
|
||||
cmd = cmd.args([
|
||||
"--data-dir",
|
||||
data_dir.to_str().ok_or("Invalid data dir path")?,
|
||||
"--port",
|
||||
&SERVER_PORT.to_string(),
|
||||
]);
|
||||
if remote.unwrap_or(false) {
|
||||
cmd = cmd.args(["--host", "0.0.0.0"]);
|
||||
}
|
||||
cmd.spawn().map_err(|e| format!("Failed to spawn CUDA backend: {}", e))?
|
||||
} else {
|
||||
// Existing sidecar launch (CPU binary bundled with app)
|
||||
sidecar.spawn().map_err(|e| format!("Failed to spawn: {}", e))?
|
||||
};
|
||||
```
|
||||
|
||||
Key decisions:
|
||||
- CUDA binary is launched via `app.shell().command()` (arbitrary path), not `app.shell().sidecar()` (bundled path). Tauri's sidecar system only resolves binaries within the app bundle.
|
||||
- The CUDA binary gets the same args (`--data-dir`, `--port`) as the CPU binary. It's the same `server.py` entry point.
|
||||
- Preference: if CUDA binary exists, use it. Otherwise fall back to bundled CPU. No user configuration needed.
|
||||
|
||||
#### 3c. Frontend: Trigger Restart After Download
|
||||
|
||||
Add to the platform lifecycle interface (`app/src/platform/types.ts`):
|
||||
|
||||
```typescript
|
||||
interface PlatformLifecycle {
|
||||
startServer(remote?: boolean): Promise<string>;
|
||||
stopServer(): Promise<void>;
|
||||
restartServer(useCuda?: boolean): Promise<string>; // new
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
Implement in `tauri/src/platform/lifecycle.ts`:
|
||||
|
||||
```typescript
|
||||
async restartServer(useCuda?: boolean): Promise<string> {
|
||||
const result = await invoke<string>('restart_server', { useCuda });
|
||||
this.onServerReady?.();
|
||||
return result;
|
||||
}
|
||||
```
|
||||
|
||||
#### 3d. Frontend: GPU Settings UI
|
||||
|
||||
Add a section to the Server Settings page (or Model Management). Minimal UI:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ GPU Acceleration │
|
||||
│ │
|
||||
│ Status: CPU only (no CUDA backend) │
|
||||
│ │
|
||||
│ [Download CUDA Backend (2.4 GB)] │
|
||||
│ │
|
||||
│ Requires an NVIDIA GPU with 4+ GB VRAM. │
|
||||
│ The app will restart its backend process │
|
||||
│ after download. Your work is preserved. │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
After download:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ GPU Acceleration │
|
||||
│ │
|
||||
│ Status: ✓ CUDA backend active (RTX 4090) │
|
||||
│ │
|
||||
│ [Switch to CPU] [Delete CUDA Backend] │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
#### 3e. Frontend: Reconnection During Restart
|
||||
|
||||
The current health poll interval is 30 seconds — too slow for a restart UX. During a restart, temporarily increase polling:
|
||||
|
||||
```typescript
|
||||
// In the component that triggers restart:
|
||||
const restart = async () => {
|
||||
setRestarting(true);
|
||||
try {
|
||||
await platform.lifecycle.restartServer(true);
|
||||
} catch (e) {
|
||||
// Frontend will show "reconnecting" state
|
||||
}
|
||||
// Aggressively poll until health check succeeds
|
||||
const interval = setInterval(async () => {
|
||||
try {
|
||||
await apiClient.getHealth();
|
||||
clearInterval(interval);
|
||||
setRestarting(false);
|
||||
queryClient.invalidateQueries(); // Refresh all data
|
||||
} catch {}
|
||||
}, 1000); // Poll every 1s during restart
|
||||
// Safety timeout
|
||||
setTimeout(() => clearInterval(interval), 30000);
|
||||
};
|
||||
```
|
||||
|
||||
### Phase 4: Auto-Detection on Startup
|
||||
|
||||
No user action needed on subsequent launches. The preference logic in `start_server` (Phase 3b) handles this:
|
||||
|
||||
1. App launches → `start_server` called
|
||||
2. Check `data/backends/voicebox-server-cuda{.exe}`
|
||||
3. If exists → launch CUDA binary
|
||||
4. If not → launch bundled CPU binary
|
||||
|
||||
The user downloads CUDA once, and every future app launch (including after updates) uses it automatically. The CUDA binary lives in the data directory, not the app bundle, so app updates don't overwrite it.
|
||||
|
||||
### Phase 5: Handling Version Mismatches
|
||||
|
||||
When the app updates but the CUDA binary is from an older version, the API might be incompatible. Handle this by:
|
||||
|
||||
1. Add `--version` flag to `server.py`:
|
||||
|
||||
```python
|
||||
parser.add_argument("--version", action="store_true")
|
||||
# If invoked with --version, print version and exit
|
||||
if args.version:
|
||||
from backend import __version__
|
||||
print(f"voicebox-server {__version__}")
|
||||
sys.exit(0)
|
||||
```
|
||||
|
||||
2. In `start_server` (Rust), before launching the CUDA binary:
|
||||
|
||||
```rust
|
||||
// Quick version check
|
||||
let version_output = std::process::Command::new(cuda_binary.to_str().unwrap())
|
||||
.arg("--version")
|
||||
.output();
|
||||
|
||||
match version_output {
|
||||
Ok(output) => {
|
||||
let version = String::from_utf8_lossy(&output.stdout);
|
||||
let app_version = env!("CARGO_PKG_VERSION");
|
||||
if !version.contains(app_version) {
|
||||
println!("CUDA binary version mismatch (app: {}, cuda: {}), falling back to CPU",
|
||||
app_version, version.trim());
|
||||
// Fall through to CPU sidecar launch
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
println!("Failed to check CUDA binary version, falling back to CPU");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. Frontend shows a notification: "Your GPU backend needs an update. [Download latest] or [Use CPU for now]"
|
||||
|
||||
## Files Changed
|
||||
|
||||
### New Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `backend/cuda_download.py` | Download, reassemble, verify CUDA binary |
|
||||
| `scripts/split_binary.py` | Split binary into <2 GB chunks for GitHub Releases |
|
||||
| `.github/workflows/build-cuda.yml` | CI: build + upload CUDA binary |
|
||||
|
||||
### Modified Files
|
||||
|
||||
| File | Change |
|
||||
|------|--------|
|
||||
| `tauri/src-tauri/src/main.rs` | Add `restart_server` command, modify `start_server` to check for CUDA binary in data dir |
|
||||
| `backend/server.py` | Add `--version` flag |
|
||||
| `backend/main.py` | Add `/backend/download-cuda`, `/backend/cuda-status`, `/backend/progress/cuda-backend` endpoints |
|
||||
| `backend/build_binary.py` | Accept `--cuda` flag to build CUDA variant |
|
||||
| `app/src/platform/types.ts` | Add `restartServer` to lifecycle interface |
|
||||
| `tauri/src/platform/lifecycle.ts` | Implement `restartServer` |
|
||||
| `app/src/components/ServerSettings/` | New GPU acceleration section |
|
||||
| `.github/workflows/release.yml` | Trigger CUDA build workflow on tag |
|
||||
|
||||
### NOT Changed
|
||||
|
||||
| File | Why |
|
||||
|------|-----|
|
||||
| `backend/backends/__init__.py` | No changes to the TTSBackend singleton or factory. CUDA binary runs the same code. |
|
||||
| `backend/backends/pytorch_backend.py` | Already detects CUDA at runtime (line 28-49). No changes needed. |
|
||||
| `app/src/lib/api/client.ts` | API is identical between CPU and CUDA backends. |
|
||||
| `app/src/lib/hooks/useGenerationForm.ts` | Generation flow is unchanged. |
|
||||
|
||||
## What This Doesn't Solve
|
||||
|
||||
- **Multi-model support** — This is purely about GPU acceleration. LuxTTS, Chatterbox, etc. need the in-process model registry, which is an independent workstream.
|
||||
- **AMD GPU support** — DirectML/ROCm needs a different PyTorch build. Same pattern applies (another binary variant) but deferred.
|
||||
- **Linux CUDA** — Same approach works, just another CI matrix entry. Can be added in the same release or shortly after.
|
||||
- **Remote server mode** — Users who want to run TTS on a different machine still need the external provider architecture. Separate concern.
|
||||
|
||||
## What This DOES Solve
|
||||
|
||||
- **19 "GPU not detected" issues** — Users download the CUDA backend, restart, GPU works.
|
||||
- **2 GB GitHub Release limit** — Binary splitting + R2 hosting.
|
||||
- **Update burden** — App updates don't re-download the 2.4 GB CUDA binary. It persists in the data directory.
|
||||
- **First-run experience** — App works immediately on CPU. GPU is an optional enhancement, not a setup blocker.
|
||||
|
||||
## Rollout Plan
|
||||
|
||||
1. Build and test CUDA binary locally on Windows with an NVIDIA GPU.
|
||||
2. Set up R2 bucket at `downloads.voicebox.sh/cuda/`.
|
||||
3. Ship the backend restart + download UI in v0.2.0.
|
||||
4. Announce: "GPU acceleration is here — one click in Settings."
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Mitigation |
|
||||
|------|-----------|
|
||||
| CUDA binary doesn't work on some GPU/driver combos | `/health` endpoint reports GPU info. Fallback to CPU if CUDA init fails. Clear error message. |
|
||||
| Antivirus flags downloaded binary (Windows) | Code-sign the CUDA binary in CI. Document AV exceptions. |
|
||||
| Data dir CUDA binary survives app uninstall | Document in uninstall notes. Not a real problem — it's just a file. |
|
||||
| Version mismatch after app update | Version check on startup (Phase 5). Auto-fallback to CPU. Prompt to re-download. |
|
||||
| R2 downtime | GitHub Releases split-binary fallback. |
|
||||
| Download interrupted | Temp file with `.download` extension. Atomic rename on completion. Resume not implemented in v1 — restart download from scratch. |
|
||||
@@ -0,0 +1,133 @@
|
||||
# CUDA Backend Swap — Implementation Summary
|
||||
|
||||
> Status: **Complete** | Branch: `feat/cuda-backend-swap` | Created: 2026-03-12
|
||||
|
||||
## What This Is
|
||||
|
||||
A standalone feature that lets users download a CUDA-enabled backend binary (~2.4 GB) and swap it in via a backend-only restart. The frontend stays running, all UI state is preserved. This solves the #1 user pain point: 19 open issues about "GPU not detected" caused by GitHub's 2 GB release asset limit preventing CUDA binaries from shipping in official releases.
|
||||
|
||||
## How It Works
|
||||
|
||||
```
|
||||
User clicks "Download CUDA Backend" in Settings
|
||||
→ Backend fetches manifest from GitHub Releases
|
||||
→ Downloads split parts (<2 GB each), concatenates them
|
||||
→ SHA-256 integrity check on reassembled binary
|
||||
→ Binary placed in {app_data_dir}/backends/voicebox-server-cuda
|
||||
→ User clicks "Switch to CUDA Backend"
|
||||
→ Tauri kills CPU process, launches CUDA binary, frontend reconnects
|
||||
→ On all future app launches, CUDA binary is auto-detected and used
|
||||
```
|
||||
|
||||
The CUDA binary is functionally identical to the CPU binary — same FastAPI app, same endpoints, same code. The only difference is PyTorch compiled with CUDA 12.1 and bundled CUDA runtime libraries.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
**Backend-only restart, not full app restart.** The Tauri shell kills the current `voicebox-server` process, waits 1 second for port release, and spawns the new binary. The React frontend stays running. Health polling detects the new backend within seconds.
|
||||
|
||||
**No provider/subprocess architecture.** This is explicitly not the PR #33 approach (10K+ lines, 136 files, 22 bugs). One process at a time. The CUDA binary replaces the CPU binary — it doesn't run alongside it.
|
||||
|
||||
**Data directory, not app bundle.** The CUDA binary lives in `{app_data_dir}/backends/`, which persists across app updates and avoids code-signing issues. The bundled CPU binary in the app bundle is untouched.
|
||||
|
||||
**Version mismatch protection.** On startup, Rust runs `voicebox-server-cuda --version` and compares to the app version from `tauri.conf.json`. If they don't match (e.g., after an app update), it falls back to the bundled CPU binary silently.
|
||||
|
||||
**GitHub Releases distribution.** The CUDA binary is split into <2 GB chunks (GitHub's asset limit) via `scripts/split_binary.py`. The app downloads a manifest, fetches each part, concatenates them, and runs a SHA-256 integrity check to verify reassembly. No external hosting needed.
|
||||
|
||||
## Files Changed
|
||||
|
||||
### New Files
|
||||
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `backend/cuda_download.py` | ~190 | Download split parts from GitHub Releases, reassemble, verify integrity |
|
||||
| `scripts/split_binary.py` | ~80 | Split large binary into <2 GB chunks with SHA-256 manifest |
|
||||
| `.github/workflows/build-cuda.yml` | ~70 | CI workflow: build CUDA binary, split, upload to GitHub Releases |
|
||||
| `app/src/components/ServerSettings/GpuAcceleration.tsx` | 371 | GPU Acceleration UI card (status, download, restart, delete) |
|
||||
| `docs/plans/CUDA_BACKEND_SWAP.md` | 581 | Original implementation plan (5 phases with code sketches) |
|
||||
| `docs/plans/CUDA_BACKEND_SWAP_FINAL.md` | this file | Final implementation summary |
|
||||
| `docs/plans/PROJECT_STATUS.md` | 462 | Full project triage (all PRs, issues, architecture) |
|
||||
| `docs/plans/PR33_CUDA_PROVIDER_REVIEW.md` | ~350 | Detailed code review of PR #33 (22 bugs documented) |
|
||||
|
||||
### Modified Files
|
||||
|
||||
| File | What Changed |
|
||||
|------|-------------|
|
||||
| `backend/build_binary.py` | Added `--cuda` flag, parameterized output binary name |
|
||||
| `backend/server.py` | Added `--version` flag, auto-detect backend variant from binary name (`VOICEBOX_BACKEND_VARIANT` env var) |
|
||||
| `backend/main.py` | 4 new endpoints (`/backend/cuda-status`, `/backend/download-cuda`, `/backend/cuda`, `/backend/cuda-progress`), health endpoint returns `backend_variant` |
|
||||
| `backend/models.py` | `HealthResponse` model: added `backend_variant` field |
|
||||
| `backend/requirements.txt` | Added `httpx>=0.27.0` for async HTTP downloads |
|
||||
| `tauri/src-tauri/src/main.rs` | `restart_server` command (stop → wait → start), `start_server` checks for CUDA binary in data dir and launches via `shell().command()`, version mismatch check |
|
||||
| `app/src/platform/types.ts` | `PlatformLifecycle.restartServer()` added |
|
||||
| `tauri/src/platform/lifecycle.ts` | `restartServer()` implementation via `invoke('restart_server')` |
|
||||
| `web/src/platform/lifecycle.ts` | `restartServer()` noop for web platform |
|
||||
| `app/src/lib/api/types.ts` | `CudaStatus`, `CudaDownloadProgress` interfaces; `HealthResponse` updated with `gpu_type`, `backend_type`, `backend_variant` |
|
||||
| `app/src/lib/api/client.ts` | `getCudaStatus()`, `downloadCudaBackend()`, `deleteCudaBackend()` methods |
|
||||
| `app/src/components/ServerTab/ServerTab.tsx` | Wired in `<GpuAcceleration />` component (Tauri-only) |
|
||||
|
||||
## Backend API Endpoints
|
||||
|
||||
| Method | Path | Purpose |
|
||||
|--------|------|---------|
|
||||
| `GET` | `/backend/cuda-status` | Returns `{ available, active, binary_path, downloading, download_progress }` |
|
||||
| `POST` | `/backend/download-cuda` | Starts background download; returns immediately. Track via SSE. |
|
||||
| `DELETE` | `/backend/cuda` | Deletes CUDA binary (blocked if CUDA is currently active) |
|
||||
| `GET` | `/backend/cuda-progress` | SSE stream of download progress (reuses existing `ProgressManager`) |
|
||||
|
||||
The existing `GET /health` endpoint now returns two new fields:
|
||||
- `backend_type`: `"pytorch"` or `"mlx"` (existing detection)
|
||||
- `backend_variant`: `"cpu"` or `"cuda"` (set from `VOICEBOX_BACKEND_VARIANT` env var)
|
||||
|
||||
## Frontend UI States
|
||||
|
||||
The `GpuAcceleration` card in Server Settings handles these states:
|
||||
|
||||
1. **Native GPU detected** (MPS, MLX, XPU, DirectML) — Shows info message, no download needed
|
||||
2. **No CUDA binary** — Download button with size estimate, description of requirements
|
||||
3. **Downloading** — SSE-driven progress bar with bytes/total and percentage
|
||||
4. **Downloaded, not active** — "Switch to CUDA Backend" button + "Remove" option
|
||||
5. **CUDA active** — Shows CUDA badge, "Switch to CPU Backend" button
|
||||
6. **Restarting** — Spinner with phase text, 1s health polling as safety net
|
||||
7. **Error** — Red error message with details
|
||||
|
||||
### Key UX detail: switching to CPU
|
||||
|
||||
Since `start_server` always prefers the CUDA binary if it exists on disk, "Switch to CPU" must delete the CUDA binary first, then restart. The user can re-download later. This avoids a persistent configuration mechanism (no new state to manage, no new config file, no DB column).
|
||||
|
||||
## Rust: Server Lifecycle
|
||||
|
||||
```
|
||||
start_server
|
||||
├── Check for CUDA binary at {data_dir}/backends/voicebox-server-cuda
|
||||
├── If found: run --version, compare to app version
|
||||
│ ├── Match: launch via shell().command() with --data-dir, --port
|
||||
│ └── Mismatch: log warning, fall through to CPU
|
||||
└── Else: launch bundled sidecar via shell().sidecar()
|
||||
|
||||
restart_server
|
||||
├── stop_server (kill process tree)
|
||||
├── wait 1 second for port release
|
||||
└── start_server (auto-detects CUDA)
|
||||
```
|
||||
|
||||
## What This Doesn't Cover
|
||||
|
||||
- **AMD GPU / ROCm / DirectML binary** — Same pattern, different PyTorch build. Future PR.
|
||||
- **Linux CUDA** — Same approach, just another CI matrix entry. Can ship same release.
|
||||
- **Multi-model support** — LuxTTS, Chatterbox, etc. are a separate architectural concern (in-process model registry). Independent of binary variant.
|
||||
- **Download resume** — If download is interrupted, it restarts from scratch. Acceptable for v1.
|
||||
- **Remote server CUDA** — Users running voicebox-server on a remote machine manage their own binaries. This feature is for the desktop app.
|
||||
|
||||
## Testing Checklist
|
||||
|
||||
- [ ] Build CUDA binary locally with `python backend/build_binary.py --cuda`
|
||||
- [ ] `voicebox-server-cuda --version` prints correct version
|
||||
- [ ] Place CUDA binary in `{data_dir}/backends/`, launch app → auto-detects and uses it
|
||||
- [ ] Version mismatch: rename binary to have wrong version → falls back to CPU
|
||||
- [ ] Frontend: GpuAcceleration card shows correct state for CPU, CUDA available, CUDA active
|
||||
- [ ] Download flow: POST triggers download, SSE progress works, completion updates status
|
||||
- [ ] Switch to CUDA: restart works, health endpoint shows `backend_variant: "cuda"`
|
||||
- [ ] Switch to CPU: deletes binary, restarts, health shows `backend_variant: "cpu"`
|
||||
- [ ] Delete CUDA while active: returns 409 error
|
||||
- [ ] Split binary script: `python scripts/split_binary.py` creates manifest + parts + sha256
|
||||
- [ ] Native GPU (macOS MPS): shows info message, no download section
|
||||
@@ -0,0 +1,758 @@
|
||||
# Docker Deployment Guide
|
||||
|
||||
**Status:** In Development for v0.2.0
|
||||
**Requested By:** Reddit community ([thread](https://reddit.com/r/LocalLLaMA/...))
|
||||
|
||||
## Overview
|
||||
|
||||
Docker support makes Voicebox easier to deploy, especially for:
|
||||
|
||||
- **Consistent Environments**: Same setup across dev/staging/prod
|
||||
- **GPU Passthrough**: Easy NVIDIA/AMD GPU access
|
||||
- **Server Deployments**: Run on headless Linux servers
|
||||
- **Multi-User Setups**: Isolate instances per user/team
|
||||
- **Cloud Platforms**: Deploy to AWS, GCP, Azure, DigitalOcean
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Using Pre-Built Images (Recommended)
|
||||
|
||||
```bash
|
||||
# CPU-only version
|
||||
docker run -p 8000:8000 -v voicebox-data:/app/data \
|
||||
ghcr.io/jamiepine/voicebox:latest
|
||||
|
||||
# NVIDIA GPU version
|
||||
docker run --gpus all -p 8000:8000 -v voicebox-data:/app/data \
|
||||
ghcr.io/jamiepine/voicebox:latest-cuda
|
||||
|
||||
# AMD GPU version (experimental)
|
||||
docker run --device=/dev/kfd --device=/dev/dri -p 8000:8000 \
|
||||
-v voicebox-data:/app/data \
|
||||
ghcr.io/jamiepine/voicebox:latest-rocm
|
||||
```
|
||||
|
||||
Then open: `http://localhost:8000`
|
||||
|
||||
### Using Docker Compose (Easiest)
|
||||
|
||||
Create `docker-compose.yml`:
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
voicebox:
|
||||
image: ghcr.io/jamiepine/voicebox:latest-cuda
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- voicebox-data:/app/data
|
||||
- huggingface-cache:/root/.cache/huggingface
|
||||
environment:
|
||||
- GPU_MEMORY_FRACTION=0.8 # Use 80% of GPU memory
|
||||
- TTS_MODE=local
|
||||
- WHISPER_MODE=local
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
|
||||
volumes:
|
||||
voicebox-data:
|
||||
huggingface-cache:
|
||||
```
|
||||
|
||||
Run:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Building From Source
|
||||
|
||||
### Basic Dockerfile
|
||||
|
||||
```dockerfile
|
||||
# Dockerfile
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
git \
|
||||
build-essential \
|
||||
ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy application
|
||||
COPY backend/ /app/backend/
|
||||
COPY requirements.txt /app/
|
||||
|
||||
# Install Python dependencies
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
RUN pip install --no-cache-dir git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /app/data
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8000
|
||||
|
||||
# Run server
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
```
|
||||
|
||||
Build and run:
|
||||
```bash
|
||||
docker build -t voicebox .
|
||||
docker run -p 8000:8000 -v $(pwd)/data:/app/data voicebox
|
||||
```
|
||||
|
||||
### Multi-Stage Build (Optimized)
|
||||
|
||||
Smaller image size by separating build and runtime:
|
||||
|
||||
```dockerfile
|
||||
# Dockerfile.optimized
|
||||
# Stage 1: Build dependencies
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
git build-essential && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/requirements.txt .
|
||||
RUN pip install --no-cache-dir --target=/build/packages \
|
||||
-r requirements.txt
|
||||
|
||||
RUN pip install --no-cache-dir --target=/build/packages \
|
||||
git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
|
||||
# Stage 2: Runtime
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install only runtime dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy installed packages from builder
|
||||
COPY --from=builder /build/packages /usr/local/lib/python3.11/site-packages/
|
||||
|
||||
# Copy application code
|
||||
COPY backend/ /app/backend/
|
||||
|
||||
# Create data directory
|
||||
RUN mkdir -p /app/data
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
```
|
||||
|
||||
Build:
|
||||
```bash
|
||||
docker build -f Dockerfile.optimized -t voicebox:slim .
|
||||
```
|
||||
|
||||
## GPU Support
|
||||
|
||||
### NVIDIA GPUs (CUDA)
|
||||
|
||||
**Dockerfile:**
|
||||
```dockerfile
|
||||
FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04
|
||||
|
||||
# Install Python
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3.11 python3-pip git ffmpeg && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install PyTorch with CUDA support
|
||||
COPY backend/requirements.txt .
|
||||
RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
|
||||
|
||||
# Install other dependencies
|
||||
RUN pip3 install -r requirements.txt
|
||||
RUN pip3 install git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
|
||||
COPY backend/ /app/backend/
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
```
|
||||
|
||||
**Run with GPU:**
|
||||
```bash
|
||||
docker run --gpus all -p 8000:8000 \
|
||||
-v voicebox-data:/app/data \
|
||||
voicebox:cuda
|
||||
```
|
||||
|
||||
**Docker Compose with GPU:**
|
||||
```yaml
|
||||
services:
|
||||
voicebox:
|
||||
image: voicebox:cuda
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
```
|
||||
|
||||
### AMD GPUs (ROCm) - Experimental
|
||||
|
||||
**Dockerfile:**
|
||||
```dockerfile
|
||||
FROM rocm/dev-ubuntu-22.04:6.0
|
||||
|
||||
# Install Python
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3.11 python3-pip git ffmpeg && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install PyTorch with ROCm support
|
||||
COPY backend/requirements.txt .
|
||||
RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0
|
||||
|
||||
# Install other dependencies
|
||||
RUN pip3 install -r requirements.txt
|
||||
RUN pip3 install git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
|
||||
# Set ROCm environment variables
|
||||
ENV HSA_OVERRIDE_GFX_VERSION=10.3.0
|
||||
ENV ROCM_PATH=/opt/rocm
|
||||
|
||||
COPY backend/ /app/backend/
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
```
|
||||
|
||||
**Run with AMD GPU:**
|
||||
```bash
|
||||
docker run --device=/dev/kfd --device=/dev/dri \
|
||||
--group-add video --ipc=host --cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
-p 8000:8000 -v voicebox-data:/app/data \
|
||||
voicebox:rocm
|
||||
```
|
||||
|
||||
**Note:** ROCm support varies by GPU model. Works best on Linux. See [AMD ROCm docs](https://rocm.docs.amd.com) for compatibility.
|
||||
|
||||
## Volume Mounts
|
||||
|
||||
### Essential Volumes
|
||||
|
||||
```bash
|
||||
docker run -v voicebox-data:/app/data \ # Profiles, generations, history
|
||||
-v huggingface-cache:/root/.cache/huggingface \ # Downloaded models
|
||||
-p 8000:8000 voicebox
|
||||
```
|
||||
|
||||
### Development Volume Mounts
|
||||
|
||||
For development with hot-reload:
|
||||
|
||||
```bash
|
||||
docker run -v $(pwd)/backend:/app/backend \ # Live code changes
|
||||
-v voicebox-data:/app/data \
|
||||
-e RELOAD=true \
|
||||
-p 8000:8000 voicebox
|
||||
```
|
||||
|
||||
### Custom Model Storage
|
||||
|
||||
Use external model directory:
|
||||
|
||||
```bash
|
||||
docker run -v /path/to/models:/models \
|
||||
-e MODELS_DIR=/models \
|
||||
-v voicebox-data:/app/data \
|
||||
-p 8000:8000 voicebox
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Configure Voicebox via environment variables:
|
||||
|
||||
```bash
|
||||
docker run -e TTS_MODE=local \
|
||||
-e WHISPER_MODE=openai-api \
|
||||
-e OPENAI_API_KEY=sk-... \
|
||||
-e GPU_MEMORY_FRACTION=0.8 \
|
||||
-e LOG_LEVEL=info \
|
||||
-p 8000:8000 voicebox
|
||||
```
|
||||
|
||||
### Available Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `TTS_MODE` | `local` | TTS provider: `local`, `remote` |
|
||||
| `TTS_REMOTE_URL` | - | URL for remote TTS server |
|
||||
| `WHISPER_MODE` | `local` | Whisper provider: `local`, `openai-api`, `remote` |
|
||||
| `WHISPER_REMOTE_URL` | - | URL for remote Whisper server |
|
||||
| `OPENAI_API_KEY` | - | OpenAI API key (if using OpenAI Whisper) |
|
||||
| `GPU_MEMORY_FRACTION` | `0.9` | Fraction of GPU memory to use (0.0-1.0) |
|
||||
| `DATA_DIR` | `/app/data` | Directory for profiles/generations |
|
||||
| `MODELS_DIR` | `/app/models` | Directory for local models |
|
||||
| `LOG_LEVEL` | `info` | Logging level: `debug`, `info`, `warning`, `error` |
|
||||
| `RELOAD` | `false` | Enable hot-reload for development |
|
||||
|
||||
## Complete Docker Compose Examples
|
||||
|
||||
### Production Deployment
|
||||
|
||||
```yaml
|
||||
# docker-compose.prod.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
voicebox:
|
||||
image: ghcr.io/jamiepine/voicebox:latest-cuda
|
||||
container_name: voicebox
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- voicebox-data:/app/data
|
||||
- huggingface-cache:/root/.cache/huggingface
|
||||
environment:
|
||||
- TTS_MODE=local
|
||||
- WHISPER_MODE=local
|
||||
- GPU_MEMORY_FRACTION=0.8
|
||||
- LOG_LEVEL=info
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
|
||||
volumes:
|
||||
voicebox-data:
|
||||
driver: local
|
||||
huggingface-cache:
|
||||
driver: local
|
||||
```
|
||||
|
||||
Run:
|
||||
```bash
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
### Development Setup
|
||||
|
||||
```yaml
|
||||
# docker-compose.dev.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
voicebox:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./backend:/app/backend:ro
|
||||
- voicebox-data:/app/data
|
||||
- huggingface-cache:/root/.cache/huggingface
|
||||
environment:
|
||||
- RELOAD=true
|
||||
- LOG_LEVEL=debug
|
||||
- TTS_MODE=local
|
||||
command: uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
volumes:
|
||||
voicebox-data:
|
||||
huggingface-cache:
|
||||
```
|
||||
|
||||
### Multi-Service Stack
|
||||
|
||||
Full stack with reverse proxy and monitoring:
|
||||
|
||||
```yaml
|
||||
# docker-compose.stack.yml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# Main Voicebox app
|
||||
voicebox:
|
||||
image: ghcr.io/jamiepine/voicebox:latest-cuda
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- voicebox-data:/app/data
|
||||
- huggingface-cache:/root/.cache/huggingface
|
||||
environment:
|
||||
- TTS_MODE=local
|
||||
- WHISPER_MODE=local
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
|
||||
# Nginx reverse proxy
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./ssl:/etc/nginx/ssl:ro
|
||||
depends_on:
|
||||
- voicebox
|
||||
|
||||
# Prometheus monitoring (optional)
|
||||
prometheus:
|
||||
image: prom/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- prometheus-data:/prometheus
|
||||
|
||||
volumes:
|
||||
voicebox-data:
|
||||
huggingface-cache:
|
||||
prometheus-data:
|
||||
```
|
||||
|
||||
## Cloud Deployment
|
||||
|
||||
### AWS EC2
|
||||
|
||||
1. **Launch GPU Instance** (g4dn.xlarge or p3.2xlarge)
|
||||
2. **Install Docker + nvidia-docker:**
|
||||
```bash
|
||||
# Amazon Linux 2
|
||||
sudo yum install -y docker
|
||||
sudo systemctl start docker
|
||||
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
|
||||
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
|
||||
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \
|
||||
sudo tee /etc/apt/sources.list.d/nvidia-docker.list
|
||||
sudo apt-get update && sudo apt-get install -y nvidia-docker2
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
3. **Deploy:**
|
||||
```bash
|
||||
docker run --gpus all -d -p 80:8000 \
|
||||
-v voicebox-data:/app/data \
|
||||
--restart unless-stopped \
|
||||
ghcr.io/jamiepine/voicebox:latest-cuda
|
||||
```
|
||||
|
||||
### DigitalOcean
|
||||
|
||||
Use GPU Droplet + Docker:
|
||||
|
||||
```bash
|
||||
# Create droplet via CLI
|
||||
doctl compute droplet create voicebox \
|
||||
--size gpu-h100x1-80gb \
|
||||
--image ubuntu-22-04-x64 \
|
||||
--region nyc3
|
||||
|
||||
# SSH and deploy
|
||||
ssh root@<droplet-ip>
|
||||
curl -fsSL https://get.docker.com -o get-docker.sh
|
||||
sh get-docker.sh
|
||||
docker run --gpus all -d -p 80:8000 voicebox:cuda
|
||||
```
|
||||
|
||||
### Google Cloud Run (CPU-only)
|
||||
|
||||
```bash
|
||||
# Build and push
|
||||
docker build -t gcr.io/your-project/voicebox .
|
||||
docker push gcr.io/your-project/voicebox
|
||||
|
||||
# Deploy to Cloud Run
|
||||
gcloud run deploy voicebox \
|
||||
--image gcr.io/your-project/voicebox \
|
||||
--platform managed \
|
||||
--region us-central1 \
|
||||
--memory 4Gi \
|
||||
--cpu 2 \
|
||||
--port 8000
|
||||
```
|
||||
|
||||
### Fly.io
|
||||
|
||||
Create `fly.toml`:
|
||||
```toml
|
||||
app = "voicebox"
|
||||
|
||||
[build]
|
||||
image = "ghcr.io/jamiepine/voicebox:latest"
|
||||
|
||||
[[services]]
|
||||
http_checks = []
|
||||
internal_port = 8000
|
||||
protocol = "tcp"
|
||||
|
||||
[[services.ports]]
|
||||
port = 80
|
||||
handlers = ["http"]
|
||||
|
||||
[[services.ports]]
|
||||
port = 443
|
||||
handlers = ["tls", "http"]
|
||||
|
||||
[mounts]
|
||||
source = "voicebox_data"
|
||||
destination = "/app/data"
|
||||
```
|
||||
|
||||
Deploy:
|
||||
```bash
|
||||
fly launch
|
||||
fly deploy
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### GPU Not Detected
|
||||
|
||||
**Check NVIDIA Docker:**
|
||||
```bash
|
||||
docker run --rm --gpus all nvidia/cuda:12.1.0-base-ubuntu22.04 nvidia-smi
|
||||
```
|
||||
|
||||
If this fails, reinstall nvidia-docker2.
|
||||
|
||||
**Check AMD ROCm:**
|
||||
```bash
|
||||
docker run --rm --device=/dev/kfd --device=/dev/dri rocm/dev-ubuntu-22.04:6.0 rocminfo
|
||||
```
|
||||
|
||||
### Permission Errors
|
||||
|
||||
Container can't write to volumes:
|
||||
```bash
|
||||
# Fix permissions
|
||||
docker run --user $(id -u):$(id -g) -v $(pwd)/data:/app/data voicebox
|
||||
```
|
||||
|
||||
### Out of Memory
|
||||
|
||||
Reduce GPU memory usage:
|
||||
```bash
|
||||
docker run -e GPU_MEMORY_FRACTION=0.5 voicebox
|
||||
```
|
||||
|
||||
Or use CPU-only:
|
||||
```bash
|
||||
docker run -e DEVICE=cpu voicebox
|
||||
```
|
||||
|
||||
### Model Download Fails
|
||||
|
||||
Ensure HuggingFace cache is writable:
|
||||
```bash
|
||||
docker run -v huggingface-cache:/root/.cache/huggingface voicebox
|
||||
```
|
||||
|
||||
Or use host cache:
|
||||
```bash
|
||||
docker run -v ~/.cache/huggingface:/root/.cache/huggingface voicebox
|
||||
```
|
||||
|
||||
### Port Already in Use
|
||||
|
||||
Change host port:
|
||||
```bash
|
||||
docker run -p 8080:8000 voicebox # Use port 8080 instead
|
||||
```
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
### 1. Don't Run as Root
|
||||
|
||||
Create non-root user in Dockerfile:
|
||||
```dockerfile
|
||||
RUN useradd -m -u 1000 voicebox
|
||||
USER voicebox
|
||||
```
|
||||
|
||||
### 2. Use Secrets for API Keys
|
||||
|
||||
Don't put API keys in docker-compose.yml:
|
||||
|
||||
```bash
|
||||
# Use Docker secrets
|
||||
echo "sk-your-key" | docker secret create openai_key -
|
||||
|
||||
docker service create \
|
||||
--secret openai_key \
|
||||
-e OPENAI_API_KEY_FILE=/run/secrets/openai_key \
|
||||
voicebox
|
||||
```
|
||||
|
||||
### 3. Network Isolation
|
||||
|
||||
Use internal networks for multi-container setups:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
voicebox:
|
||||
networks:
|
||||
- internal
|
||||
nginx:
|
||||
networks:
|
||||
- internal
|
||||
- external
|
||||
ports:
|
||||
- "80:80"
|
||||
|
||||
networks:
|
||||
internal:
|
||||
internal: true
|
||||
external:
|
||||
```
|
||||
|
||||
### 4. Resource Limits
|
||||
|
||||
Prevent resource exhaustion:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
voicebox:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 8G
|
||||
reservations:
|
||||
cpus: '2'
|
||||
memory: 4G
|
||||
```
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
### GPU Memory Management
|
||||
|
||||
```bash
|
||||
# Use 80% of GPU (default 90%)
|
||||
docker run -e GPU_MEMORY_FRACTION=0.8 voicebox
|
||||
|
||||
# Allow GPU memory growth (prevents OOM)
|
||||
docker run -e TF_FORCE_GPU_ALLOW_GROWTH=true voicebox
|
||||
```
|
||||
|
||||
### Model Caching
|
||||
|
||||
Pre-download models to volume:
|
||||
|
||||
```bash
|
||||
# Download models first
|
||||
docker run --rm -v huggingface-cache:/root/.cache/huggingface \
|
||||
voicebox python -c "
|
||||
from transformers import WhisperProcessor, WhisperForConditionalGeneration
|
||||
WhisperProcessor.from_pretrained('openai/whisper-base')
|
||||
WhisperForConditionalGeneration.from_pretrained('openai/whisper-base')
|
||||
"
|
||||
|
||||
# Then run normally
|
||||
docker run -v huggingface-cache:/root/.cache/huggingface voicebox
|
||||
```
|
||||
|
||||
### Multi-Worker Setup
|
||||
|
||||
Use uvicorn workers for better throughput:
|
||||
|
||||
```dockerfile
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "4"]
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Health Checks
|
||||
|
||||
Built-in health endpoint:
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
Docker health check:
|
||||
```yaml
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
```
|
||||
|
||||
### Prometheus Metrics
|
||||
|
||||
Add metrics exporter:
|
||||
```python
|
||||
# backend/main.py
|
||||
from prometheus_fastapi_instrumentator import Instrumentator
|
||||
|
||||
Instrumentator().instrument(app).expose(app)
|
||||
```
|
||||
|
||||
Then scrape `/metrics` with Prometheus.
|
||||
|
||||
### Logs
|
||||
|
||||
View container logs:
|
||||
```bash
|
||||
docker logs -f voicebox
|
||||
|
||||
# Or with compose
|
||||
docker compose logs -f voicebox
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- [ ] Publish official images to GitHub Container Registry
|
||||
- [ ] Add Kubernetes Helm charts
|
||||
- [ ] Create Docker Desktop extension
|
||||
- [ ] Add automated vulnerability scanning
|
||||
- [ ] Support ARM64 builds for Raspberry Pi / Apple Silicon
|
||||
|
||||
## Contributing
|
||||
|
||||
Help improve Docker support:
|
||||
1. Test on different platforms (AMD GPU, ARM64, etc.)
|
||||
2. Submit Dockerfile optimizations
|
||||
3. Share deployment configurations
|
||||
4. Report issues: [GitHub Issues](https://github.com/jamiepine/voicebox/issues)
|
||||
|
||||
## Resources
|
||||
|
||||
- [Docker Documentation](https://docs.docker.com)
|
||||
- [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker)
|
||||
- [AMD ROCm Docker](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html)
|
||||
- [Docker Compose Reference](https://docs.docker.com/compose/compose-file/)
|
||||
@@ -0,0 +1,235 @@
|
||||
# OpenAI API Compatibility
|
||||
|
||||
**Status:** Planned for v0.2.0
|
||||
|
||||
**Issue:** [#10 OpenAI API compatibility](https://github.com/jamiepine/voicebox/issues/10)
|
||||
|
||||
## Overview
|
||||
|
||||
This feature exposes OpenAI-compatible endpoints from Voicebox, allowing any tool, library, or application that speaks the OpenAI Audio API to use Voicebox as a drop-in local replacement.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph clients [External Clients]
|
||||
SDK[OpenAI SDK]
|
||||
Curl[curl / HTTP]
|
||||
Apps[Third-party Apps]
|
||||
end
|
||||
|
||||
subgraph voicebox [Voicebox Server]
|
||||
OpenAI["/v1/audio/* endpoints"]
|
||||
TTS[TTSModel]
|
||||
Whisper[WhisperModel]
|
||||
Profiles[Voice Profiles]
|
||||
end
|
||||
|
||||
SDK --> OpenAI
|
||||
Curl --> OpenAI
|
||||
Apps --> OpenAI
|
||||
OpenAI --> TTS
|
||||
OpenAI --> Whisper
|
||||
OpenAI --> Profiles
|
||||
```
|
||||
|
||||
## Use Cases
|
||||
|
||||
- **OpenAI SDK users**: `openai.audio.speech.create()` works with Voicebox
|
||||
- **LLM frameworks**: LangChain, AutoGen, etc. can use Voicebox for TTS
|
||||
- **Shell scripts**: `curl` commands copy-pasted from OpenAI docs work
|
||||
- **Existing integrations**: Any tool expecting OpenAI's API works without code changes
|
||||
|
||||
## Endpoints to Implement
|
||||
|
||||
### 1. `POST /v1/audio/speech` (TTS)
|
||||
|
||||
OpenAI spec: https://platform.openai.com/docs/api-reference/audio/createSpeech
|
||||
|
||||
**Request:**
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "tts-1",
|
||||
"input": "Hello world!",
|
||||
"voice": "alloy",
|
||||
"response_format": "mp3",
|
||||
"speed": 1.0
|
||||
}
|
||||
```
|
||||
|
||||
**Response:** Audio file (mp3, wav, opus, aac, flac, pcm)
|
||||
|
||||
**Voice Mapping Strategy:**
|
||||
|
||||
- `voice` parameter maps to Voicebox profile names (case-insensitive)
|
||||
- If no match, use a configurable default profile
|
||||
- Support special syntax: `voice: "profile:uuid"` for explicit profile ID
|
||||
|
||||
### 2. `POST /v1/audio/transcriptions` (Whisper)
|
||||
|
||||
OpenAI spec: https://platform.openai.com/docs/api-reference/audio/createTranscription
|
||||
|
||||
**Request:** (multipart/form-data)
|
||||
|
||||
- `file`: Audio file
|
||||
- `model`: "whisper-1"
|
||||
- `language`: Optional language hint
|
||||
- `response_format`: json, text, srt, verbose_json, vtt
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"text": "Hello world!"
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### New File: `backend/openai_compat.py`
|
||||
|
||||
Create a dedicated module with an APIRouter for OpenAI-compatible endpoints:
|
||||
|
||||
```python
|
||||
from fastapi import APIRouter, UploadFile, File, Form, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from typing import Literal, Optional
|
||||
|
||||
router = APIRouter(prefix="/v1/audio", tags=["OpenAI Compatible"])
|
||||
|
||||
class SpeechRequest(BaseModel):
|
||||
model: str = "tts-1"
|
||||
input: str
|
||||
voice: str = "alloy"
|
||||
response_format: Literal["mp3", "wav", "opus", "aac", "flac", "pcm"] = "mp3"
|
||||
speed: float = 1.0
|
||||
|
||||
@router.post("/speech")
|
||||
async def create_speech(request: SpeechRequest, db: Session = Depends(get_db)):
|
||||
# 1. Map voice name to profile
|
||||
# 2. Generate audio using existing TTSModel
|
||||
# 3. Convert to requested format
|
||||
# 4. Return audio stream
|
||||
...
|
||||
|
||||
@router.post("/transcriptions")
|
||||
async def create_transcription(
|
||||
file: UploadFile = File(...),
|
||||
model: str = Form("whisper-1"),
|
||||
language: Optional[str] = Form(None),
|
||||
response_format: str = Form("json"),
|
||||
):
|
||||
# 1. Save uploaded file
|
||||
# 2. Transcribe using existing WhisperModel
|
||||
# 3. Return in requested format
|
||||
...
|
||||
```
|
||||
|
||||
### Voice Profile Resolution
|
||||
|
||||
Add helper in [backend/profiles.py](backend/profiles.py):
|
||||
|
||||
```python
|
||||
async def resolve_voice_for_openai(voice: str, db: Session) -> Optional[VoiceProfile]:
|
||||
"""
|
||||
Resolve OpenAI voice parameter to a Voicebox profile.
|
||||
|
||||
Priority:
|
||||
1. Exact profile name match (case-insensitive)
|
||||
2. Profile ID match (if voice starts with "profile:")
|
||||
3. Default profile from config
|
||||
4. First available profile
|
||||
"""
|
||||
...
|
||||
```
|
||||
|
||||
### Audio Format Conversion
|
||||
|
||||
Add conversion utilities in [backend/utils/audio.py](backend/utils/audio.py):
|
||||
|
||||
```python
|
||||
def convert_audio_format(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int,
|
||||
target_format: str, # mp3, wav, opus, aac, flac, pcm
|
||||
) -> bytes:
|
||||
"""Convert audio to target format using ffmpeg or pydub."""
|
||||
...
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
Add to [backend/config.py](backend/config.py):
|
||||
|
||||
```python
|
||||
# OpenAI API Compatibility
|
||||
OPENAI_COMPAT_ENABLED = True
|
||||
OPENAI_COMPAT_DEFAULT_VOICE = None # Profile ID or name for default voice
|
||||
OPENAI_COMPAT_REQUIRE_AUTH = False # Require API key validation
|
||||
OPENAI_COMPAT_API_KEY = None # If set, validate against this
|
||||
```
|
||||
|
||||
### Integration with main.py
|
||||
|
||||
In [backend/main.py](backend/main.py), include the router:
|
||||
|
||||
```python
|
||||
from . import openai_compat
|
||||
|
||||
# Add OpenAI-compatible routes
|
||||
if config.OPENAI_COMPAT_ENABLED:
|
||||
app.include_router(openai_compat.router)
|
||||
```
|
||||
|
||||
## Streaming Support (Future Enhancement)
|
||||
|
||||
Initial implementation returns complete audio. Streaming can be added later:
|
||||
|
||||
```python
|
||||
@router.post("/speech")
|
||||
async def create_speech(request: SpeechRequest):
|
||||
if request.stream:
|
||||
return StreamingResponse(
|
||||
generate_audio_chunks(request),
|
||||
media_type=f"audio/{request.response_format}"
|
||||
)
|
||||
...
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
Example usage after implementation:
|
||||
|
||||
```bash
|
||||
# TTS with curl
|
||||
curl http://localhost:8000/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "tts-1", "input": "Hello!", "voice": "MyProfile"}' \
|
||||
--output speech.mp3
|
||||
|
||||
# With OpenAI Python SDK
|
||||
from openai import OpenAI
|
||||
client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused")
|
||||
response = client.audio.speech.create(
|
||||
model="tts-1",
|
||||
voice="MyProfile",
|
||||
input="Hello world!"
|
||||
)
|
||||
response.stream_to_file("output.mp3")
|
||||
|
||||
# Transcription
|
||||
curl http://localhost:8000/v1/audio/transcriptions \
|
||||
-F file=@audio.mp3 \
|
||||
-F model="whisper-1"
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- Optional API key validation (for shared deployments)
|
||||
- Rate limiting on endpoints
|
||||
- Input length limits (same as existing `/generate` endpoint)
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `pydub` or `ffmpeg-python` for audio format conversion (mp3, opus, etc.)
|
||||
- No changes to existing TTS/Whisper model code
|
||||
@@ -0,0 +1,497 @@
|
||||
# Voicebox Project Status & Roadmap
|
||||
|
||||
> Last updated: 2026-03-13 | Current version: **v0.1.13** | 13.1k stars | ~176 open issues | 25 open PRs
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Architecture Overview](#architecture-overview)
|
||||
2. [Current State](#current-state)
|
||||
3. [Open PRs — Triage & Analysis](#open-prs--triage--analysis)
|
||||
4. [Open Issues — Categorized](#open-issues--categorized)
|
||||
5. [Existing Plan Documents — Status](#existing-plan-documents--status)
|
||||
6. [New Model Integration — Landscape](#new-model-integration--landscape)
|
||||
7. [Architectural Bottlenecks](#architectural-bottlenecks)
|
||||
8. [Recommended Priorities](#recommended-priorities)
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ Tauri Shell (Rust) │
|
||||
│ ┌───────────────────────────────────────────────┐ │
|
||||
│ │ React Frontend (app/) │ │
|
||||
│ │ Zustand stores · API client · Generation UI │ │
|
||||
│ │ Stories Editor · Voice Profiles · Model Mgmt │ │
|
||||
│ └──────────────────────┬────────────────────────┘ │
|
||||
│ │ HTTP :17493 │
|
||||
│ ┌──────────────────────▼────────────────────────┐ │
|
||||
│ │ FastAPI Backend (backend/) │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ TTSBackend Protocol │ │ │
|
||||
│ │ │ ┌──────────┐ ┌───────┐ ┌───────────┐ │ │ │
|
||||
│ │ │ │ Qwen3-TTS│ │LuxTTS │ │Chatterbox │ │ │ │
|
||||
│ │ │ │(Py/MLX) │ │ │ │(MTL+Turbo)│ │ │ │
|
||||
│ │ │ └──────────┘ └───────┘ └───────────┘ │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ ┌───────────┐ ┌─────────┐ │ │
|
||||
│ │ │ STTBackend│ │ Profiles│ │ │
|
||||
│ │ │ (Whisper) │ │ History │ │ │
|
||||
│ │ └───────────┘ │ Stories │ │ │
|
||||
│ │ └─────────┘ │ │
|
||||
│ └───────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key Files
|
||||
|
||||
| Layer | File | Purpose |
|
||||
|-------|------|---------|
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2850 lines) |
|
||||
| TTS protocol | `backend/backends/__init__.py:32-101` | `TTSBackend` Protocol definition |
|
||||
| Model registry | `backend/backends/__init__.py:17-29,153-366` | `ModelConfig` dataclass + registry helpers |
|
||||
| TTS factory | `backend/backends/__init__.py:382-426` | Thread-safe engine registry (double-checked locking) |
|
||||
| PyTorch TTS | `backend/backends/pytorch_backend.py` | Qwen3-TTS via `qwen_tts` package |
|
||||
| MLX TTS | `backend/backends/mlx_backend.py` | Qwen3-TTS via `mlx_audio.tts` |
|
||||
| LuxTTS | `backend/backends/luxtts_backend.py` | LuxTTS — fast, CPU-friendly |
|
||||
| Chatterbox MTL | `backend/backends/chatterbox_backend.py` | Chatterbox Multilingual — 23 languages |
|
||||
| Chatterbox Turbo | `backend/backends/chatterbox_turbo_backend.py` | Chatterbox Turbo — English, paralinguistic tags |
|
||||
| Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch |
|
||||
| API types | `backend/models.py` | Pydantic request/response models |
|
||||
| HF progress | `backend/utils/hf_progress.py` | HFProgressTracker (tqdm patching for download progress) |
|
||||
| Audio utils | `backend/utils/audio.py` | `trim_tts_output()`, normalize, load/save audio |
|
||||
| Frontend API | `app/src/lib/api/client.ts` | Hand-written fetch wrapper |
|
||||
| Frontend types | `app/src/lib/api/types.ts` | TypeScript API types |
|
||||
| Engine selector | `app/src/components/Generation/EngineModelSelector.tsx` | Shared engine/model dropdown |
|
||||
| Generation form | `app/src/components/Generation/GenerationForm.tsx` | TTS generation UI |
|
||||
| Floating gen box | `app/src/components/Generation/FloatingGenerateBox.tsx` | Compact generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status/progress UI |
|
||||
| GPU acceleration | `app/src/components/ServerSettings/GpuAcceleration.tsx` | CUDA backend swap UI |
|
||||
| Gen form hook | `app/src/lib/hooks/useGenerationForm.ts` | Form validation + submission |
|
||||
| Language constants | `app/src/lib/constants/languages.ts` | Per-engine language maps |
|
||||
|
||||
### How TTS Generation Works (Current Flow)
|
||||
|
||||
```
|
||||
POST /generate
|
||||
1. Look up voice profile from DB
|
||||
2. Resolve engine from request (qwen | luxtts | chatterbox | chatterbox_turbo)
|
||||
3. Get backend: get_tts_backend_for_engine(engine) # thread-safe singleton per engine
|
||||
4. Check model cache → if missing, trigger background download, return HTTP 202
|
||||
5. Load model (lazy): tts_backend.load_model(model_size)
|
||||
6. Create voice prompt: profiles.create_voice_prompt_for_profile(engine=engine)
|
||||
→ tts_backend.create_voice_prompt(audio_path, reference_text)
|
||||
7. Generate: tts_backend.generate(text, voice_prompt, language, seed, instruct)
|
||||
8. Post-process: trim_tts_output() for Chatterbox engines
|
||||
9. Save WAV → data/generations/{id}.wav
|
||||
10. Insert history record in SQLite
|
||||
11. Return GenerationResponse
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Current State
|
||||
|
||||
### What's Shipped (v0.1.13 + recent merges)
|
||||
|
||||
**Core TTS:**
|
||||
- Qwen3-TTS voice cloning (1.7B and 0.6B models)
|
||||
- MLX backend for Apple Silicon, PyTorch for everything else
|
||||
- Multi-engine TTS architecture with thread-safe backend registry (PR #254)
|
||||
- LuxTTS integration — fast, CPU-friendly English TTS (PR #254)
|
||||
- Chatterbox Multilingual TTS — 23 languages including Hebrew (PR #257)
|
||||
- Instruct parameter UI exists but is non-functional across all backends (see #224, Known Limitations)
|
||||
- Single flat model dropdown (Qwen 1.7B, Qwen 0.6B, LuxTTS, Chatterbox, Chatterbox Turbo)
|
||||
- Centralized model config registry (`ModelConfig` dataclass) — no per-engine dispatch maps in `main.py`
|
||||
- Shared `EngineModelSelector` component — engine/model dropdown defined once, used in both generation forms
|
||||
|
||||
**Infrastructure:**
|
||||
- CUDA backend swap via binary download and restart (PR #252)
|
||||
- GPU acceleration settings UI
|
||||
- Voice profiles with multi-sample support
|
||||
- Stories editor (multi-track DAW timeline)
|
||||
- Whisper transcription (base, small, medium, large variants)
|
||||
- Model management UI with inline download progress bars (HFProgressTracker)
|
||||
- Download cancel/clear UI with error panel (PR #238)
|
||||
- Generation history with caching
|
||||
- Streaming generation endpoint (MLX only)
|
||||
- Duplicate profile name validation (PR #175)
|
||||
- Linux NVIDIA GBM buffer + WebKitGTK microphone fix (PR #210)
|
||||
|
||||
### What's In-Flight
|
||||
|
||||
| Feature | Branch/PR | Status |
|
||||
|---------|-----------|--------|
|
||||
| Chatterbox Turbo + per-engine language lists | `feat/chatterbox-turbo` / PR #258 | Open, ready for review |
|
||||
|
||||
### TTS Engine Comparison
|
||||
|
||||
| Engine | Model Name | Languages | Size | Key Features | Instruct Support |
|
||||
|--------|-----------|-----------|------|-------------|-----------------|
|
||||
| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Highest quality, voice cloning | None (Base model has no instruct path) |
|
||||
| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster | None |
|
||||
| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast | None |
|
||||
| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual | Partial — `exaggeration` float (0-1) for expressiveness |
|
||||
| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency | Partial — inline tags only, no separate instruct param |
|
||||
|
||||
### Multi-Engine Architecture (Shipped)
|
||||
|
||||
The singleton TTS backend blocker described in the previous version of this doc has been **resolved**. The architecture now supports:
|
||||
|
||||
- **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking
|
||||
- **Per-engine backend instances** — each engine gets its own singleton, loaded lazily
|
||||
- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo'`
|
||||
- **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages
|
||||
- **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend
|
||||
- **Trim post-processing** — `trim_tts_output()` for Chatterbox engines (cuts trailing silence/hallucination)
|
||||
|
||||
### Known Limitations
|
||||
|
||||
- **HF XET progress**: Large files downloaded via `hf-xet` (HuggingFace's new transfer backend) report `n=0` in tqdm updates. Progress bars may appear stuck for large `.safetensors` files even though the download is proceeding. This is a known upstream limitation.
|
||||
- **Chatterbox Turbo upstream token bug**: `from_pretrained()` passes `token=os.getenv("HF_TOKEN") or True` which fails without a stored HF token. Our backend works around this by calling `snapshot_download(token=None)` + `from_local()`.
|
||||
- **chatterbox-tts must install with `--no-deps`**: It pins `numpy<1.26`, `torch==2.6.0`, `transformers==4.46.3` — all incompatible with our stack (Python 3.12, torch 2.10, transformers 4.57.3). Sub-deps listed explicitly in `requirements.txt`.
|
||||
- **Instruct parameter is non-functional** (#224): The UI exposes an instruct text field, but it's silently dropped by every backend. The Qwen3-TTS Base model we ship only supports voice cloning — instruct requires the separate CustomVoice model variant (`Qwen3-TTS-12Hz-1.7B-CustomVoice`), which uses predefined speakers instead of ref audio. The instruct UI should be hidden until a backend with real support is integrated.
|
||||
- **Streaming generation** only works for Qwen on MLX. Other engines use the non-streaming `/generate` endpoint.
|
||||
- **dicta-onnx** (Hebrew diacritization) not included — upstream Chatterbox bug requires `model_path` arg but calls `Dicta()` with none. Hebrew works fine without it.
|
||||
|
||||
---
|
||||
|
||||
## Open PRs — Triage & Analysis
|
||||
|
||||
### Recently Merged (Since Last Update)
|
||||
|
||||
| PR | Title | Merged |
|
||||
|----|-------|--------|
|
||||
| **#257** | feat: Chatterbox TTS engine with multilingual voice cloning | 2026-03-13 |
|
||||
| **#254** | feat: LuxTTS integration — multi-engine TTS support | 2026-03-13 |
|
||||
| **#252** | feat: CUDA backend swap via binary download and restart | 2026-03-13 |
|
||||
| **#238** | Download cancel/clear UI, fixed model downloading | 2026-03-13 |
|
||||
| **#250** | docs: align local API port examples | 2026-03-13 |
|
||||
| **#210** | fix: Linux NVIDIA GBM buffer crash | 2026-03-13 |
|
||||
| **#175** | Fix #134: duplicate profile name validation | 2026-03-13 |
|
||||
|
||||
### In-Flight (Our Work)
|
||||
|
||||
| PR | Title | Status | Notes |
|
||||
|----|-------|--------|-------|
|
||||
| **#258** | feat: Chatterbox Turbo engine + per-engine language lists | Open | Ready for review. Adds Turbo engine + dynamic language dropdown. |
|
||||
|
||||
### Merge-Ready / Near-Ready (Bug Fixes & Small Features)
|
||||
|
||||
| PR | Title | Risk | Notes |
|
||||
|----|-------|------|-------|
|
||||
| **#230** | docs: fix README grammar | None | Docs-only |
|
||||
| **#243** | a11y: screen reader and keyboard improvements | Low | Accessibility, no backend changes |
|
||||
| **#178** | Fix #168 #140: generation error handling | Low | Error handling improvements |
|
||||
| **#152** | Fix: prevent crashes when HuggingFace unreachable | Medium | Monkey-patches HF hub; solves real offline bug (#150, #151) |
|
||||
| **#218** | fix: unify qwen tts cache dir on Windows | Low | Windows-specific path fix |
|
||||
| **#214** | fix: panic on launch from tokio::spawn | Low | Rust-side Tauri fix |
|
||||
| **#88** | security: restrict CORS to known local origins | Low | Security hardening |
|
||||
| **#133** | feat: network access toggle | Low | Wires up existing plumbing |
|
||||
|
||||
### Significant Feature PRs
|
||||
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#253** | Enhance speech tokenizer with 48kHz version | Medium | Qwen tokenizer upgrade |
|
||||
| **#97** | fix: pass language parameter to TTS models | Medium | May be partially obsoleted by multi-engine work — needs review |
|
||||
| **#99** | feat: chunked TTS with quality selector | Medium | Solves 500-char limit. Addresses #191, #203, #69, #111. |
|
||||
| **#154** | feat: Audiobook tab | Medium | Full audiobook workflow. Depends on #99 concepts. |
|
||||
| **#91** | fix: CoreAudio device enumeration | Medium | macOS audio device handling |
|
||||
|
||||
### Architectural PRs (Need Careful Review)
|
||||
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#225** | feat: custom HuggingFace model support | High | Arbitrary HF repo loading. May need rework given multi-engine arch is now shipped. |
|
||||
| **#194** | feat: Hebrew + Chatterbox TTS | High | **Superseded** by PR #257 which shipped Chatterbox multilingual (23 langs incl. Hebrew). May be closeable. |
|
||||
| **#195** | feat: per-profile LoRA fine-tuning | Very High | Training pipeline, adapter management, 15 new endpoints. Depends on #194 (now superseded). |
|
||||
| **#161** | feat: Docker + web deployment | High | 3-stage Dockerfile, SPA serving. Independent of TTS engine work. |
|
||||
| **#124** / **#123** | Docker (simpler attempts) | Low-Medium | Overlap with #161 |
|
||||
| **#227** | fix: harden input validation & file safety | Medium | Coupled to #225 (custom models) |
|
||||
|
||||
### PRs That Need Author Action / Are Stale
|
||||
|
||||
| PR | Title | Notes |
|
||||
|----|-------|-------|
|
||||
| **#237** | fix: bundle qwen_tts source files in PyInstaller | Build system, needs review |
|
||||
| **#215** | Update prerequisites with Tauri deps | Branch is `main` — will have conflicts |
|
||||
| **#89** | Linux Support | Branch is `main` — will have conflicts. Broad scope. |
|
||||
| **#83** | Update download links for v0.1.12 | Outdated (we're on v0.1.13) |
|
||||
|
||||
### PRs Likely Superseded
|
||||
|
||||
| PR | Superseded By | Notes |
|
||||
|----|--------------|-------|
|
||||
| **#194** (Hebrew + Chatterbox) | PR #257 (merged) | #257 ships Chatterbox multilingual with 23 languages including Hebrew. #194 took a different approach (route by language). Can likely be closed. |
|
||||
| **#33** (External provider binaries) | PR #252 (merged) | #252 shipped CUDA backend swap. #33's broader provider architecture may still have value but needs reassessment. |
|
||||
|
||||
---
|
||||
|
||||
## Open Issues — Categorized
|
||||
|
||||
### GPU / Hardware Detection (19 issues)
|
||||
|
||||
The single most reported category. Users on Windows with NVIDIA GPUs frequently report "GPU not detected."
|
||||
|
||||
**Root causes (likely):**
|
||||
- PyInstaller binary doesn't bundle CUDA correctly → falls back to CPU
|
||||
- DirectML/Vulkan path not implemented (AMD on Windows)
|
||||
- Binary size limit means CUDA can't ship in the main release
|
||||
|
||||
**Key issues:** #239, #222, #220, #217, #208, #198, #192, #167, #164, #141, #130, #127
|
||||
|
||||
**Fix path:** PR #252 (CUDA backend swap) is now merged. Users can download the CUDA binary separately from the GPU acceleration settings. Many of these issues may now be resolvable — needs triage to confirm.
|
||||
|
||||
### Model Downloads (20 issues)
|
||||
|
||||
Second most reported. Users get stuck downloads, can't resume, no offline fallback.
|
||||
|
||||
**Key issues:** #249, #240, #221, #216, #212, #181, #180, #159, #150, #149, #145, #143, #135, #134
|
||||
|
||||
**Fix path:** PR #238 (cancel/clear UI) is now merged. PR #152 (offline crash fix) still open. Inline progress bars now show for all engines. Resume support not yet addressed.
|
||||
|
||||
### Language Requests (18 issues)
|
||||
|
||||
Strong demand for: Hindi (#245), Indonesian (#247), Dutch (#236), Hebrew (#199), Greek (#188), Portuguese (#183), Persian (#162), and many more.
|
||||
|
||||
**Key issues:** #247, #245, #236, #211, #205, #199, #189, #188, #187, #183, #179, #162
|
||||
|
||||
**Fix path:** Chatterbox Multilingual (merged via #257) now supports 23 languages including many of the requested ones: Arabic, Danish, German, Greek, Finnish, Hebrew, Hindi, Dutch, Norwegian, Polish, Swedish, Swahili, Turkish. Per-engine language filtering (PR #258) ensures the UI shows correct options. Several of these issues may be closeable.
|
||||
|
||||
### New Model Requests (5 explicit issues)
|
||||
|
||||
| Issue | Model Requested |
|
||||
|-------|----------------|
|
||||
| #226 | GGUF support |
|
||||
| #172 | VibeVoice |
|
||||
| #138 | Export to ONNX/Piper format |
|
||||
| #132 | LavaSR (transcription) |
|
||||
| #76 | (General model expansion) |
|
||||
|
||||
Community also requests: XTTS-v2, Fish Speech, CosyVoice, Kokoro. The multi-engine architecture is now in place, making new model integration significantly easier.
|
||||
|
||||
### Long-Form / Chunking (5 issues)
|
||||
|
||||
Users hitting the ~500 character practical limit.
|
||||
|
||||
**Key issues:** #234 (queue system), #203 (500 char limit), #191 (auto-split), #111, #69
|
||||
|
||||
**Fix path:** PR #99 (chunked TTS + quality selector) directly addresses this. PR #154 (Audiobook tab) builds on it.
|
||||
|
||||
### Feature Requests (23 issues)
|
||||
|
||||
Notable requests:
|
||||
- **#234** — Queue system for batch generation
|
||||
- **#182** — Concurrent/multi-thread generation
|
||||
- **#173** — Vocal intonation/inflection control
|
||||
- **#165** — Audiobook mode
|
||||
- **#144** — Copy text to clipboard
|
||||
- **#184** — Cancel button for progress bar
|
||||
- **#242** — Seed value pinning for consistency
|
||||
- **#228** — Always use 0.6B option
|
||||
- **#233** — Transcribe audio API improvements
|
||||
- **#235** — Finetuned Qwen3-TTS tokenizer
|
||||
|
||||
### Bugs (19 issues)
|
||||
|
||||
| Category | Issues |
|
||||
|----------|--------|
|
||||
| Generation failures | #248 (broken pipe), #219 (unsupported scalarType), #202 (clipping error), #170 (load failed) |
|
||||
| UI bugs | #231 (history not updating), #190 (mobile landing), #169 (blank interface) |
|
||||
| File operations | #207 (transcribe file error), #168 (no such file), #142 (download audio fail) |
|
||||
| Server lifecycle | #166 (server processes remain), #164 (no auto-update) |
|
||||
| Database | #174 (sqlite3 IntegrityError) |
|
||||
| Dependency | #131 (numpy ABI mismatch), #209 (import error) |
|
||||
|
||||
---
|
||||
|
||||
## Existing Plan Documents — Status
|
||||
|
||||
| Document | Target Version | Status | Relevance |
|
||||
|----------|---------------|--------|-----------|
|
||||
| `TTS_PROVIDER_ARCHITECTURE.md` | v0.1.13 | **Partially superseded** by multi-engine arch + CUDA swap | Core concepts implemented differently than planned |
|
||||
| `CUDA_BACKEND_SWAP.md` | — | **Shipped** (PR #252) | CUDA binary download + backend restart |
|
||||
| `CUDA_BACKEND_SWAP_FINAL.md` | — | **Shipped** (PR #252) | Final implementation plan |
|
||||
| `EXTERNAL_PROVIDERS.md` | v0.2.0 | **Not started** | Remote server support |
|
||||
| `MLX_AUDIO.md` | — | **Shipped** | MLX backend is live |
|
||||
| `DOCKER_DEPLOYMENT.md` | v0.2.0 | **PR exists** (#161) | Waiting on review |
|
||||
| `OPENAI_SUPPORT.md` | v0.2.0 | **Not started** | OpenAI-compatible API layer |
|
||||
| `PR33_CUDA_PROVIDER_REVIEW.md` | — | **Reference** | Analysis of the original provider approach |
|
||||
|
||||
---
|
||||
|
||||
## New Model Integration — Landscape
|
||||
|
||||
### Models Worth Supporting (2026 SOTA — updated March 13)
|
||||
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Instruct Support | Integration Ease | Status |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|-----------------|--------|
|
||||
| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | None (Base); Yes (CustomVoice variant, predefined speakers only) | **Shipped** | v0.1.13 |
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | None | **Shipped** | PR #254 |
|
||||
| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | Partial — `exaggeration` float | **Shipped** | PR #257 |
|
||||
| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | Partial — inline tags only | **PR #258** | In review |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | **Yes** — `inference_instruct2()`, works with cloning | Ready | Best instruct candidate |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | **Yes** — inline text descriptions, word-level control | Ready | Multi-engine arch in place |
|
||||
| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | **Yes** — text prompts for style + timbre design | Needs vetting | Apache 2.0, multi-speaker dialogue |
|
||||
| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Partial — automatic prosody from text context | Needs vetting | MIT, 700s+ coherent, synced transcript output |
|
||||
| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Partial — automatic context-aware prosody | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Partial — automatic style inference | Ready | Apache 2.0, multi-engine arch in place |
|
||||
| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Partial — style transfer from ref audio only | Ready | Multi-engine arch in place |
|
||||
| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | None | Needs vetting | MIT, Kyutai Labs, no GPU required |
|
||||
|
||||
#### Notes on New Candidates (March 2026)
|
||||
|
||||
- **CosyVoice2-0.5B** — Best candidate for instruct support. `inference_instruct2()` accepts a text instruct parameter for emotions, speed, volume, dialects — and it works alongside voice cloning. This is the closest match to what users expect from our instruct UI. [HF: FunAudioLLM/CosyVoice2-0.5B](https://huggingface.co/FunAudioLLM/CosyVoice2-0.5B)
|
||||
- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. Prosody/emotion is automatic from text context, not user-controllable. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada)
|
||||
- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions). VoiceGenerator unifies timbre design and style control via text prompts, usable as a layer for downstream TTS including cloning. [HF: OpenMOSS-Team/MOSS-VoiceGenerator](https://huggingface.co/OpenMOSS-Team/MOSS-VoiceGenerator) | [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS)
|
||||
- **Fish Speech** — Word-level fine-grained control using plain language descriptions inline in the script. Works with cloning. Note: Fish Audio S2 has a restrictive research license (commercial use requires approval), but the open-source Fish Speech model may differ. Needs license clarification. [fish.audio blog](https://fish.audio/blog/fish-audio-s2-fine-grained-ai-voice-control-at-the-word-level)
|
||||
- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Prosody/emotion is context-aware but automatic, not explicitly controllable via text prompt. Real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM)
|
||||
- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. No style control. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts)
|
||||
- **Watch list:** MioTTS-2.6B (fast LLM-based EN/JP, vLLM compatible), Oolel-Voices (Soynade Research, expressive modular control)
|
||||
|
||||
### Adding a New Engine (Now Straightforward)
|
||||
|
||||
With the model config registry and shared `EngineModelSelector` component, adding a new TTS engine requires:
|
||||
|
||||
1. **Create `backend/backends/<engine>_backend.py`** — implement `TTSBackend` protocol (~200-300 lines)
|
||||
2. **Register in `backend/backends/__init__.py`** — add `ModelConfig` entry + `TTS_ENGINES` entry + factory elif
|
||||
3. **Update `backend/models.py`** — add engine name to regex
|
||||
4. **Update frontend** — add to engine union type, `EngineModelSelector` options, form schema, language map (4 files)
|
||||
|
||||
`main.py` requires **zero changes** — the registry handles all dispatch automatically.
|
||||
|
||||
Total effort: **~1 day** for a well-documented model with a PyPI package. See `docs/plans/ADDING_TTS_ENGINES.md` for the full guide.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Bottlenecks
|
||||
|
||||
### ~~1. Single Backend Singleton~~ — RESOLVED
|
||||
|
||||
The singleton TTS backend was replaced with a thread-safe per-engine registry in PR #254. Multiple engines can now be loaded simultaneously.
|
||||
|
||||
### ~~2. `main.py` Dispatch Point Duplication~~ — RESOLVED
|
||||
|
||||
Previously, each engine required updates to 6+ hardcoded dispatch maps across `main.py` (~320 lines of if/elif chains). A model config registry in `backend/backends/__init__.py` now centralizes all model metadata (`ModelConfig` dataclass) with helper functions (`load_engine_model()`, `check_model_loaded()`, `engine_needs_trim()`, etc.). Adding a new engine requires zero changes to `main.py`.
|
||||
|
||||
### ~~3. Model Config is Scattered~~ — RESOLVED
|
||||
|
||||
Model identifiers, HF repo IDs, display names, and engine metadata are now consolidated in the `ModelConfig` registry. Backend-aware branching (e.g. MLX vs PyTorch Qwen repo IDs) happens inside the registry. Frontend model options are centralized in `EngineModelSelector.tsx`.
|
||||
|
||||
### 4. Voice Prompt Cache Assumes PyTorch Tensors
|
||||
|
||||
`backend/utils/cache.py` uses `torch.save()` / `torch.load()`. LuxTTS and Chatterbox backends work around this by storing reference audio paths instead of tensors in their voice prompt dicts. Not ideal but functional.
|
||||
|
||||
### 5. ~~Frontend Assumes Qwen Model Sizes~~ — RESOLVED
|
||||
|
||||
The generation form now uses a flat model dropdown with engine-based routing. Per-engine language filtering is in place. Model size is only sent for Qwen.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Priorities
|
||||
|
||||
### Tier 1 — Ship Now (Low Risk)
|
||||
|
||||
| Priority | PR/Item | Impact | Effort |
|
||||
|----------|---------|--------|--------|
|
||||
| 1 | **#258** — Chatterbox Turbo + per-engine languages | Paralinguistic tags, proper language filtering | Review only |
|
||||
| 2 | **#152** — Offline mode crash fix | Fixes #150, #151 | Low |
|
||||
| 3 | **#99** — Chunked TTS + quality selector | Removes 500-char limit, addresses 5 issues | Medium |
|
||||
| 4 | **#218** — Windows HF cache dir fix | Windows-specific pain | Low |
|
||||
| 5 | **#178** — Generation error handling | Error UX | Low |
|
||||
| 6 | **#230** — Docs fixes | Zero risk | None |
|
||||
| 7 | **#133** — Network access toggle | Wires up existing code | Low |
|
||||
| 8 | **#88** — CORS restriction | Security improvement | Low |
|
||||
| 9 | **#214** — Tauri window close panic fix | Stability | Low |
|
||||
| 10 | Triage GPU issues | Many may be resolved by CUDA swap (#252) | Low |
|
||||
| 11 | Close superseded PRs | #194 (superseded by #257), #83 (outdated) | None |
|
||||
|
||||
### Tier 2 — Next Release (v0.2.0)
|
||||
|
||||
| Priority | Item | Impact | Effort |
|
||||
|----------|------|--------|--------|
|
||||
| 1 | **#253** — 48kHz speech tokenizer | Quality improvement | Medium |
|
||||
| 2 | **#161** — Docker deployment | Server/headless users | Medium |
|
||||
| 3 | **#154** — Audiobook tab | Long-form users | Medium |
|
||||
| 4 | ~~**Model config registry**~~ | ~~Reduce dispatch duplication in main.py~~ | **Done** |
|
||||
| 5 | **#225** — Custom HuggingFace models | User-supplied models | High (needs rework for multi-engine) |
|
||||
|
||||
### Tier 3 — Future (v0.3.0+)
|
||||
|
||||
| Priority | Item | Notes |
|
||||
|----------|------|-------|
|
||||
| 1 | **HumeAI TADA** | Long-form reliability for Stories, synced transcripts. Addresses #234, #203, #191, #111, #69. Needs API vetting. |
|
||||
| 2 | **Pocket TTS** (Kyutai) | CPU-first 100M model, broadens hardware support. Kyutai ships clean code. Needs API vetting. |
|
||||
| 3 | **MOSS-TTS** | Text-to-voice design (no ref audio) is unique. Multi-speaker dialogue for Stories. Needs thorough API vetting. |
|
||||
| 4 | **Kokoro-82M** | 82M params, CPU realtime, Apache 2.0. Easy win. |
|
||||
| 5 | ~~**Model config registry refactor**~~ | **Done** — consolidated in `backend/backends/__init__.py` + `EngineModelSelector.tsx` |
|
||||
| 6 | XTTS-v2 / Fish Speech / CosyVoice | Multi-engine arch is ready; just needs backend implementation |
|
||||
| 7 | **VoxCPM 1.5** | Tokenizer-free streaming, interesting but uncertain integration surface |
|
||||
| 8 | OpenAI-compatible API (plan doc exists) | Low effort once API is stable |
|
||||
| 9 | LoRA fine-tuning (PR #195) | Complex, needs rework for multi-engine |
|
||||
| 10 | External/remote providers | Depends on use case demand |
|
||||
| 11 | GGUF support (#226) | Depends on model ecosystem maturity |
|
||||
| 12 | Queue system (#234) | Batch generation |
|
||||
| 13 | Streaming for non-MLX engines | Currently MLX-only |
|
||||
|
||||
---
|
||||
|
||||
## Branch Inventory
|
||||
|
||||
| Branch | PR | Status | Notes |
|
||||
|--------|-----|--------|-------|
|
||||
| `feat/chatterbox-turbo` | #258 | Open | Chatterbox Turbo + per-engine languages |
|
||||
| `feat/chatterbox` | #257 | **Merged** | Chatterbox Multilingual |
|
||||
| `feat/luxtts` | #254 | **Merged** | LuxTTS + multi-engine arch |
|
||||
| `external-provider-binaries` | #33 | Superseded by #252 | Original CUDA provider approach |
|
||||
| `feat/dual-server-binaries` | — | No PR | Related to provider split |
|
||||
| `fix-multi-sample` | — | No PR | Voice profile multi-sample fix |
|
||||
| `fix-dl-notification-...` | — | No PR | Model download UX |
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference: API Endpoints
|
||||
|
||||
<details>
|
||||
<summary>All current endpoints</summary>
|
||||
|
||||
| Endpoint | Method | Purpose |
|
||||
|----------|--------|---------|
|
||||
| `/health` | GET | Health check, model/GPU status |
|
||||
| `/profiles` | POST, GET | Create/list voice profiles |
|
||||
| `/profiles/{id}` | GET, PUT, DELETE | Profile CRUD |
|
||||
| `/profiles/{id}/samples` | POST, GET | Add/list voice samples |
|
||||
| `/profiles/{id}/avatar` | POST, GET, DELETE | Avatar management |
|
||||
| `/profiles/{id}/export` | GET | Export profile as ZIP |
|
||||
| `/profiles/import` | POST | Import profile from ZIP |
|
||||
| `/generate` | POST | Generate speech (engine param selects TTS backend) |
|
||||
| `/generate/stream` | POST | Stream speech (MLX only) |
|
||||
| `/history` | GET | List generation history |
|
||||
| `/history/{id}` | GET, DELETE | Get/delete generation |
|
||||
| `/history/{id}/export` | GET | Export generation ZIP |
|
||||
| `/history/{id}/export-audio` | GET | Export audio only |
|
||||
| `/transcribe` | POST | Transcribe audio (Whisper) |
|
||||
| `/models/status` | GET | All model statuses (Qwen, LuxTTS, Chatterbox, Chatterbox Turbo, Whisper) |
|
||||
| `/models/download` | POST | Trigger model download |
|
||||
| `/models/download/cancel` | POST | Cancel/dismiss download |
|
||||
| `/models/{name}` | DELETE | Delete downloaded model |
|
||||
| `/models/load` | POST | Load model into memory |
|
||||
| `/models/unload` | POST | Unload model |
|
||||
| `/models/progress/{name}` | GET | SSE download progress |
|
||||
| `/tasks/active` | GET | Active downloads/generations (with inline progress) |
|
||||
| `/stories` | POST, GET | Create/list stories |
|
||||
| `/stories/{id}` | GET, PUT, DELETE | Story CRUD |
|
||||
| `/stories/{id}/items` | POST, GET | Story items CRUD |
|
||||
| `/stories/{id}/export` | GET | Export story audio |
|
||||
| `/channels` | POST, GET | Audio channel CRUD |
|
||||
| `/channels/{id}` | PUT, DELETE | Channel update/delete |
|
||||
| `/cache/clear` | POST | Clear voice prompt cache |
|
||||
| `/server/cuda/status` | GET | CUDA binary availability |
|
||||
| `/server/cuda/download` | POST | Download CUDA binary |
|
||||
| `/server/cuda/switch` | POST | Switch to CUDA backend |
|
||||
|
||||
</details>
|
||||
Reference in New Issue
Block a user