diff --git a/docs/content/docs/developer/voice-profiles.mdx b/docs/content/docs/developer/voice-profiles.mdx index 0bfdc0d8..5342d6e2 100644 --- a/docs/content/docs/developer/voice-profiles.mdx +++ b/docs/content/docs/developer/voice-profiles.mdx @@ -5,17 +5,22 @@ description: "How voice profile management works in Voicebox" ## Overview -Voice profiles are the foundation of Voicebox's voice cloning capability. Each profile stores reference audio samples and metadata that the TTS model uses to clone a voice. +Voice profiles are the unit of "a saved voice" in Voicebox. As of 0.4 they support two flavors backed by the same `profiles` table: + +- **Cloned profiles** — store one or more reference audio samples; the cloning engine generates a voice embedding at use time +- **Preset profiles** — store no audio; just a pointer to an engine-specific pre-built voice (e.g. Kokoro's `am_adam`, Qwen CustomVoice's `Ryan`) + +The schema also reserves a third type, `designed`, for future text-described voices. Not currently used by any shipped engine. ## Architecture The voice profile system consists of three main components: -**Database Layer:** SQLite tables store profile metadata and sample references. +**Database Layer:** SQLite tables store profile metadata, sample references (cloned), and engine + voice ID (preset). -**File Storage:** Audio samples are stored on disk in a structured directory format. +**File Storage:** Audio samples are stored on disk in a structured directory format. Preset profiles have no on-disk audio. -**Profile Module:** The `profiles.py` module provides the business logic for CRUD operations. +**Profile Module:** `backend/services/profiles.py` provides the business logic for CRUD operations and dispatches to the appropriate engine based on `voice_type`. ## Data Model @@ -24,27 +29,49 @@ The voice profile system consists of three main components: ```python class VoiceProfile(Base): __tablename__ = "profiles" - - id = Column(String, primary_key=True) + + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) name = Column(String, unique=True, nullable=False) description = Column(Text) language = Column(String, default="en") - created_at = Column(DateTime) - updated_at = Column(DateTime) + avatar_path = Column(String, nullable=True) + effects_chain = Column(Text, nullable=True) + + # Voice type system — added v0.3.x + voice_type = Column(String, default="cloned") # "cloned" | "preset" | "designed" + preset_engine = Column(String, nullable=True) # e.g. "kokoro" — only for preset + preset_voice_id = Column(String, nullable=True) # e.g. "am_adam" — only for preset + design_prompt = Column(Text, nullable=True) # text description — only for designed (reserved) + default_engine = Column(String, nullable=True) # auto-selected engine, locked for preset + + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) ``` +The `voice_type` column discriminates the three flavors: + +| `voice_type` | `preset_engine` | `preset_voice_id` | Samples in `profile_samples` | +| ------------ | --------------- | ----------------- | ---------------------------- | +| `cloned` | NULL | NULL | Required (≥1 row) | +| `preset` | engine name | voice ID string | None | +| `designed` | NULL | NULL | None (uses `design_prompt`) | + +The `default_engine` column is set automatically when the profile is created. For preset profiles it's locked to the source engine — switching engines at generation time will skip the profile (and the UI auto-switches back when the user clicks a greyed-out card; see the floating generate box and profile grid). + ### ProfileSample Table ```python class ProfileSample(Base): __tablename__ = "profile_samples" - - id = Column(String, primary_key=True) + + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) profile_id = Column(String, ForeignKey("profiles.id")) audio_path = Column(String, nullable=False) reference_text = Column(Text, nullable=False) ``` +Only populated for cloned profiles. Preset and designed profiles have zero rows in this table. + ## File Structure Profiles are stored in the data directory: diff --git a/docs/content/docs/overview/creating-voice-profiles.mdx b/docs/content/docs/overview/creating-voice-profiles.mdx index 49fd8341..d3fd7d15 100644 --- a/docs/content/docs/overview/creating-voice-profiles.mdx +++ b/docs/content/docs/overview/creating-voice-profiles.mdx @@ -1,32 +1,43 @@ --- title: "Creating Voice Profiles" -description: "Advanced guide to creating high-quality voice profiles" +description: "How to create voice profiles, both cloning-based and preset-based" --- ## Overview -Voice profiles are the foundation of voice cloning in Voicebox. This guide covers best practices for creating professional-quality voice profiles. +A **voice profile** is a saved voice you can reuse across generations, stories, and the API. As of 0.4, Voicebox profiles come in two flavors that map to two different ways of getting a voice: -## Quick Start +| Profile type | What it stores | Use when… | +| -------------- | ---------------------------------------------------- | -------------------------------------------------------- | +| **Cloned** | One or more reference audio samples + a voice embedding | You want to replicate a specific person's voice | +| **Preset** | A reference to a pre-built voice in a specific engine | You want a curated, production-ready voice with no audio prep | + +Both types live in the same Profiles tab and behave the same way at generation time — pick the type that matches your goal and follow the workflow below. + + + Not sure which to use? Cloning gives you a *specific* voice but needs clean audio. Preset gives you *good* voices instantly but you don't get to choose who they sound like. + + +## Workflow A — Cloned Profiles + +Use this when you want to replicate a specific person's voice from a recording. - 10-30 seconds of clear speech + 10-30 seconds of clear speech, minimal background noise. See [Voice Cloning](/overview/voice-cloning) for the engine catalog. - **Profiles** → **+ New Profile** + **Profiles** → **+ New Profile** → choose a cloning engine (Qwen3-TTS, Chatterbox, LuxTTS, or TADA) - - Add your audio file + + Drag in an audio file, or record directly with the in-app recorder - - Use the profile to generate speech + + Use the profile to generate a test phrase. If quality is poor, add more samples -## Audio Requirements - -### Ideal Sample Characteristics +### Audio Requirements (Cloning Only) @@ -44,7 +55,7 @@ Voice profiles are the foundation of voice cloning in Voicebox. This guide cover **High fidelity** - 44.1kHz or 48kHz sample rate + 44.1 kHz or 48 kHz sample rate Minimal compression @@ -58,18 +69,16 @@ Voice profiles are the foundation of voice cloning in Voicebox. This guide cover ### File Formats Supported formats: -- **WAV** (recommended) - Lossless quality -- **MP3** - Acceptable, minimal compression -- **M4A** - Acceptable -- **FLAC** - Lossless alternative +- **WAV** (recommended) — Lossless quality +- **MP3** — Acceptable, minimal compression +- **M4A** — Acceptable +- **FLAC** — Lossless alternative Use WAV for best results. Avoid heavily compressed formats. -## Recording Tips - -### Environment +### Recording Tips @@ -87,27 +96,25 @@ Supported formats: - - 44.1kHz or 48kHz sample rate + - 44.1 kHz or 48 kHz sample rate - 16-bit or 24-bit depth - Mono is fine (stereo will be converted) - Avoid automatic gain control -### Speaking +### Speaking Style -- **Natural pace** - Don't rush or speak too slowly -- **Clear articulation** - Pronounce words clearly -- **Consistent volume** - Maintain steady loudness -- **Normal tone** - Speak as you normally would -- **Complete sentences** - Avoid fragments or "ums" +- **Natural pace** — Don't rush or speak too slowly +- **Clear articulation** — Pronounce words clearly +- **Consistent volume** — Maintain steady loudness +- **Normal tone** — Speak as you normally would +- **Complete sentences** — Avoid fragments or "ums" -## Multiple Samples +### Multiple Samples Adding multiple samples can significantly improve quality: -### Why Multiple Samples? - Model learns a more complete representation @@ -123,110 +130,57 @@ Adding multiple samples can significantly improve quality: -### Sample Variety - Consider adding samples with: -1. **Different tones** - - Casual conversation - - Professional/formal - - Excited/enthusiastic - - Calm/serious - -2. **Different content** - - Narratives - - Questions - - Statements - - Emotions (happy, sad, neutral) - -3. **Different recording conditions** - - Studio quality - - Phone call quality (if needed) - - Room acoustics +1. **Different tones** — casual, formal, excited, calm +2. **Different content** — narratives, questions, statements +3. **Different recording conditions** — studio quality, room acoustics All samples should be from the **same speaker**. Mixing voices will produce poor results. -## Processing Existing Audio +### Processing Existing Audio If you have existing audio (podcasts, videos, etc.): -### Extracting Clean Segments - - Look for segments with: - - Just the target speaker - - No background music - - Minimal noise + Look for segments with just the target speaker, no background music, minimal noise - - Tools like Audacity or Adobe Audition: - - Cut out clean 10-30s segments - - Remove silence at start/end - - Normalize volume if needed + Tools like Audacity or Adobe Audition: cut clean 10-30s segments, remove silence at start/end, normalize volume - Save as high-quality WAV file -### Noise Reduction +For light background noise, use Audacity's noise reduction (gentle settings — over-processing introduces artifacts). -If you have light background noise: +### Testing & Iteration -``` -1. Use noise reduction in Audacity: - - Select noise-only section - - Get Noise Profile - - Select full audio - - Apply noise reduction (gentle settings) - -2. Avoid over-processing: - - Can introduce artifacts - - May reduce voice quality -``` - -## Testing & Iteration - -### Test Your Profile - -After creating a profile: +After creating a cloned profile: - Generate a simple phrase: - ``` - "Hello, this is a test of my voice profile." - ``` + Try a simple phrase: `"Hello, this is a test of my voice profile."` - - Listen for: - - Natural tone - - Clear pronunciation - - Proper prosody - - Lack of artifacts + Listen for natural tone, clear pronunciation, proper prosody, lack of artifacts - - If quality is poor: - - Add more samples - - Try different source audio - - Check sample quality + If quality is poor: add more samples, try different source audio, check sample quality -### Common Issues +#### Common Issues **Cause**: Poor quality samples or too short - **Fix**: Use longer, higher quality samples + **Fix**: Use longer, higher-quality samples @@ -242,51 +196,89 @@ After creating a profile: +## Workflow B — Preset Profiles + +Use this when you want a ready-made voice without recording anything. Available engines: **Kokoro 82M** (50 voices) and **Qwen CustomVoice** (9 voices). See [Preset Voices](/overview/preset-voices) for the full catalog. + + + + **Profiles** → **+ New Profile** → choose **Kokoro** or **Qwen CustomVoice** as the engine + + + The engine's voice catalog appears. Click any voice to preview it + + + Give the profile a name. No audio sample required + + + The profile is ready immediately — use it in the floating generate box or Generate page + + + + + Preset profiles are **locked to their source engine**. Switching to a different engine in the floating generate box greys out the profile, since the voice only exists in that engine. Clicking a greyed profile auto-switches the engine back. + + +### Qwen CustomVoice + Instruct + +Preset voices in Qwen CustomVoice support **delivery instructions** — natural-language style control over tone, pace, and emotion. The floating generate box shows a slider icon next to the generate button when a Qwen CustomVoice profile is selected; click it to reveal the instruct textarea. + +See [Preset Voices → Using Instruct Mode](/overview/preset-voices#using-instruct-mode) for examples. + ## Advanced Tips -### Celebrity/Character Voices +### Celebrity / Character Voices (Cloning) For cloning public figures or characters: -1. **Legal considerations** - Ensure you have rights or it's fair use -2. **Source quality** - Find high-quality interview audio or clean clips -3. **Consistency** - Use clips where they speak similarly -4. **Multiple samples** - Very important for recognizable voices +1. **Legal considerations** — Ensure you have rights or it's clearly fair use +2. **Source quality** — Find high-quality interview audio or clean clips +3. **Consistency** — Use clips where they speak similarly +4. **Multiple samples** — Very important for recognizable voices -### Accent & Dialect +### Accent & Dialect (Cloning) -The model will preserve accent and dialect: +Cloning models preserve accent and dialect: -- British English will generate British English -- Southern accent will produce Southern accent -- Regional pronunciations will be maintained +- British English samples generate British English output +- Southern accent samples produce Southern accent output +- Regional pronunciations are maintained -### Emotion Transfer +### Emotion Transfer (Cloning) The emotional tone of samples affects generation: -- Energetic samples → Energetic output -- Calm samples → Calm output -- Mix samples for versatile profile +- Energetic samples → energetic output +- Calm samples → calm output +- Mix samples for a more versatile profile + +For Qwen CustomVoice presets, use the **instruct** field instead of relying on sample emotion — that's exactly what it controls. ## Managing Profiles ### Organization -- **Descriptive names** - "John Smith - Professional Narrator" -- **Add descriptions** - Note recording conditions, use cases -- **Language tags** - Mark the primary language -- **Archive unused** - Keep profile list manageable +- **Descriptive names** — "John Smith - Professional Narrator" +- **Add descriptions** — Note recording conditions, use cases, or which preset voice +- **Language tags** — Mark the primary language +- **Archive unused** — Keep profile list manageable -### Export/Import +### Export / Import - **Export** profiles to share or backup - **Import** from colleagues or teammates -- Profiles include voice embeddings, not original audio +- **Cloned profiles** export with their voice embeddings (not the original audio) +- **Preset profiles** export as engine + voice ID metadata only — the importer must have that engine's model installed ## Next Steps + + Engine catalog and best practices for cloning + + + Full catalog of Kokoro and Qwen CustomVoice voices + Use your profile to generate speech diff --git a/docs/content/docs/overview/gpu-acceleration.mdx b/docs/content/docs/overview/gpu-acceleration.mdx new file mode 100644 index 00000000..b9730b7d --- /dev/null +++ b/docs/content/docs/overview/gpu-acceleration.mdx @@ -0,0 +1,236 @@ +--- +title: "GPU Acceleration" +description: "How Voicebox uses your GPU — auto-detection, manual setup, troubleshooting" +--- + +## Overview + +Voicebox auto-detects available accelerators on first launch and picks the fastest backend it can use. For most people this just works — open the app and you're already on the right backend. + +This page is for the cases where it doesn't: + +- You have a GPU but Voicebox is running on CPU +- You upgraded GPUs (especially to RTX 50-series / Blackwell) and generation broke +- You want to switch backends manually (e.g. force MLX over PyTorch on Apple Silicon) +- You see `[UNSUPPORTED - see logs]` next to your GPU in Settings + +## Backend Matrix + +| Platform | Auto-selected backend | Notes | +| --------------------------- | ------------------------- | ---------------------------------------------------- | +| **macOS Apple Silicon** | MLX (Metal) | 4-5x faster than PyTorch via Apple Neural Engine | +| **macOS Intel** | PyTorch CPU | No GPU acceleration available; PyTorch ≥ 2.2 only | +| **Windows + NVIDIA** | PyTorch CUDA (cu128) | Auto-downloads the CUDA backend binary on first use | +| **Windows + Intel Arc** | PyTorch XPU (IPEX) | New in 0.4 — works with Arc A-series and B-series | +| **Windows generic GPU** | DirectML | Universal Windows GPU support; slower than CUDA | +| **Linux + NVIDIA** | PyTorch CUDA (cu128) | Same auto-download flow as Windows | +| **Linux + AMD** | PyTorch ROCm | Auto-configures `HSA_OVERRIDE_GFX_VERSION` | +| **Linux + Intel Arc** | PyTorch XPU (IPEX) | | +| **Any (no GPU)** | PyTorch CPU | Works everywhere; expect 5-50x slower than GPU | + +The detected backend is shown in Settings → GPU. Logs at startup also print the chosen backend and the device name. + +## Apple Silicon — MLX vs PyTorch + +On M-series Macs, Voicebox ships an MLX-optimized backend that uses the Apple Neural Engine. It's **4-5x faster** than the PyTorch (CPU/Metal) path for supported engines. + +| Engine | MLX support | Notes | +| -------------------- | ----------- | ------------------------------------------- | +| Qwen3-TTS | ✅ Native | Uses MLX exclusively when available | +| Chatterbox / Turbo | PyTorch MPS | Falls back to Metal via PyTorch | +| LuxTTS | PyTorch MPS | | +| TADA | PyTorch MPS | | +| Kokoro | PyTorch MPS | Requires `PYTORCH_ENABLE_MPS_FALLBACK=1` | +| Qwen CustomVoice | PyTorch MPS | | +| Whisper (transcribe) | ✅ Native | MLX-Whisper is the default on Apple Silicon | + +The Whisper Turbo + MLX combo dropped transcription latency from ~20s to ~2-3s on M-series chips (see CHANGELOG entry for v0.1.10). + +## Windows / Linux + NVIDIA — The CUDA Backend Swap + +Voicebox doesn't bundle CUDA into the main installer (it would balloon downloads to multi-gigabyte territory for users who don't have an NVIDIA GPU). Instead, when you first need it, the app downloads a separate **CUDA backend binary** that contains the PyTorch + CUDA runtime. + + + + If an NVIDIA GPU is detected, you'll see "Install CUDA backend" in the GPU panel + + + The app downloads two archives separately: + - **Server core** (~200-400 MB) — versioned with each Voicebox release + - **CUDA libs** (~4 GB) — the heavy PyTorch + CUDA DLLs, versioned independently + + + Voicebox restarts to swap in the CUDA backend + + + + + The split-archive design (added in v0.4) means most Voicebox upgrades only redownload the small server-core archive. The 4 GB libs archive is only refreshed when the underlying CUDA toolkit or torch major version changes. + + +### Auto-update + +When a new Voicebox release ships, the GPU panel checks if the bundled server-core matches the installed CUDA version. If only the core changed (typical), it pulls the new core in the background. If the libs version changed (rare — only happens on cu126 → cu128 type bumps), you'll be prompted to confirm the larger download. + +## RTX 50-series / Blackwell + +Voicebox 0.4 added explicit RTX 50-series support: + +- CUDA toolkit upgraded to **cu128** (previous releases used cu126 which lacks Blackwell kernels) +- Build pinned with `TORCH_CUDA_ARCH_LIST=...12.0+PTX` for forward-compatibility + +If you're on an RTX 5070 / 5080 / 5090 and you see "no kernel image is available" errors: + +1. Make sure you're on Voicebox **≥ 0.4.0** (Settings → About) +2. Reinstall the CUDA backend (Settings → GPU → Reinstall CUDA backend) — older installs may have stale cu126 libs +3. If errors persist, see the GPU compatibility warnings section below + +## Intel Arc (XPU) + +New in 0.4. Works with both Arc A-series (Alchemist: A380, A580, A750, A770) and B-series (Battlemage). + +### Setup + +Voicebox auto-detects Arc GPUs and routes through Intel's PyTorch XPU backend (powered by IPEX — Intel Extension for PyTorch). No extra installation step beyond the standard Voicebox install. + +Verify it's working: +- Settings → GPU should show **XPU** followed by your Arc model name (e.g. `XPU (Intel Arc A770)`) +- Startup logs print `Backend: PYTORCH` and `GPU: XPU (Intel Arc ...)` + +### Engines on XPU + +All PyTorch-based engines work on XPU. Performance is generally between CPU and CUDA — expect ~2-3x speedup over CPU for the larger models. + +## DirectML + +The fallback for Windows users with non-NVIDIA, non-Intel-Arc GPUs (older AMD discrete, integrated GPUs, etc.). Slower than CUDA and XPU but provides some acceleration over CPU. + +Auto-selected when no other GPU backend is available. + +## AMD ROCm (Linux) + +ROCm provides PyTorch GPU acceleration on AMD discrete GPUs. Voicebox auto-configures `HSA_OVERRIDE_GFX_VERSION` for common cards that need the override. + +### Verifying + +```bash +# In a terminal +echo $HSA_OVERRIDE_GFX_VERSION +# Should show e.g. 10.3.0 for RX 6000 series +``` + +If detection fails, set the variable manually before launching Voicebox: + +```bash +export HSA_OVERRIDE_GFX_VERSION=10.3.0 +voicebox +``` + +Common values: +- `10.3.0` — RX 6000 series (RDNA 2) +- `11.0.0` — RX 7000 series (RDNA 3) +- `9.0.0` — Older Vega cards + +## GPU Compatibility Warnings + +Voicebox 0.4 added a runtime check that compares your GPU's compute capability against the architectures the bundled PyTorch was compiled for. If they don't match, you'll see: + +- A startup log line: `WARNING: GPU COMPATIBILITY: is not supported by this PyTorch build...` +- The GPU label in Settings shows `[UNSUPPORTED - see logs]` +- The `/health` API returns a populated `gpu_compatibility_warning` field + +### What to do + +The most common trigger is a brand-new GPU architecture that pre-built PyTorch wheels don't yet cover natively. In order of preference: + +1. **Update Voicebox** — newer releases ship newer PyTorch with broader arch support +2. **Reinstall the CUDA backend** — Settings → GPU → Reinstall CUDA backend +3. **For bleeding-edge GPUs (newer than current Blackwell):** install PyTorch nightly manually: + ```bash + pip install torch --index-url https://download.pytorch.org/whl/nightly/cu128 --force-reinstall + ``` + Then point Voicebox at that environment via [Remote Mode](/overview/remote-mode) until stable PyTorch catches up. +4. **Fall back to CPU** temporarily — set `VOICEBOX_FORCE_CPU=1` before launching + +## CPU-Only Fallback + +When no GPU is available (or you've forced it off), Voicebox runs the PyTorch CPU backend. Expect: + +- 5-50x slower generation depending on engine and text length +- Heavy CPU usage during generation +- Some engines work better than others on CPU: + - **Kokoro 82M** — runs at realtime on modern CPUs + - **LuxTTS** — exceeds 150x realtime on CPU + - **Chatterbox Turbo (350M)** — usable but slow + - Larger models (Qwen 1.7B, Chatterbox Multilingual, TADA 3B) — painful + +For CPU-bound use cases, prefer the smaller, lighter engines. + +## Verifying Your Setup + +Three places to check that the right backend is being used: + + + + Shows the detected backend, GPU model, and VRAM (when applicable). Look for the `[UNSUPPORTED - see logs]` suffix + + + The "Server logs" tab shows the startup banner with `Backend: ` and `GPU: ` + + + `curl http://localhost:17493/health` returns a JSON payload with `backend_type`, `backend_variant`, and `gpu_compatibility_warning` (when applicable) + + + +## Troubleshooting + + + + - On NVIDIA: install the CUDA backend (Settings → GPU) + - On Intel Arc: confirm IPEX detection in startup logs; restart the app after a driver update + - On AMD Linux: check `HSA_OVERRIDE_GFX_VERSION` is set + + + + Almost always means the bundled PyTorch doesn't have kernels for your GPU's compute capability. + + 1. Update to Voicebox ≥ 0.4.0 (Blackwell support added there) + 2. Reinstall the CUDA backend + 3. If still broken, install PyTorch nightly via Remote Mode + + + + - Switch to a smaller model size (e.g. Qwen3 0.6B instead of 1.7B) + - Use Settings → Models to unload other engines you're not using + - Enable `low_cpu_mem_usage` is already on for CPU; for CUDA, the engine's `device_map` handles offload automatically + - Close other GPU applications + + + + Some operations don't have a Metal implementation. Voicebox sets `PYTORCH_ENABLE_MPS_FALLBACK=1` for engines that need it (notably Kokoro), but if you launch from a custom env, set it manually: + ```bash + export PYTORCH_ENABLE_MPS_FALLBACK=1 + ``` + + + + - Check Settings → GPU shows your GPU (not CPU) + - Check VRAM usage — you may be paging to system memory + - Try a smaller model + - For NVIDIA: confirm cu128 is installed (Settings → GPU → version) + + + +## Next Steps + + + + Run the backend on a different machine with a stronger GPU + + + Unload models to free GPU memory + + + General troubleshooting beyond GPU + + diff --git a/docs/content/docs/overview/meta.json b/docs/content/docs/overview/meta.json index 17dab7bf..90ce7e2e 100644 --- a/docs/content/docs/overview/meta.json +++ b/docs/content/docs/overview/meta.json @@ -6,7 +6,9 @@ "installation", "docker", "quick-start", + "gpu-acceleration", "voice-cloning", + "preset-voices", "stories-editor", "recording-transcription", "generation-history", diff --git a/docs/content/docs/overview/preset-voices.mdx b/docs/content/docs/overview/preset-voices.mdx new file mode 100644 index 00000000..f7b3c92b --- /dev/null +++ b/docs/content/docs/overview/preset-voices.mdx @@ -0,0 +1,202 @@ +--- +title: "Preset Voices" +description: "Use built-in, ready-made voices without recording audio samples" +--- + +## Overview + +Some Voicebox engines ship with a curated set of pre-built voices. Instead of cloning from your own audio sample, you pick a voice from a fixed catalog and the model speaks in that voice. No recording, no upload, no per-voice training required. + +Two engines in 0.4 ship preset voices: + +| Engine | Voices | Languages | Strengths | +| --------------------- | ----------------------- | --------- | ------------------------------------------------------- | +| **Kokoro 82M** | 50 | 9 | Tiny model, CPU-friendly, lowest VRAM of any engine | +| **Qwen CustomVoice** | 9 (premium curated) | 4 | Natural-language style control over tone, emotion, pace | + + + Looking for cloning a specific person's voice instead? See [Voice Cloning](/overview/voice-cloning). + + +## When to Use Preset Voices + + + + You don't have (or don't want to provide) a recording of the target voice + + + Curated voices have predictable quality across any text input + + + Skip the audio cleanup, sample preparation, and quality iteration loop + + + Kokoro runs at CPU realtime with ~150 MB on disk — no GPU needed + + + +## Creating a Preset-Voice Profile + + + + Same entry point as cloning profiles + + + Select **Kokoro** or **Qwen CustomVoice** from the engine dropdown + + + The voice catalog for the chosen engine appears — preview each by clicking it + + + Give the profile a name. No audio sample needed — just save + + + Use the profile like any other in the floating generate box or the Generate page + + + + + Preset profiles are locked to their source engine — switching engines won't work since the voice exists only for that model. The profile grid greys out preset profiles when you switch to a different engine, and clicking one auto-switches the engine back to the right one. + + +## Kokoro 82M — 50 Voices Across 9 Languages + +Kokoro is the smallest engine in Voicebox at 82M parameters. It runs at CPU realtime with negligible VRAM, making it the best option for lightweight local inference. Voices are pre-built style vectors trained into the model — there's no concept of cloning here. + +**Repository:** [`hexgrad/Kokoro-82M`](https://huggingface.co/hexgrad/Kokoro-82M) · Apache 2.0 licensed + +### American English + +| Female | Male | +| ------- | ------- | +| Alloy | Adam | +| Aoede | Echo | +| Bella | Eric | +| Heart | Fenrir | +| Jessica | Liam | +| Kore | Michael | +| Nicole | Onyx | +| Nova | Puck | +| River | Santa | +| Sarah | | +| Sky | | + +### British English + +| Female | Male | +| -------- | ------ | +| Alice | Daniel | +| Emma | Fable | +| Isabella | George | +| Lily | Lewis | + +### Other Languages + +| Language | Voices | +| ----------------- | ------------------------------------------- | +| Spanish (`es`) | Dora (f), Alex (m), Santa (m) | +| French (`fr`) | Siwis (f) | +| Hindi (`hi`) | Alpha (f), Beta (f), Omega (m), Psi (m) | +| Italian (`it`) | Sara (f), Nicola (m) | +| Japanese (`ja`) | Alpha (f), Gongitsune (f), Nezumi (f), Tebukuro (f), Kumo (m) | +| Portuguese (`pt`) | Dora (f), Alex (m), Santa (m) | +| Chinese (`zh`) | Xiaobei (f), Xiaoni (f), Xiaoxiao (f), Xiaoyi (f) | + +### Kokoro at a Glance + +| Property | Value | +| --------------- | -------------------------------------------- | +| Parameters | 82M | +| Sample rate | 24 kHz | +| VRAM | ~150 MB (negligible on CPU) | +| Speed | Realtime on CPU, faster on GPU | +| Instruct | Not supported (preset voice carries the style) | +| License | Apache 2.0 | + +## Qwen CustomVoice — 9 Premium Voices with Instruct Control + +Qwen CustomVoice ships with 9 curated speakers and supports **natural-language style control** — you tell the model how to deliver the line ("speak slowly with warmth", "authoritative and clear") and it adapts tone, emotion, and pace. + +Two model sizes: +- **1.7B** — full quality, recommended default +- **0.6B** — lighter, faster, lower-end hardware + +**Repository:** [`Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice`](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) (and 0.6B variant) · by Alibaba + +### Voice Catalog + +| Speaker | Gender | Language | Description | +| --------- | ------ | -------- | ------------------------------------------------------------ | +| Vivian | female | Chinese | Bright, slightly edgy young female voice | +| Serena | female | Chinese | Warm, gentle young female voice | +| Uncle Fu | male | Chinese | Seasoned male voice with a low, mellow timbre | +| Dylan | male | Chinese | Youthful Beijing male voice with a clear, natural timbre | +| Eric | male | Chinese | Lively Chengdu male voice with a slightly husky brightness | +| Ryan | male | English | Dynamic male voice with strong rhythmic drive (default) | +| Aiden | male | English | Sunny American male voice with a clear midrange | +| Ono Anna | female | Japanese | Playful Japanese female voice with a light, nimble timbre | +| Sohee | female | Korean | Warm Korean female voice with rich emotion | + +### Using Instruct Mode + +In the floating generate box, switch to a Qwen CustomVoice profile and click the **delivery instructions** toggle (slider icon, left of the generate button). A second textarea appears below the main text: + +- Main text → what you want the voice to say +- Instruct text → how you want it delivered + +Examples of effective instruct prompts: + +``` +Speak slowly with emphasis, like reading bedtime stories +Warm and friendly, conversational tone +Professional and authoritative, broadcast quality +Whisper, intimate and close +Excited and energetic, like sports commentary +``` + +The full Generate page also surfaces the instruct field as a separate input. + +### Qwen CustomVoice at a Glance + +| Property | Value | +| --------------- | -------------------------------------------------- | +| Parameters | 1.7B / 0.6B | +| Languages | Chinese, English, Japanese, Korean (10 supported) | +| Voices | 9 curated preset speakers | +| VRAM | ~3.5 GB (1.7B), ~1.2 GB (0.6B) | +| Instruct | Yes — natural-language style control | +| Cloning | No — paired Base Qwen3-TTS engine handles cloning | + +## Cloning vs Preset — Quick Decision + +| You want… | Use | +| -------------------------------------------------- | ----------------------------------------- | +| To replicate a specific person's voice | [Voice Cloning](/overview/voice-cloning) | +| Production-ready voices with no audio prep | Kokoro or Qwen CustomVoice | +| The smallest possible footprint (CPU-only) | Kokoro | +| Fine control over delivery (tone, pace, emotion) | Qwen CustomVoice | +| The broadest language coverage | [Voice Cloning](/overview/voice-cloning) via Chatterbox Multilingual (23 langs) | + +## Limitations + + + Preset voices are fixed — you can't fine-tune or modify the underlying voice. If you want a specific voice that isn't in the catalog, use a cloning engine and provide a reference sample. + + +- Preset voices can't be exported to use in other Voicebox installations as audio (only as profile metadata pointing to the same engine + voice ID) +- The Kokoro voice catalog is set by the upstream model — new voices appear only when hexgrad publishes new model releases +- Qwen CustomVoice's 9 speakers are part of the model checkpoint — same constraint + +## Next Steps + + + + Clone a specific voice from your own audio + + + Use a profile to generate audio + + + Compose multi-voice narratives + + diff --git a/docs/content/docs/overview/voice-cloning.mdx b/docs/content/docs/overview/voice-cloning.mdx index bfb25525..400e114f 100644 --- a/docs/content/docs/overview/voice-cloning.mdx +++ b/docs/content/docs/overview/voice-cloning.mdx @@ -1,11 +1,25 @@ --- title: "Voice Cloning" -description: "Clone any voice from just a few seconds of audio" +description: "Clone any voice from a few seconds of reference audio" --- ## Overview -Voicebox uses **Qwen3-TTS** from Alibaba to achieve near-perfect voice cloning from just a few seconds of audio. The model captures prosody, emotion, and natural cadence. +Voicebox can replicate a specific person's voice from a short audio sample — known as **zero-shot voice cloning**. You provide 10-30 seconds of clear speech, the model extracts a voice embedding, and from then on you can generate any text in that voice. + +Five engines in 0.4 support cloning: + +| Engine | Languages | Strengths | +| --------------------------- | --------- | -------------------------------------------------------------------------- | +| **Qwen3-TTS** (0.6B / 1.7B) | 10 | High-quality multilingual, supports delivery instructions on the same kwarg | +| **Chatterbox Multilingual** | 23 | Broadest language coverage — Arabic, Hindi, Swahili, Hebrew, more | +| **Chatterbox Turbo** | English | Fast 350M model with paralinguistic emotion tags (`[laugh]`, `[sigh]`) | +| **LuxTTS** | English | Lightweight (~1 GB VRAM), 48 kHz output, 150x realtime on CPU | +| **TADA** (1B / 3B) | 10 | Speech-language model with 700s+ coherent long-form generation | + + + Don't want to record audio? Use a curated voice from Kokoro or Qwen CustomVoice instead — see [Preset Voices](/overview/preset-voices). + ## How It Works @@ -13,17 +27,30 @@ Voicebox uses **Qwen3-TTS** from Alibaba to achieve near-perfect voice cloning f Provide 10-30 seconds of clear speech from the target voice - - Qwen3-TTS analyzes vocal characteristics, tone, and speaking patterns + + The selected engine analyzes vocal characteristics, tone, and speaking patterns - The model generates a voice embedding for synthesis + A voice embedding is generated and stored with your profile Use the profile to generate any text in the cloned voice +## Choosing an Engine for Cloning + +Different engines suit different use cases. The profile grid greys out unsupported engines so you can switch easily. + +| If you want… | Pick | +| -------------------------------------------------- | --------------------- | +| Best overall quality on a few common languages | **Qwen3-TTS 1.7B** | +| Faster generation, slightly lower quality | **Qwen3-TTS 0.6B** | +| Languages outside Qwen's 10 (Arabic, Hindi, etc.) | **Chatterbox Multilingual** | +| Expressive English with `[laugh]` `[sigh]` tags | **Chatterbox Turbo** | +| CPU-only or GPU-light setup, English | **LuxTTS** | +| Long-form generation (audiobooks, full chapters) | **TADA 3B** | + ## Best Practices ### Sample Quality @@ -52,24 +79,40 @@ Adding multiple samples from the same speaker can improve quality: - Different recording conditions - The model will learn a more robust representation from diverse samples. + The model will learn a more robust representation from diverse samples. Especially helpful for distinctive voices the model might otherwise smooth over. -## Supported Languages +## Supported Languages by Engine -Currently supported: -- English -- Chinese (Mandarin) +- **Qwen3-TTS** — English, Chinese, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian (10) +- **Chatterbox Multilingual** — Arabic, Chinese, Danish, Dutch, English, Finnish, French, German, Greek, Hebrew, Hindi, Italian, Japanese, Korean, Malay, Norwegian, Polish, Portuguese, Russian, Spanish, Swahili, Swedish, Turkish (23) +- **Chatterbox Turbo** — English +- **LuxTTS** — English +- **TADA 3B** — 10 multilingual; **TADA 1B** — English -More languages coming soon. +For complete language tables and engine-specific notes, see the [TTS Engines developer guide](/developer/tts-engines). ## Limitations - Voice cloning should only be used with consent. Ensure you have permission to clone someone's voice. + Voice cloning should only be used with consent. Ensure you have permission to clone someone's voice. See the project's [SECURITY.md](https://github.com/jamiepine/voicebox/blob/main/SECURITY.md) and your local laws on synthetic voice content. -- Quality depends on sample clarity -- Works best with consistent speaking tone +- Quality depends on sample clarity — noisy samples produce noisy clones +- Works best with consistent speaking tone within a sample - May struggle with extreme accents or speech impediments -- Background noise reduces quality +- Background noise reduces quality and can introduce artifacts + +## Next Steps + + + + Step-by-step guide to creating profiles + + + Use built-in voices instead of cloning + + + Use a profile to generate audio + + diff --git a/landing/src/app/page.tsx b/landing/src/app/page.tsx index e1641d8a..abcdb916 100644 --- a/landing/src/app/page.tsx +++ b/landing/src/app/page.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Github, Globe, Languages, MessageSquare, Zap } from 'lucide-react'; +import { Github, Globe, Languages, MessageSquare, SlidersHorizontal, Zap } from 'lucide-react'; import { useEffect, useState } from 'react'; import { ControlUI } from '@/components/ControlUI'; import { Features } from '@/components/Features'; @@ -236,6 +236,101 @@ export default function Home() { + + {/* Qwen CustomVoice */} +
+
+
+

Qwen CustomVoice

+ by Alibaba +
+
+ + 1.7B + + + 0.6B + +
+
+

+ Nine premium preset speakers with natural-language style control. Tell the model how + to deliver — "speak slowly with warmth", "authoritative and clear" — and it adapts + tone, emotion, and pace. +

+
+ + + Instruct control + + + + 10 languages + + + 9 preset voices + +
+
+ + {/* HumeAI TADA */} +
+
+
+

TADA

+ by Hume AI +
+
+ + 3B + + + 1B + +
+
+

+ Speech-language model with text-acoustic dual alignment. Built for long-form + generation — produces 700s+ of coherent audio without drift. Multilingual at 3B, + English-focused at 1B. +

+
+ + + 10 languages + + + Long-form coherent + +
+
+ + {/* Kokoro 82M */} +
+
+
+

Kokoro

+ by hexgrad · Apache 2.0 +
+ + 82M + +
+

+ Tiny 82M-parameter TTS that runs at CPU realtime with negligible VRAM. Pre-built + voice styles instead of cloning — pick a voice, type, generate. Smallest footprint + of any engine. +

+
+ + + CPU realtime + + + Preset voices + +
+
diff --git a/landing/src/components/Footer.tsx b/landing/src/components/Footer.tsx index 3fea8244..de3d6031 100644 --- a/landing/src/components/Footer.tsx +++ b/landing/src/components/Footer.tsx @@ -1,6 +1,7 @@ +import { Coffee } from 'lucide-react'; import Image from 'next/image'; import Link from 'next/link'; -import { GITHUB_REPO } from '@/lib/constants'; +import { DONATE_URL, GITHUB_REPO } from '@/lib/constants'; export function Footer() { return ( @@ -19,9 +20,19 @@ export function Footer() { /> Voicebox -

+

Open source voice cloning studio. Local-first, free forever.

+ + + Donate + {/* Product */} diff --git a/landing/src/components/Navbar.tsx b/landing/src/components/Navbar.tsx index bd704443..5f1881cb 100644 --- a/landing/src/components/Navbar.tsx +++ b/landing/src/components/Navbar.tsx @@ -1,9 +1,9 @@ 'use client'; -import { Github } from 'lucide-react'; +import { Coffee, Github } from 'lucide-react'; import Image from 'next/image'; import { useEffect, useState } from 'react'; -import { GITHUB_REPO } from '@/lib/constants'; +import { DONATE_URL, GITHUB_REPO } from '@/lib/constants'; function formatStarCount(count: number): string { if (count >= 1000) { @@ -75,21 +75,33 @@ export function Navbar() { - {/* GitHub star button */} - - - Star - {starCount !== null && ( - - {formatStarCount(starCount)} - - )} - + {/* Donate + GitHub star buttons */} + ); diff --git a/landing/src/lib/constants.ts b/landing/src/lib/constants.ts index f9378564..8659c996 100644 --- a/landing/src/lib/constants.ts +++ b/landing/src/lib/constants.ts @@ -4,6 +4,7 @@ export const LATEST_VERSION = 'v0.1.0'; export const GITHUB_REPO = 'https://github.com/jamiepine/voicebox'; export const GITHUB_RELEASES_PAGE = `${GITHUB_REPO}/releases`; +export const DONATE_URL = 'https://buymeacoffee.com/jamiepine'; export const DOWNLOAD_LINKS = { macArm: GITHUB_RELEASES_PAGE, diff --git a/tauri/src-tauri/Cargo.lock b/tauri/src-tauri/Cargo.lock index 194314de..58dbadec 100644 --- a/tauri/src-tauri/Cargo.lock +++ b/tauri/src-tauri/Cargo.lock @@ -5041,7 +5041,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "voicebox" -version = "0.3.1" +version = "0.4.0" dependencies = [ "base64 0.22.1", "core-foundation-sys",