Labyricorn/voicebox
1
0
Fork 0
mirror of https://github.com/jamiepine/voicebox.git synced 2026-09-18 06:10:43 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Chatterbox TTS engine for multilingual voice cloning

Browse Source 
- New ChatterboxTTSBackend wrapping ChatterboxMultilingualTTS (ResembleAI/chatterbox)
- Supports 23 languages including Hebrew, forces CPU on macOS (MPS issue)
- Monkey-patches torch.load for CPU loading, forces eager attention for compatibility
- trim_tts_output utility cuts trailing silence/hallucination from Chatterbox output
- Full engine integration: /generate, /generate/stream, model status/download/delete
- Hebrew (he) added to supported languages in frontend and backend validation
- Single flat model dropdown extended with Chatterbox option in both generation UIs
- ModelManagement UI groups LuxTTS and Chatterbox under 'Other Voice Models' section
This commit is contained in:
James Pine
2026-03-13 02:09:32 -07:00
parent 3576521d62
commit 76bb207b2b
16 changed files with 1401 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/main.py
+72
View File
@@ -676,6 +676,29 @@ async def generate_speech(
)
await tts_model.load_model()
elif engine == "chatterbox":
if not tts_model._is_model_cached():
model_name = "chatterbox-tts"
async def download_chatterbox_background():
try:
await tts_model.load_model()
except Exception as e:
task_manager.error_download(model_name, str(e))
task_manager.start_download(model_name)
asyncio.create_task(download_chatterbox_background())
raise HTTPException(
status_code=202,
detail={
"message": "Chatterbox model is being downloaded. Please wait and try again.",
"model_name": model_name,
"downloading": True,
},
)
await tts_model.load_model()
# Create voice prompt from profile
voice_prompt = await profiles.create_voice_prompt_for_profile(
@@ -693,6 +716,11 @@ async def generate_speech(
data.instruct,
)
# Trim trailing silence/hallucination for Chatterbox output
if engine == "chatterbox":
from .utils.audio import trim_tts_output
audio = trim_tts_output(audio, sample_rate)
# Calculate duration
duration = len(audio) / sample_rate
@@ -763,6 +791,13 @@ async def stream_speech(
detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
)
await tts_model.load_model()
elif engine == "chatterbox":
if not tts_model._is_model_cached():
raise HTTPException(
status_code=400,
detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.",
)
await tts_model.load_model()
voice_prompt = await profiles.create_voice_prompt_for_profile(
data.profile_id, db, engine=engine,
@@ -776,6 +811,11 @@ async def stream_speech(
data.instruct,
)
# Trim trailing silence/hallucination for Chatterbox output
if engine == "chatterbox":
from .utils.audio import trim_tts_output
audio = trim_tts_output(audio, sample_rate)
wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
async def _wav_stream():
@@ -1384,6 +1424,15 @@ async def get_model_status():
except Exception:
return False
# Check if Chatterbox backend is loaded
def check_chatterbox_loaded():
try:
from .backends import get_tts_backend_for_engine
backend = get_tts_backend_for_engine("chatterbox")
return backend.is_loaded()
except Exception:
return False
model_configs = [
{
"model_name": "qwen-tts-1.7B",
@@ -1406,6 +1455,13 @@ async def get_model_status():
"model_size": "default",
"check_loaded": check_luxtts_loaded,
},
{
"model_name": "chatterbox-tts",
"display_name": "Chatterbox TTS (Multilingual)",
"hf_repo_id": "ResembleAI/chatterbox",
"model_size": "default",
"check_loaded": check_chatterbox_loaded,
},
{
"model_name": "whisper-base",
"display_name": "Whisper Base",
@@ -1557,6 +1613,7 @@ async def get_model_status():
statuses.append(models.ModelStatus(
model_name=config["model_name"],
display_name=config["display_name"],
hf_repo_id=config["hf_repo_id"],
downloaded=downloaded,
downloading=is_downloading,
size_mb=size_mb,
@@ -1575,6 +1632,7 @@ async def get_model_status():
statuses.append(models.ModelStatus(
model_name=config["model_name"],
display_name=config["display_name"],
hf_repo_id=config["hf_repo_id"],
downloaded=False, # Assume not downloaded if check failed
downloading=is_downloading,
size_mb=None,
@@ -1606,6 +1664,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
"model_size": "default",
"load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
},
"chatterbox-tts": {
"model_size": "default",
"load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(),
},
"whisper-base": {
"model_size": "base",
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
@@ -1723,6 +1785,11 @@ async def delete_model(model_name: str):
"model_size": "default",
"model_type": "luxtts",
},
"chatterbox-tts": {
"hf_repo_id": "ResembleAI/chatterbox",
"model_size": "default",
"model_type": "chatterbox",
},
"whisper-base": {
"hf_repo_id": "openai/whisper-base",
"model_size": "base",
@@ -1762,6 +1829,11 @@ async def delete_model(model_name: str):
luxtts = get_tts_backend_for_engine("luxtts")
if luxtts.is_loaded():
luxtts.unload_model()
elif config["model_type"] == "chatterbox":
from .backends import get_tts_backend_for_engine
chatterbox = get_tts_backend_for_engine("chatterbox")
if chatterbox.is_loaded():
chatterbox.unload_model()
elif config["model_type"] == "whisper":
whisper_model = transcribe.get_whisper_model()
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]: