feat: add Chatterbox TTS engine for multilingual voice cloning

- New ChatterboxTTSBackend wrapping ChatterboxMultilingualTTS (ResembleAI/chatterbox)
- Supports 23 languages including Hebrew, forces CPU on macOS (MPS issue)
- Monkey-patches torch.load for CPU loading, forces eager attention for compatibility
- trim_tts_output utility cuts trailing silence/hallucination from Chatterbox output
- Full engine integration: /generate, /generate/stream, model status/download/delete
- Hebrew (he) added to supported languages in frontend and backend validation
- Single flat model dropdown extended with Chatterbox option in both generation UIs
- ModelManagement UI groups LuxTTS and Chatterbox under 'Other Voice Models' section
This commit is contained in:
James Pine
2026-03-13 02:09:32 -07:00
parent 3576521d62
commit 76bb207b2b
16 changed files with 1401 additions and 346 deletions
+18 -1
View File
@@ -34,7 +34,7 @@ export interface GenerationRequest {
language: LanguageCode;
seed?: number;
model_size?: '1.7B' | '0.6B';
engine?: 'qwen' | 'luxtts';
engine?: 'qwen' | 'luxtts' | 'chatterbox';
instruct?: string;
}
@@ -119,12 +119,29 @@ export interface ModelProgress {
export interface ModelStatus {
model_name: string;
display_name: string;
hf_repo_id?: string; // HuggingFace repository ID
downloaded: boolean;
downloading: boolean; // True if download is in progress
size_mb?: number;
loaded: boolean;
}
export interface HuggingFaceModelInfo {
id: string;
author: string;
lastModified: string;
pipeline_tag?: string;
library_name?: string;
downloads: number;
likes: number;
tags: string[];
cardData?: {
license?: string;
language?: string[];
pipeline_tag?: string;
};
}
export interface ModelStatusListResponse {
models: ModelStatus[];
}
+3 -2
View File
@@ -1,6 +1,6 @@
/**
* Supported languages for Qwen3-TTS
* Based on: https://github.com/QwenLM/Qwen3-TTS
* Supported languages for voice generation.
* Most languages use Qwen3-TTS; Hebrew uses Chatterbox TTS.
*/
export const SUPPORTED_LANGUAGES = {
@@ -14,6 +14,7 @@ export const SUPPORTED_LANGUAGES = {
pt: 'Portuguese',
es: 'Spanish',
it: 'Italian',
he: 'Hebrew',
} as const;
export type LanguageCode = keyof typeof SUPPORTED_LANGUAGES;
+15 -7
View File
@@ -16,7 +16,7 @@ const generationSchema = z.object({
seed: z.number().int().optional(),
modelSize: z.enum(['1.7B', '0.6B']).optional(),
instruct: z.string().max(500).optional(),
engine: z.enum(['qwen', 'luxtts']).optional(),
engine: z.enum(['qwen', 'luxtts', 'chatterbox']).optional(),
});
export type GenerationFormValues = z.infer<typeof generationSchema>;
@@ -70,13 +70,20 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
setIsGenerating(true);
const engine = data.engine || 'qwen';
const modelName = engine === 'luxtts' ? 'luxtts' : `qwen-tts-${data.modelSize}`;
const modelName =
engine === 'luxtts'
? 'luxtts'
: engine === 'chatterbox'
? 'chatterbox-tts'
: `qwen-tts-${data.modelSize}`;
const displayName =
engine === 'luxtts'
? 'LuxTTS'
: data.modelSize === '1.7B'
? 'Qwen TTS 1.7B'
: 'Qwen TTS 0.6B';
: engine === 'chatterbox'
? 'Chatterbox TTS'
: data.modelSize === '1.7B'
? 'Qwen TTS 1.7B'
: 'Qwen TTS 0.6B';
try {
const modelStatus = await apiClient.getModelStatus();
@@ -90,14 +97,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
console.error('Failed to check model status:', error);
}
const isQwen = engine === 'qwen';
const result = await generation.mutateAsync({
profile_id: selectedProfileId,
text: data.text,
language: data.language,
seed: data.seed,
model_size: engine === 'luxtts' ? undefined : data.modelSize,
model_size: isQwen ? data.modelSize : undefined,
engine,
instruct: engine === 'luxtts' ? undefined : data.instruct || undefined,
instruct: isQwen ? data.instruct || undefined : undefined,
});
toast({