mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-16 21:30:39 -07:00
add HumeAI TADA TTS engine (1B English + 3B Multilingual)
Integrates HumeAI's TADA (Text-Acoustic Dual Alignment) speech-language model as a new TTS engine. TADA uses a novel 1:1 token-audio alignment that produces coherent speech over long sequences (700s+). Two model variants: - tada-1b: English-only, ~4GB, built on Llama 3.2 1B - tada-3b-ml: 10 languages, ~8GB, built on Llama 3.2 3B Backend uses the Encoder for voice prompt encoding with caching, and TadaForCausalLM with flow-matching diffusion for generation. Supports bf16 inference on CUDA, forces CPU on macOS (MPS compatibility). Installed with --no-deps due to torch>=2.7 pin conflict; descript-audio-codec and torchaudio added as explicit sub-dependencies.
This commit is contained in:
@@ -20,6 +20,8 @@ const ENGINE_OPTIONS = [
|
||||
{ value: 'luxtts', label: 'LuxTTS' },
|
||||
{ value: 'chatterbox', label: 'Chatterbox' },
|
||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
|
||||
{ value: 'tada:1B', label: 'TADA 1B' },
|
||||
{ value: 'tada:3B', label: 'TADA 3B Multilingual' },
|
||||
] as const;
|
||||
|
||||
const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
||||
@@ -27,6 +29,7 @@ const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
||||
luxtts: 'Fast, English-focused',
|
||||
chatterbox: '23 languages, incl. Hebrew',
|
||||
chatterbox_turbo: 'English, [laugh] [cough] tags',
|
||||
tada: 'HumeAI, 700s+ coherent audio',
|
||||
};
|
||||
|
||||
/** Engines that only support English and should force language to 'en' on select. */
|
||||
@@ -34,6 +37,7 @@ const ENGLISH_ONLY_ENGINES = new Set(['luxtts', 'chatterbox_turbo']);
|
||||
|
||||
function getSelectValue(engine: string, modelSize?: string): string {
|
||||
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
|
||||
if (engine === 'tada') return `tada:${modelSize || '1B'}`;
|
||||
return engine;
|
||||
}
|
||||
|
||||
@@ -48,6 +52,20 @@ function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: st
|
||||
if (!available.some((l) => l.value === currentLang)) {
|
||||
form.setValue('language', available[0]?.value ?? 'en');
|
||||
}
|
||||
} else if (value.startsWith('tada:')) {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'tada');
|
||||
form.setValue('modelSize', modelSize as '1B' | '3B');
|
||||
// TADA 1B is English-only; 3B is multilingual
|
||||
if (modelSize === '1B') {
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const currentLang = form.getValues('language');
|
||||
const available = getLanguageOptionsForEngine('tada');
|
||||
if (!available.some((l) => l.value === currentLang)) {
|
||||
form.setValue('language', available[0]?.value ?? 'en');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
form.setValue('engine', value as GenerationFormValues['engine']);
|
||||
form.setValue('modelSize', undefined as unknown as '1.7B' | '0.6B');
|
||||
|
||||
@@ -62,6 +62,10 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
|
||||
'Production-grade open source TTS by Resemble AI. Supports 23 languages with voice cloning and emotion exaggeration control.',
|
||||
'chatterbox-turbo':
|
||||
'Streamlined 350M parameter TTS by Resemble AI. High-quality English speech with less compute and VRAM than larger models.',
|
||||
'tada-1b':
|
||||
'HumeAI TADA 1B — English speech-language model built on Llama 3.2 1B. Generates 700s+ of coherent audio with synchronized text-acoustic alignment.',
|
||||
'tada-3b-ml':
|
||||
'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.',
|
||||
'whisper-base':
|
||||
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
|
||||
'whisper-small':
|
||||
@@ -391,7 +395,8 @@ export function ModelManagement() {
|
||||
(m) =>
|
||||
m.model_name.startsWith('qwen-tts') ||
|
||||
m.model_name.startsWith('luxtts') ||
|
||||
m.model_name.startsWith('chatterbox'),
|
||||
m.model_name.startsWith('chatterbox') ||
|
||||
m.model_name.startsWith('tada'),
|
||||
) ?? [];
|
||||
const whisperModels = modelStatus?.models.filter((m) => m.model_name.startsWith('whisper')) ?? [];
|
||||
|
||||
|
||||
@@ -42,8 +42,8 @@ export interface GenerationRequest {
|
||||
text: string;
|
||||
language: LanguageCode;
|
||||
seed?: number;
|
||||
model_size?: '1.7B' | '0.6B';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo';
|
||||
model_size?: '1.7B' | '0.6B' | '1B' | '3B';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada';
|
||||
instruct?: string;
|
||||
max_chunk_chars?: number;
|
||||
crossfade_ms?: number;
|
||||
|
||||
@@ -66,6 +66,7 @@ export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
|
||||
'zh',
|
||||
],
|
||||
chatterbox_turbo: ['en'],
|
||||
tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'],
|
||||
} as const;
|
||||
|
||||
/** Helper: get language options for a given engine. */
|
||||
|
||||
@@ -15,9 +15,9 @@ const generationSchema = z.object({
|
||||
text: z.string().min(1, '').max(50000),
|
||||
language: z.enum(LANGUAGE_CODES as [LanguageCode, ...LanguageCode[]]),
|
||||
seed: z.number().int().optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B']).optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(),
|
||||
instruct: z.string().max(500).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo']).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada']).optional(),
|
||||
});
|
||||
|
||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||
@@ -79,7 +79,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
? 'chatterbox-tts'
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'chatterbox-turbo'
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
: engine === 'tada'
|
||||
? `tada-${(data.modelSize || '1B').toLowerCase()}`
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
const displayName =
|
||||
engine === 'luxtts'
|
||||
? 'LuxTTS'
|
||||
@@ -87,9 +89,13 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
? 'Chatterbox TTS'
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'Chatterbox Turbo'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
: engine === 'tada'
|
||||
? data.modelSize === '3B'
|
||||
? 'TADA 3B Multilingual'
|
||||
: 'TADA 1B'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
|
||||
// Check if model needs downloading
|
||||
try {
|
||||
@@ -104,7 +110,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
console.error('Failed to check model status:', error);
|
||||
}
|
||||
|
||||
const isQwen = engine === 'qwen';
|
||||
const hasModelSizes = engine === 'qwen' || engine === 'tada';
|
||||
const effectsChain = options.getEffectsChain?.();
|
||||
// This now returns immediately with status="generating"
|
||||
const result = await generation.mutateAsync({
|
||||
@@ -112,9 +118,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
text: data.text,
|
||||
language: data.language,
|
||||
seed: data.seed,
|
||||
model_size: isQwen ? data.modelSize : undefined,
|
||||
model_size: hasModelSizes ? data.modelSize : undefined,
|
||||
engine,
|
||||
instruct: isQwen ? data.instruct || undefined : undefined,
|
||||
instruct: engine === 'qwen' ? data.instruct || undefined : undefined,
|
||||
max_chunk_chars: maxChunkChars,
|
||||
crossfade_ms: crossfadeMs,
|
||||
normalize: normalizeAudio,
|
||||
|
||||
Reference in New Issue
Block a user