feat: add LuxTTS as second TTS engine with multi-engine support

Introduce LuxTTS (ZipVoice) alongside Qwen TTS, enabling users to choose
between engines at generation time. LuxTTS offers fast, English-focused
voice cloning at 48kHz with ~1GB VRAM.

Backend:
- Add LuxTTSBackend with encode_prompt/generate_speech integration
- Multi-engine registry (get_tts_backend_for_engine) replacing singleton
- Engine-prefixed voice prompt cache keys to avoid collisions
- Engine field on GenerationRequest (default 'qwen' for backward compat)
- Engine dispatch in /generate and /generate/stream endpoints
- LuxTTS in model status, download, and delete maps

Frontend:
- TTS Engine selector dropdown in GenerationForm (Qwen TTS / LuxTTS)
- Conditionally hide Model Size and Delivery Instructions for LuxTTS
- Engine field added to TypeScript types and Zod schema
- LuxTTS section in Model Management page
This commit is contained in:
James Pine
2026-03-13 00:21:43 -07:00
parent 6359dee406
commit d46eb5bcc6
10 changed files with 584 additions and 121 deletions
@@ -76,29 +76,56 @@ export function GenerationForm() {
)}
/>
<FormField
control={form.control}
name="instruct"
render={({ field }) => (
<FormItem>
<FormLabel>Delivery Instructions (optional)</FormLabel>
<FormControl>
<Textarea
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
className="min-h-[80px]"
{...field}
/>
</FormControl>
<FormDescription>
Natural language instructions to control speech delivery (tone, emotion, pace).
Max 500 characters
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
{form.watch('engine') !== 'luxtts' && (
<FormField
control={form.control}
name="instruct"
render={({ field }) => (
<FormItem>
<FormLabel>Delivery Instructions (optional)</FormLabel>
<FormControl>
<Textarea
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
className="min-h-[80px]"
{...field}
/>
</FormControl>
<FormDescription>
Natural language instructions to control speech delivery (tone, emotion,
pace). Max 500 characters
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
<div className="grid gap-4 md:grid-cols-3">
<FormField
control={form.control}
name="engine"
render={({ field }) => (
<FormItem>
<FormLabel>TTS Engine</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
<SelectItem value="qwen">Qwen TTS</SelectItem>
<SelectItem value="luxtts">LuxTTS</SelectItem>
</SelectContent>
</Select>
<FormDescription>
{field.value === 'luxtts' ? 'Fast, English-focused' : 'Multi-language'}
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="language"
@@ -124,29 +151,6 @@ export function GenerationForm() {
)}
/>
<FormField
control={form.control}
name="modelSize"
render={({ field }) => (
<FormItem>
<FormLabel>Model Size</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
<SelectItem value="1.7B">Qwen TTS 1.7B (Higher Quality)</SelectItem>
<SelectItem value="0.6B">Qwen TTS 0.6B (Faster)</SelectItem>
</SelectContent>
</Select>
<FormDescription>Larger models produce better quality</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="seed"
@@ -170,11 +174,32 @@ export function GenerationForm() {
/>
</div>
<Button
type="submit"
className="w-full"
disabled={isPending || !selectedProfileId}
>
{form.watch('engine') !== 'luxtts' && (
<FormField
control={form.control}
name="modelSize"
render={({ field }) => (
<FormItem>
<FormLabel>Model Size</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
<SelectItem value="1.7B">Qwen TTS 1.7B (Higher Quality)</SelectItem>
<SelectItem value="0.6B">Qwen TTS 0.6B (Faster)</SelectItem>
</SelectContent>
</Select>
<FormDescription>Larger models produce better quality</FormDescription>
<FormMessage />
</FormItem>
)}
/>
)}
<Button type="submit" className="w-full" disabled={isPending || !selectedProfileId}>
{isPending ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
@@ -80,16 +80,19 @@ export function ModelManagement() {
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
}, [queryClient]);
const handleDownloadError = useCallback((error: string) => {
console.log('[ModelManagement] Download error, clearing state');
if (downloadingModel) {
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
setConsoleOpen(true);
}
setDownloadingModel(null);
setDownloadingDisplayName(null);
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
}, [queryClient, downloadingModel]);
const handleDownloadError = useCallback(
(error: string) => {
console.log('[ModelManagement] Download error, clearing state');
if (downloadingModel) {
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
setConsoleOpen(true);
}
setDownloadingModel(null);
setDownloadingDisplayName(null);
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
},
[queryClient, downloadingModel],
);
// Use progress toast hook for the downloading model
useModelDownloadToast({
@@ -165,7 +168,11 @@ export function ModelManagement() {
// Optimistically hide the error and suppress downloading state in UI
setDismissedErrors((prev) => new Set(prev).add(modelName));
setLocalErrors((prev) => { const next = new Map(prev); next.delete(modelName); return next; });
setLocalErrors((prev) => {
const next = new Map(prev);
next.delete(modelName);
return next;
});
if (downloadingModel === modelName) {
setDownloadingModel(null);
setDownloadingDisplayName(null);
@@ -178,7 +185,11 @@ export function ModelManagement() {
setLocalErrors(prevLocalErrors);
setDownloadingModel(prevDownloadingModel);
setDownloadingDisplayName(prevDownloadingDisplayName);
toast({ title: 'Cancel failed', description: 'Could not cancel the download task.', variant: 'destructive' });
toast({
title: 'Cancel failed',
description: 'Could not cancel the download task.',
variant: 'destructive',
});
},
});
};
@@ -273,7 +284,9 @@ export function ModelManagement() {
}}
onCancel={() => handleCancel(model.model_name)}
isDownloading={downloadingModel === model.model_name}
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
isCancelling={
cancelMutation.isPending && cancelMutation.variables === model.model_name
}
isDismissed={dismissedErrors.has(model.model_name)}
erroredDownload={erroredDownloads.get(model.model_name)}
formatSize={formatSize}
@@ -282,6 +295,34 @@ export function ModelManagement() {
</div>
</div>
{/* LuxTTS Models */}
{modelStatus.models.some((m) => m.model_name.startsWith('luxtts')) && (
<div>
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">LuxTTS Models</h3>
<div className="space-y-2">
{modelStatus.models
.filter((m) => m.model_name.startsWith('luxtts'))
.map((model) => (
<ModelItem
key={model.model_name}
model={model}
onDownload={() => handleDownload(model.model_name)}
onDelete={() => {
setModelToDelete({
name: model.model_name,
displayName: model.display_name,
sizeMb: model.size_mb,
});
setDeleteDialogOpen(true);
}}
isDownloading={downloadingModel === model.model_name}
formatSize={formatSize}
/>
))}
</div>
</div>
)}
{/* Whisper Models */}
<div>
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">
@@ -305,7 +346,9 @@ export function ModelManagement() {
}}
onCancel={() => handleCancel(model.model_name)}
isDownloading={downloadingModel === model.model_name}
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
isCancelling={
cancelMutation.isPending && cancelMutation.variables === model.model_name
}
isDismissed={dismissedErrors.has(model.model_name)}
erroredDownload={erroredDownloads.get(model.model_name)}
formatSize={formatSize}
@@ -353,12 +396,16 @@ export function ModelManagement() {
{dl.error ? (
<>
{': '}
<span className="text-[#ce9178] whitespace-pre-wrap break-all">{dl.error}</span>
<span className="text-[#ce9178] whitespace-pre-wrap break-all">
{dl.error}
</span>
</>
) : (
<>
{': '}
<span className="text-[#808080]">No error details available. Try downloading again.</span>
<span className="text-[#808080]">
No error details available. Try downloading again.
</span>
</>
)}
<div className="text-[#6a9955] mt-0.5">
@@ -422,21 +469,31 @@ interface ModelItemProps {
model_name: string;
display_name: string;
downloaded: boolean;
downloading?: boolean; // From server - true if download in progress
downloading?: boolean; // From server - true if download in progress
size_mb?: number;
loaded: boolean;
};
onDownload: () => void;
onDelete: () => void;
onCancel: () => void;
isDownloading: boolean; // Local state - true if user just clicked download
isDownloading: boolean; // Local state - true if user just clicked download
isCancelling: boolean;
isDismissed: boolean;
erroredDownload?: ActiveDownloadTask;
formatSize: (sizeMb?: number) => string;
}
function ModelItem({ model, onDownload, onDelete, onCancel, isDownloading, isCancelling, isDismissed, erroredDownload, formatSize }: ModelItemProps) {
function ModelItem({
model,
onDownload,
onDelete,
onCancel,
isDownloading,
isCancelling,
isDismissed,
erroredDownload,
formatSize,
}: ModelItemProps) {
// Use server's downloading state OR local state (for immediate feedback before server updates)
// Suppress downloading if user just dismissed/cancelled this model
const showDownloading = (model.downloading || isDownloading) && !erroredDownload && !isDismissed;
+2
View File
@@ -34,6 +34,8 @@ export interface GenerationRequest {
language: LanguageCode;
seed?: number;
model_size?: '1.7B' | '0.6B';
engine?: 'qwen' | 'luxtts';
instruct?: string;
}
export interface GenerationResponse {
+13 -4
View File
@@ -16,6 +16,7 @@ const generationSchema = z.object({
seed: z.number().int().optional(),
modelSize: z.enum(['1.7B', '0.6B']).optional(),
instruct: z.string().max(500).optional(),
engine: z.enum(['qwen', 'luxtts']).optional(),
});
export type GenerationFormValues = z.infer<typeof generationSchema>;
@@ -47,6 +48,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
seed: undefined,
modelSize: '1.7B',
instruct: '',
engine: 'qwen',
...options.defaultValues,
},
});
@@ -67,8 +69,14 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
try {
setIsGenerating(true);
const modelName = `qwen-tts-${data.modelSize}`;
const displayName = data.modelSize === '1.7B' ? 'Qwen TTS 1.7B' : 'Qwen TTS 0.6B';
const engine = data.engine || 'qwen';
const modelName = engine === 'luxtts' ? 'luxtts' : `qwen-tts-${data.modelSize}`;
const displayName =
engine === 'luxtts'
? 'LuxTTS'
: data.modelSize === '1.7B'
? 'Qwen TTS 1.7B'
: 'Qwen TTS 0.6B';
try {
const modelStatus = await apiClient.getModelStatus();
@@ -87,8 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
text: data.text,
language: data.language,
seed: data.seed,
model_size: data.modelSize,
instruct: data.instruct || undefined,
model_size: engine === 'luxtts' ? undefined : data.modelSize,
engine,
instruct: engine === 'luxtts' ? undefined : data.instruct || undefined,
});
toast({