mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-16 13:20:39 -07:00
feat: add LuxTTS as second TTS engine with multi-engine support
Introduce LuxTTS (ZipVoice) alongside Qwen TTS, enabling users to choose between engines at generation time. LuxTTS offers fast, English-focused voice cloning at 48kHz with ~1GB VRAM. Backend: - Add LuxTTSBackend with encode_prompt/generate_speech integration - Multi-engine registry (get_tts_backend_for_engine) replacing singleton - Engine-prefixed voice prompt cache keys to avoid collisions - Engine field on GenerationRequest (default 'qwen' for backward compat) - Engine dispatch in /generate and /generate/stream endpoints - LuxTTS in model status, download, and delete maps Frontend: - TTS Engine selector dropdown in GenerationForm (Qwen TTS / LuxTTS) - Conditionally hide Model Size and Delivery Instructions for LuxTTS - Engine field added to TypeScript types and Zod schema - LuxTTS section in Model Management page
This commit is contained in:
James Pine
@@ -76,29 +76,56 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Delivery Instructions (optional)</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
|
||||
className="min-h-[80px]"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Natural language instructions to control speech delivery (tone, emotion, pace).
|
||||
Max 500 characters
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
{form.watch('engine') !== 'luxtts' && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Delivery Instructions (optional)</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
|
||||
className="min-h-[80px]"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Natural language instructions to control speech delivery (tone, emotion,
|
||||
pace). Max 500 characters
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid gap-4 md:grid-cols-3">
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="engine"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>TTS Engine</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="qwen">Qwen TTS</SelectItem>
|
||||
<SelectItem value="luxtts">LuxTTS</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{field.value === 'luxtts' ? 'Fast, English-focused' : 'Multi-language'}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
@@ -124,29 +151,6 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="modelSize"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Model Size</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="1.7B">Qwen TTS 1.7B (Higher Quality)</SelectItem>
|
||||
<SelectItem value="0.6B">Qwen TTS 0.6B (Faster)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>Larger models produce better quality</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="seed"
|
||||
@@ -170,11 +174,32 @@ export function GenerationForm() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
type="submit"
|
||||
className="w-full"
|
||||
disabled={isPending || !selectedProfileId}
|
||||
>
|
||||
{form.watch('engine') !== 'luxtts' && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="modelSize"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Model Size</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="1.7B">Qwen TTS 1.7B (Higher Quality)</SelectItem>
|
||||
<SelectItem value="0.6B">Qwen TTS 0.6B (Faster)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>Larger models produce better quality</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Button type="submit" className="w-full" disabled={isPending || !selectedProfileId}>
|
||||
{isPending ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
|
||||
@@ -80,16 +80,19 @@ export function ModelManagement() {
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
}, [queryClient]);
|
||||
|
||||
const handleDownloadError = useCallback((error: string) => {
|
||||
console.log('[ModelManagement] Download error, clearing state');
|
||||
if (downloadingModel) {
|
||||
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
|
||||
setConsoleOpen(true);
|
||||
}
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
}, [queryClient, downloadingModel]);
|
||||
const handleDownloadError = useCallback(
|
||||
(error: string) => {
|
||||
console.log('[ModelManagement] Download error, clearing state');
|
||||
if (downloadingModel) {
|
||||
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
|
||||
setConsoleOpen(true);
|
||||
}
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
},
|
||||
[queryClient, downloadingModel],
|
||||
);
|
||||
|
||||
// Use progress toast hook for the downloading model
|
||||
useModelDownloadToast({
|
||||
@@ -165,7 +168,11 @@ export function ModelManagement() {
|
||||
|
||||
// Optimistically hide the error and suppress downloading state in UI
|
||||
setDismissedErrors((prev) => new Set(prev).add(modelName));
|
||||
setLocalErrors((prev) => { const next = new Map(prev); next.delete(modelName); return next; });
|
||||
setLocalErrors((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelName);
|
||||
return next;
|
||||
});
|
||||
if (downloadingModel === modelName) {
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
@@ -178,7 +185,11 @@ export function ModelManagement() {
|
||||
setLocalErrors(prevLocalErrors);
|
||||
setDownloadingModel(prevDownloadingModel);
|
||||
setDownloadingDisplayName(prevDownloadingDisplayName);
|
||||
toast({ title: 'Cancel failed', description: 'Could not cancel the download task.', variant: 'destructive' });
|
||||
toast({
|
||||
title: 'Cancel failed',
|
||||
description: 'Could not cancel the download task.',
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -273,7 +284,9 @@ export function ModelManagement() {
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
@@ -282,6 +295,34 @@ export function ModelManagement() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* LuxTTS Models */}
|
||||
{modelStatus.models.some((m) => m.model_name.startsWith('luxtts')) && (
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">LuxTTS Models</h3>
|
||||
<div className="space-y-2">
|
||||
{modelStatus.models
|
||||
.filter((m) => m.model_name.startsWith('luxtts'))
|
||||
.map((model) => (
|
||||
<ModelItem
|
||||
key={model.model_name}
|
||||
model={model}
|
||||
onDownload={() => handleDownload(model.model_name)}
|
||||
onDelete={() => {
|
||||
setModelToDelete({
|
||||
name: model.model_name,
|
||||
displayName: model.display_name,
|
||||
sizeMb: model.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
formatSize={formatSize}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Whisper Models */}
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">
|
||||
@@ -305,7 +346,9 @@ export function ModelManagement() {
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
@@ -353,12 +396,16 @@ export function ModelManagement() {
|
||||
{dl.error ? (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">{dl.error}</span>
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">
|
||||
{dl.error}
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#808080]">No error details available. Try downloading again.</span>
|
||||
<span className="text-[#808080]">
|
||||
No error details available. Try downloading again.
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
<div className="text-[#6a9955] mt-0.5">
|
||||
@@ -422,21 +469,31 @@ interface ModelItemProps {
|
||||
model_name: string;
|
||||
display_name: string;
|
||||
downloaded: boolean;
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
size_mb?: number;
|
||||
loaded: boolean;
|
||||
};
|
||||
onDownload: () => void;
|
||||
onDelete: () => void;
|
||||
onCancel: () => void;
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
isCancelling: boolean;
|
||||
isDismissed: boolean;
|
||||
erroredDownload?: ActiveDownloadTask;
|
||||
formatSize: (sizeMb?: number) => string;
|
||||
}
|
||||
|
||||
function ModelItem({ model, onDownload, onDelete, onCancel, isDownloading, isCancelling, isDismissed, erroredDownload, formatSize }: ModelItemProps) {
|
||||
function ModelItem({
|
||||
model,
|
||||
onDownload,
|
||||
onDelete,
|
||||
onCancel,
|
||||
isDownloading,
|
||||
isCancelling,
|
||||
isDismissed,
|
||||
erroredDownload,
|
||||
formatSize,
|
||||
}: ModelItemProps) {
|
||||
// Use server's downloading state OR local state (for immediate feedback before server updates)
|
||||
// Suppress downloading if user just dismissed/cancelled this model
|
||||
const showDownloading = (model.downloading || isDownloading) && !erroredDownload && !isDismissed;
|
||||
|
||||
Reference in New Issue
Block a user