mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-17 05:40:42 -07:00
Merge pull request #254 from jamiepine/feat/luxtts
feat: LuxTTS integration — multi-engine TTS support
This commit is contained in:
Jamie Pine
+20
-1
@@ -32,7 +32,26 @@ Thank you for your interest in contributing to Voicebox! This document provides
|
||||
|
||||
### Development Setup
|
||||
|
||||
**Using the Makefile (recommended for macOS/Linux):** Run `make setup` to install all dependencies, then `make dev` to start development servers. See `make help` for all available commands.
|
||||
**Using `just` (recommended):**
|
||||
|
||||
Install [just](https://github.com/casey/just) (`brew install just` or `cargo install just`), then:
|
||||
|
||||
```bash
|
||||
just setup # creates venv, installs Python + JS deps
|
||||
just dev # starts backend + desktop app in one terminal
|
||||
```
|
||||
|
||||
Other useful commands:
|
||||
|
||||
```bash
|
||||
just dev-web # backend + web app (no Tauri/Rust build)
|
||||
just dev-backend # backend only
|
||||
just kill # stop all dev processes
|
||||
just clean-all # nuke everything and start fresh
|
||||
just --list # see all available commands
|
||||
```
|
||||
|
||||
**Using the Makefile:** Run `make setup` then `make dev`. See `make help` for all commands.
|
||||
|
||||
**Manual setup (required for Windows):**
|
||||
|
||||
|
||||
@@ -228,42 +228,21 @@ Voicebox aims to be the **one-stop shop for everything voice** — cloning, synt
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed setup and contribution guidelines.
|
||||
|
||||
**Using the Makefile (recommended):** Run `make help` to see all available commands for setup, development, building, and testing.
|
||||
|
||||
### Quick Start
|
||||
|
||||
**With Makefile (Unix/macOS/Linux):**
|
||||
|
||||
```bash
|
||||
# Clone the repo
|
||||
git clone https://github.com/jamiepine/voicebox.git
|
||||
cd voicebox
|
||||
|
||||
# Setup everything
|
||||
make setup
|
||||
|
||||
# Start development
|
||||
make dev
|
||||
just setup # creates Python venv, installs all deps
|
||||
just dev # starts backend + desktop app
|
||||
```
|
||||
|
||||
**Manual setup (all platforms):**
|
||||
Install [just](https://github.com/casey/just): `brew install just` or `cargo install just`. Run `just --list` to see all commands.
|
||||
|
||||
```bash
|
||||
# Clone the repo
|
||||
git clone https://github.com/jamiepine/voicebox.git
|
||||
cd voicebox
|
||||
Also available via Makefile: `make setup && make dev` (run `make help` for all commands).
|
||||
|
||||
# Install dependencies
|
||||
bun install
|
||||
|
||||
# Install Python dependencies
|
||||
cd backend && pip install -r requirements.txt && cd ..
|
||||
|
||||
# Start development
|
||||
bun run dev
|
||||
```
|
||||
|
||||
**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org). [XCode on macOS](https://developer.apple.com/xcode/).
|
||||
**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org), [XCode on macOS](https://developer.apple.com/xcode/).
|
||||
|
||||
**Performance:**
|
||||
- **Apple Silicon (M1/M2/M3)**: Uses MLX backend with native Metal acceleration for 4-5x faster inference
|
||||
|
||||
@@ -316,7 +316,7 @@ export function FloatingGenerateBox({
|
||||
</span>
|
||||
</div>
|
||||
<AnimatePresence>
|
||||
{isExpanded && (
|
||||
{isExpanded && form.watch('engine') !== 'luxtts' && (
|
||||
<motion.div
|
||||
initial={{ opacity: 0, scale: 0.8 }}
|
||||
animate={{ opacity: 1, scale: 1 }}
|
||||
@@ -402,30 +402,41 @@ export function FloatingGenerateBox({
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="modelSize"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="1.7B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 1.7B
|
||||
</SelectItem>
|
||||
<SelectItem value="0.6B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 0.6B
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage className="text-xs" />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
}
|
||||
}}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="qwen:1.7B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 1.7B
|
||||
</SelectItem>
|
||||
<SelectItem value="qwen:0.6B" className="text-xs text-muted-foreground">
|
||||
Qwen3-TTS 0.6B
|
||||
</SelectItem>
|
||||
<SelectItem value="luxtts" className="text-xs text-muted-foreground">
|
||||
LuxTTS
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
</div>
|
||||
</motion.div>
|
||||
</AnimatePresence>
|
||||
|
||||
@@ -76,29 +76,67 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Delivery Instructions (optional)</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
|
||||
className="min-h-[80px]"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Natural language instructions to control speech delivery (tone, emotion, pace).
|
||||
Max 500 characters
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
{form.watch('engine') !== 'luxtts' && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Delivery Instructions (optional)</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="e.g. Speak slowly with emphasis, Warm and friendly tone, Professional and authoritative..."
|
||||
className="min-h-[80px]"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Natural language instructions to control speech delivery (tone, emotion,
|
||||
pace). Max 500 characters
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid gap-4 md:grid-cols-3">
|
||||
<FormItem>
|
||||
<FormLabel>Model</FormLabel>
|
||||
<Select
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
}
|
||||
}}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="qwen:1.7B">Qwen3-TTS 1.7B</SelectItem>
|
||||
<SelectItem value="qwen:0.6B">Qwen3-TTS 0.6B</SelectItem>
|
||||
<SelectItem value="luxtts">LuxTTS</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{form.watch('engine') === 'luxtts'
|
||||
? 'Fast, English-focused'
|
||||
: 'Multi-language, two sizes'}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
@@ -124,29 +162,6 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="modelSize"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Model Size</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="1.7B">Qwen TTS 1.7B (Higher Quality)</SelectItem>
|
||||
<SelectItem value="0.6B">Qwen TTS 0.6B (Faster)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>Larger models produce better quality</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="seed"
|
||||
@@ -170,11 +185,7 @@ export function GenerationForm() {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
type="submit"
|
||||
className="w-full"
|
||||
disabled={isPending || !selectedProfileId}
|
||||
>
|
||||
<Button type="submit" className="w-full" disabled={isPending || !selectedProfileId}>
|
||||
{isPending ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
|
||||
@@ -80,16 +80,19 @@ export function ModelManagement() {
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
}, [queryClient]);
|
||||
|
||||
const handleDownloadError = useCallback((error: string) => {
|
||||
console.log('[ModelManagement] Download error, clearing state');
|
||||
if (downloadingModel) {
|
||||
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
|
||||
setConsoleOpen(true);
|
||||
}
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
}, [queryClient, downloadingModel]);
|
||||
const handleDownloadError = useCallback(
|
||||
(error: string) => {
|
||||
console.log('[ModelManagement] Download error, clearing state');
|
||||
if (downloadingModel) {
|
||||
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
|
||||
setConsoleOpen(true);
|
||||
}
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
},
|
||||
[queryClient, downloadingModel],
|
||||
);
|
||||
|
||||
// Use progress toast hook for the downloading model
|
||||
useModelDownloadToast({
|
||||
@@ -165,7 +168,11 @@ export function ModelManagement() {
|
||||
|
||||
// Optimistically hide the error and suppress downloading state in UI
|
||||
setDismissedErrors((prev) => new Set(prev).add(modelName));
|
||||
setLocalErrors((prev) => { const next = new Map(prev); next.delete(modelName); return next; });
|
||||
setLocalErrors((prev) => {
|
||||
const next = new Map(prev);
|
||||
next.delete(modelName);
|
||||
return next;
|
||||
});
|
||||
if (downloadingModel === modelName) {
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
@@ -178,7 +185,11 @@ export function ModelManagement() {
|
||||
setLocalErrors(prevLocalErrors);
|
||||
setDownloadingModel(prevDownloadingModel);
|
||||
setDownloadingDisplayName(prevDownloadingDisplayName);
|
||||
toast({ title: 'Cancel failed', description: 'Could not cancel the download task.', variant: 'destructive' });
|
||||
toast({
|
||||
title: 'Cancel failed',
|
||||
description: 'Could not cancel the download task.',
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -273,7 +284,9 @@ export function ModelManagement() {
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
@@ -282,6 +295,34 @@ export function ModelManagement() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* LuxTTS Models */}
|
||||
{modelStatus.models.some((m) => m.model_name.startsWith('luxtts')) && (
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">LuxTTS Models</h3>
|
||||
<div className="space-y-2">
|
||||
{modelStatus.models
|
||||
.filter((m) => m.model_name.startsWith('luxtts'))
|
||||
.map((model) => (
|
||||
<ModelItem
|
||||
key={model.model_name}
|
||||
model={model}
|
||||
onDownload={() => handleDownload(model.model_name)}
|
||||
onDelete={() => {
|
||||
setModelToDelete({
|
||||
name: model.model_name,
|
||||
displayName: model.display_name,
|
||||
sizeMb: model.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
formatSize={formatSize}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Whisper Models */}
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">
|
||||
@@ -305,7 +346,9 @@ export function ModelManagement() {
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={cancelMutation.isPending && cancelMutation.variables === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
@@ -353,12 +396,16 @@ export function ModelManagement() {
|
||||
{dl.error ? (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">{dl.error}</span>
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">
|
||||
{dl.error}
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#808080]">No error details available. Try downloading again.</span>
|
||||
<span className="text-[#808080]">
|
||||
No error details available. Try downloading again.
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
<div className="text-[#6a9955] mt-0.5">
|
||||
@@ -422,21 +469,31 @@ interface ModelItemProps {
|
||||
model_name: string;
|
||||
display_name: string;
|
||||
downloaded: boolean;
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
size_mb?: number;
|
||||
loaded: boolean;
|
||||
};
|
||||
onDownload: () => void;
|
||||
onDelete: () => void;
|
||||
onCancel: () => void;
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
isCancelling: boolean;
|
||||
isDismissed: boolean;
|
||||
erroredDownload?: ActiveDownloadTask;
|
||||
formatSize: (sizeMb?: number) => string;
|
||||
}
|
||||
|
||||
function ModelItem({ model, onDownload, onDelete, onCancel, isDownloading, isCancelling, isDismissed, erroredDownload, formatSize }: ModelItemProps) {
|
||||
function ModelItem({
|
||||
model,
|
||||
onDownload,
|
||||
onDelete,
|
||||
onCancel,
|
||||
isDownloading,
|
||||
isCancelling,
|
||||
isDismissed,
|
||||
erroredDownload,
|
||||
formatSize,
|
||||
}: ModelItemProps) {
|
||||
// Use server's downloading state OR local state (for immediate feedback before server updates)
|
||||
// Suppress downloading if user just dismissed/cancelled this model
|
||||
const showDownloading = (model.downloading || isDownloading) && !erroredDownload && !isDismissed;
|
||||
|
||||
@@ -34,6 +34,8 @@ export interface GenerationRequest {
|
||||
language: LanguageCode;
|
||||
seed?: number;
|
||||
model_size?: '1.7B' | '0.6B';
|
||||
engine?: 'qwen' | 'luxtts';
|
||||
instruct?: string;
|
||||
}
|
||||
|
||||
export interface GenerationResponse {
|
||||
|
||||
@@ -16,6 +16,7 @@ const generationSchema = z.object({
|
||||
seed: z.number().int().optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B']).optional(),
|
||||
instruct: z.string().max(500).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts']).optional(),
|
||||
});
|
||||
|
||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||
@@ -47,6 +48,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
seed: undefined,
|
||||
modelSize: '1.7B',
|
||||
instruct: '',
|
||||
engine: 'qwen',
|
||||
...options.defaultValues,
|
||||
},
|
||||
});
|
||||
@@ -67,8 +69,14 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
try {
|
||||
setIsGenerating(true);
|
||||
|
||||
const modelName = `qwen-tts-${data.modelSize}`;
|
||||
const displayName = data.modelSize === '1.7B' ? 'Qwen TTS 1.7B' : 'Qwen TTS 0.6B';
|
||||
const engine = data.engine || 'qwen';
|
||||
const modelName = engine === 'luxtts' ? 'luxtts' : `qwen-tts-${data.modelSize}`;
|
||||
const displayName =
|
||||
engine === 'luxtts'
|
||||
? 'LuxTTS'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
|
||||
try {
|
||||
const modelStatus = await apiClient.getModelStatus();
|
||||
@@ -87,8 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
text: data.text,
|
||||
language: data.language,
|
||||
seed: data.seed,
|
||||
model_size: data.modelSize,
|
||||
instruct: data.instruct || undefined,
|
||||
model_size: engine === 'luxtts' ? undefined : data.modelSize,
|
||||
engine,
|
||||
instruct: engine === 'luxtts' ? undefined : data.instruct || undefined,
|
||||
});
|
||||
|
||||
toast({
|
||||
@@ -99,7 +108,14 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
const audioUrl = apiClient.getAudioUrl(result.id);
|
||||
setAudioWithAutoPlay(audioUrl, result.id, selectedProfileId, data.text.substring(0, 50));
|
||||
|
||||
form.reset();
|
||||
form.reset({
|
||||
text: '',
|
||||
language: data.language,
|
||||
seed: undefined,
|
||||
modelSize: data.modelSize,
|
||||
instruct: '',
|
||||
engine: data.engine,
|
||||
});
|
||||
options.onSuccess?.(result.id);
|
||||
} catch (error) {
|
||||
toast({
|
||||
|
||||
@@ -4,6 +4,7 @@ Backend abstraction layer for TTS and STT.
|
||||
Provides a unified interface for MLX and PyTorch backends.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from typing import Protocol, Optional, Tuple, List
|
||||
from typing_extensions import runtime_checkable
|
||||
import numpy as np
|
||||
@@ -112,29 +113,65 @@ class STTBackend(Protocol):
|
||||
|
||||
# Global backend instances
|
||||
_tts_backend: Optional[TTSBackend] = None
|
||||
_tts_backends: dict[str, TTSBackend] = {}
|
||||
_tts_backends_lock = threading.Lock()
|
||||
_stt_backend: Optional[STTBackend] = None
|
||||
|
||||
# Supported TTS engines
|
||||
TTS_ENGINES = {
|
||||
"qwen": "Qwen TTS",
|
||||
"luxtts": "LuxTTS",
|
||||
}
|
||||
|
||||
|
||||
def get_tts_backend() -> TTSBackend:
|
||||
"""
|
||||
Get or create TTS backend instance based on platform.
|
||||
Get or create the default (Qwen) TTS backend instance based on platform.
|
||||
|
||||
Returns:
|
||||
TTS backend instance (MLX or PyTorch)
|
||||
"""
|
||||
global _tts_backend
|
||||
return get_tts_backend_for_engine("qwen")
|
||||
|
||||
|
||||
def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
"""
|
||||
Get or create a TTS backend for the given engine.
|
||||
|
||||
if _tts_backend is None:
|
||||
backend_type = get_backend_type()
|
||||
Args:
|
||||
engine: Engine name ("qwen" or "luxtts")
|
||||
|
||||
Returns:
|
||||
TTS backend instance
|
||||
"""
|
||||
global _tts_backends
|
||||
|
||||
# Fast path: check without lock
|
||||
if engine in _tts_backends:
|
||||
return _tts_backends[engine]
|
||||
|
||||
# Slow path: create with lock to avoid duplicate instantiation
|
||||
with _tts_backends_lock:
|
||||
# Double-check after acquiring lock
|
||||
if engine in _tts_backends:
|
||||
return _tts_backends[engine]
|
||||
|
||||
if backend_type == "mlx":
|
||||
from .mlx_backend import MLXTTSBackend
|
||||
_tts_backend = MLXTTSBackend()
|
||||
if engine == "qwen":
|
||||
backend_type = get_backend_type()
|
||||
if backend_type == "mlx":
|
||||
from .mlx_backend import MLXTTSBackend
|
||||
backend = MLXTTSBackend()
|
||||
else:
|
||||
from .pytorch_backend import PyTorchTTSBackend
|
||||
backend = PyTorchTTSBackend()
|
||||
elif engine == "luxtts":
|
||||
from .luxtts_backend import LuxTTSBackend
|
||||
backend = LuxTTSBackend()
|
||||
else:
|
||||
from .pytorch_backend import PyTorchTTSBackend
|
||||
_tts_backend = PyTorchTTSBackend()
|
||||
|
||||
return _tts_backend
|
||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||
|
||||
_tts_backends[engine] = backend
|
||||
return backend
|
||||
|
||||
|
||||
def get_stt_backend() -> STTBackend:
|
||||
@@ -161,6 +198,7 @@ def get_stt_backend() -> STTBackend:
|
||||
|
||||
def reset_backends():
|
||||
"""Reset backend instances (useful for testing)."""
|
||||
global _tts_backend, _stt_backend
|
||||
global _tts_backend, _tts_backends, _stt_backend
|
||||
_tts_backend = None
|
||||
_tts_backends.clear()
|
||||
_stt_backend = None
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
LuxTTS backend implementation.
|
||||
|
||||
Wraps the LuxTTS (ZipVoice) model for zero-shot voice cloning.
|
||||
~1GB VRAM, 48kHz output, 150x realtime on CPU.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import TTSBackend
|
||||
from ..utils.audio import normalize_audio, load_audio
|
||||
from ..utils.cache import get_cache_key, get_cached_voice_prompt, cache_voice_prompt
|
||||
from ..utils.progress import get_progress_manager
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# HuggingFace repo for model weight detection
|
||||
LUXTTS_HF_REPO = "YatharthS/LuxTTS"
|
||||
|
||||
|
||||
class LuxTTSBackend:
|
||||
"""LuxTTS backend for zero-shot voice cloning."""
|
||||
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
self.model_size = "default" # LuxTTS has only one model size
|
||||
self._device = None
|
||||
|
||||
def _get_device(self) -> str:
|
||||
"""Get the best available device."""
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
||||
return "mps"
|
||||
return "cpu"
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
return self.model is not None
|
||||
|
||||
@property
|
||||
def device(self) -> str:
|
||||
if self._device is None:
|
||||
self._device = self._get_device()
|
||||
return self._device
|
||||
|
||||
def _get_model_path(self, model_size: str) -> str:
|
||||
return LUXTTS_HF_REPO
|
||||
|
||||
def _is_model_cached(self, model_size: str = "default") -> bool:
|
||||
"""Check if LuxTTS model weights are cached locally."""
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
repo_cache = (
|
||||
Path(hf_constants.HF_HUB_CACHE)
|
||||
/ ("models--" + LUXTTS_HF_REPO.replace("/", "--"))
|
||||
)
|
||||
|
||||
if not repo_cache.exists():
|
||||
return False
|
||||
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
|
||||
return False
|
||||
|
||||
snapshots_dir = repo_cache / "snapshots"
|
||||
if snapshots_dir.exists():
|
||||
has_weights = any(snapshots_dir.rglob("*.pt")) or any(
|
||||
snapshots_dir.rglob("*.safetensors")
|
||||
) or any(snapshots_dir.rglob("*.onnx")) or any(
|
||||
snapshots_dir.rglob("*.bin")
|
||||
)
|
||||
return has_weights
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking LuxTTS cache: {e}")
|
||||
return False
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None:
|
||||
"""Load the LuxTTS model."""
|
||||
if self.model is not None:
|
||||
return
|
||||
|
||||
await asyncio.to_thread(self._load_model_sync)
|
||||
|
||||
def _load_model_sync(self):
|
||||
"""Synchronous model loading."""
|
||||
progress_manager = get_progress_manager()
|
||||
task_manager = get_task_manager()
|
||||
model_name = "luxtts"
|
||||
|
||||
is_cached = self._is_model_cached()
|
||||
|
||||
if not is_cached:
|
||||
task_manager.start_download(model_name)
|
||||
progress_manager.update_progress(
|
||||
model_name=model_name,
|
||||
current=0,
|
||||
total=0,
|
||||
filename="Downloading LuxTTS model...",
|
||||
status="downloading",
|
||||
)
|
||||
|
||||
try:
|
||||
from zipvoice.luxvoice import LuxTTS
|
||||
|
||||
device = self.device
|
||||
logger.info(f"Loading LuxTTS on {device}...")
|
||||
|
||||
# LuxTTS constructor downloads model and loads everything
|
||||
if device == "cpu":
|
||||
import os
|
||||
threads = os.cpu_count() or 4
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device="cpu",
|
||||
threads=min(threads, 8),
|
||||
)
|
||||
else:
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device=device,
|
||||
)
|
||||
|
||||
if not is_cached:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
logger.info("LuxTTS loaded successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load LuxTTS: {e}")
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
if self.model is not None:
|
||||
del self.model
|
||||
self.model = None
|
||||
|
||||
import torch
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
logger.info("LuxTTS unloaded")
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
reference_text: str,
|
||||
use_cache: bool = True,
|
||||
) -> Tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt from reference audio.
|
||||
|
||||
LuxTTS uses its own encode_prompt() which runs Whisper ASR internally
|
||||
to transcribe the reference. The reference_text parameter is not used
|
||||
by LuxTTS itself, but we include it in the cache key for consistency.
|
||||
"""
|
||||
await self.load_model()
|
||||
|
||||
# Compute cache key once for both lookup and storage
|
||||
cache_key = ("luxtts_" + get_cache_key(audio_path, reference_text)) if use_cache else None
|
||||
|
||||
if cache_key:
|
||||
cached = get_cached_voice_prompt(cache_key)
|
||||
if cached is not None and isinstance(cached, dict):
|
||||
return cached, True
|
||||
|
||||
def _encode_sync():
|
||||
return self.model.encode_prompt(
|
||||
prompt_audio=str(audio_path),
|
||||
duration=5,
|
||||
rms=0.01,
|
||||
)
|
||||
|
||||
encoded = await asyncio.to_thread(_encode_sync)
|
||||
|
||||
if cache_key:
|
||||
cache_voice_prompt(cache_key, encoded)
|
||||
|
||||
return encoded, False
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: List[str],
|
||||
reference_texts: List[str],
|
||||
) -> Tuple[np.ndarray, str]:
|
||||
"""
|
||||
Combine multiple reference samples.
|
||||
|
||||
LuxTTS doesn't have native multi-prompt support, so we concatenate
|
||||
the audio and let encode_prompt handle the combined clip.
|
||||
"""
|
||||
combined_audio = []
|
||||
for path in audio_paths:
|
||||
audio, _sr = load_audio(path, sample_rate=24000)
|
||||
audio = normalize_audio(audio)
|
||||
combined_audio.append(audio)
|
||||
|
||||
mixed = np.concatenate(combined_audio)
|
||||
mixed = normalize_audio(mixed)
|
||||
combined_text = " ".join(reference_texts)
|
||||
|
||||
return mixed, combined_text
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: Optional[int] = None,
|
||||
instruct: Optional[str] = None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio from text using LuxTTS.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize
|
||||
voice_prompt: Encoded prompt dict from encode_prompt()
|
||||
language: Language code (LuxTTS is English-focused)
|
||||
seed: Random seed for reproducibility
|
||||
instruct: Not supported by LuxTTS (ignored)
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
await self.load_model()
|
||||
|
||||
def _generate_sync():
|
||||
import torch
|
||||
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
wav = self.model.generate_speech(
|
||||
text=text,
|
||||
encode_dict=voice_prompt,
|
||||
num_steps=4,
|
||||
guidance_scale=3.0,
|
||||
t_shift=0.5,
|
||||
speed=1.0,
|
||||
return_smooth=False, # 48kHz output
|
||||
)
|
||||
|
||||
# LuxTTS returns a tensor (may be on GPU/MPS), move to CPU first
|
||||
audio = wav.detach().cpu().numpy().squeeze()
|
||||
return audio, 48000
|
||||
|
||||
return await asyncio.to_thread(_generate_sync)
|
||||
+118
-42
@@ -48,6 +48,18 @@ from .utils.tasks import get_task_manager
|
||||
from .utils.cache import clear_voice_prompt_cache
|
||||
from .platform_detect import get_backend_type
|
||||
|
||||
# Keep references to fire-and-forget background tasks to prevent GC
|
||||
_background_tasks: set = set()
|
||||
|
||||
|
||||
def _create_background_task(coro) -> asyncio.Task:
|
||||
"""Create a background task and prevent it from being garbage collected."""
|
||||
task = asyncio.create_task(coro)
|
||||
_background_tasks.add(task)
|
||||
task.add_done_callback(_background_tasks.discard)
|
||||
return task
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="voicebox API",
|
||||
description="Production-quality Qwen3-TTS voice cloning API",
|
||||
@@ -608,47 +620,69 @@ async def generate_speech(
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
|
||||
# Generate audio
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
# Resolve model size and load the correct model FIRST.
|
||||
# This must happen before create_voice_prompt_for_profile because that
|
||||
# function calls load_model_async(None), which falls back to self.model_size.
|
||||
# If the model is already loaded with the right size at that point, it
|
||||
# returns immediately and the voice prompt is created by the correct model.
|
||||
tts_model = tts.get_tts_model()
|
||||
engine = data.engine or "qwen"
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
|
||||
# Resolve model size (only relevant for Qwen engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
|
||||
# Check if model needs to be downloaded first
|
||||
model_path = tts_model._get_model_path(model_size)
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
# Model is not fully cached — kick off a background download and tell
|
||||
# the client to retry once it's ready.
|
||||
model_name = f"qwen-tts-{model_size}"
|
||||
if engine == "qwen":
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
model_name = f"qwen-tts-{model_size}"
|
||||
|
||||
async def download_model_background():
|
||||
try:
|
||||
await tts_model.load_model_async(model_size)
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
async def download_model_background():
|
||||
try:
|
||||
await tts_model.load_model_async(model_size)
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
asyncio.create_task(download_model_background())
|
||||
task_manager.start_download(model_name)
|
||||
_create_background_task(download_model_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": f"Model {model_size} is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": f"Model {model_size} is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
# Load (or switch to) the requested model before building the voice prompt
|
||||
await tts_model.load_model_async(model_size)
|
||||
# Load (or switch to) the requested model
|
||||
await tts_model.load_model_async(model_size)
|
||||
elif engine == "luxtts":
|
||||
if not tts_model._is_model_cached():
|
||||
model_name = "luxtts"
|
||||
|
||||
# Create voice prompt from profile (model is already loaded with correct size)
|
||||
async def download_luxtts_background():
|
||||
try:
|
||||
await tts_model.load_model()
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
_create_background_task(download_luxtts_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": "LuxTTS model is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
|
||||
# Create voice prompt from profile
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id,
|
||||
db,
|
||||
use_cache=True,
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
audio, sample_rate = await tts_model.generate(
|
||||
@@ -705,23 +739,34 @@ async def stream_speech(
|
||||
playing audio before the entire file has been received. This endpoint
|
||||
does NOT create a history entry — use /generate for that.
|
||||
"""
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
profile = await profiles.get_profile(data.profile_id, db)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
|
||||
tts_model = tts.get_tts_model()
|
||||
engine = data.engine or "qwen"
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
if engine == "qwen":
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model_async(model_size)
|
||||
elif engine == "luxtts":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
|
||||
# Load the correct model before building the voice prompt (fixes issue #96)
|
||||
await tts_model.load_model_async(model_size)
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(data.profile_id, db)
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id, db, engine=engine,
|
||||
)
|
||||
|
||||
audio, sample_rate = await tts_model.generate(
|
||||
data.text,
|
||||
@@ -959,7 +1004,7 @@ async def transcribe_audio(
|
||||
get_task_manager().error_download(progress_model_name, str(e))
|
||||
|
||||
get_task_manager().start_download(progress_model_name)
|
||||
asyncio.create_task(download_whisper_background())
|
||||
_create_background_task(download_whisper_background())
|
||||
|
||||
# Return 202 Accepted
|
||||
raise HTTPException(
|
||||
@@ -1330,6 +1375,15 @@ async def get_model_status():
|
||||
whisper_medium_id = "openai/whisper-medium"
|
||||
whisper_large_id = "openai/whisper-large-v3"
|
||||
|
||||
# Check if LuxTTS backend is loaded
|
||||
def check_luxtts_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("luxtts")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
model_configs = [
|
||||
{
|
||||
"model_name": "qwen-tts-1.7B",
|
||||
@@ -1345,6 +1399,13 @@ async def get_model_status():
|
||||
"model_size": "0.6B",
|
||||
"check_loaded": lambda: check_tts_loaded("0.6B"),
|
||||
},
|
||||
{
|
||||
"model_name": "luxtts",
|
||||
"display_name": "LuxTTS (Fast, CPU-friendly)",
|
||||
"hf_repo_id": "YatharthS/LuxTTS",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_luxtts_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-base",
|
||||
"display_name": "Whisper Base",
|
||||
@@ -1527,6 +1588,7 @@ async def get_model_status():
|
||||
async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"""Trigger download of a specific model."""
|
||||
import asyncio
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
task_manager = get_task_manager()
|
||||
progress_manager = get_progress_manager()
|
||||
@@ -1540,6 +1602,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"model_size": "0.6B",
|
||||
"load_func": lambda: tts.get_tts_model().load_model("0.6B"),
|
||||
},
|
||||
"luxtts": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
|
||||
},
|
||||
"whisper-base": {
|
||||
"model_size": "base",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
|
||||
@@ -1591,7 +1657,7 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
)
|
||||
|
||||
# Start download in background task (don't await)
|
||||
asyncio.create_task(download_in_background())
|
||||
_create_background_task(download_in_background())
|
||||
|
||||
# Return immediately - frontend should poll progress endpoint
|
||||
return {"message": f"Model {request.model_name} download started"}
|
||||
@@ -1652,6 +1718,11 @@ async def delete_model(model_name: str):
|
||||
"model_size": "0.6B",
|
||||
"model_type": "tts",
|
||||
},
|
||||
"luxtts": {
|
||||
"hf_repo_id": "YatharthS/LuxTTS",
|
||||
"model_size": "default",
|
||||
"model_type": "luxtts",
|
||||
},
|
||||
"whisper-base": {
|
||||
"hf_repo_id": "openai/whisper-base",
|
||||
"model_size": "base",
|
||||
@@ -1686,6 +1757,11 @@ async def delete_model(model_name: str):
|
||||
tts_model = tts.get_tts_model()
|
||||
if tts_model.is_loaded() and tts_model.model_size == config["model_size"]:
|
||||
tts.unload_tts_model()
|
||||
elif config["model_type"] == "luxtts":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
luxtts = get_tts_backend_for_engine("luxtts")
|
||||
if luxtts.is_loaded():
|
||||
luxtts.unload_model()
|
||||
elif config["model_type"] == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:
|
||||
@@ -1831,7 +1907,7 @@ async def download_cuda_backend():
|
||||
import logging
|
||||
logging.getLogger(__name__).error(f"CUDA download failed: {e}")
|
||||
|
||||
asyncio.create_task(_download())
|
||||
_create_background_task(_download())
|
||||
return {"message": "CUDA backend download started", "progress_key": "cuda-backend"}
|
||||
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ class GenerationRequest(BaseModel):
|
||||
seed: Optional[int] = Field(None, ge=0)
|
||||
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
|
||||
instruct: Optional[str] = Field(None, max_length=500)
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts)$")
|
||||
|
||||
|
||||
class GenerationResponse(BaseModel):
|
||||
|
||||
+5
-1
@@ -344,6 +344,7 @@ async def create_voice_prompt_for_profile(
|
||||
profile_id: str,
|
||||
db: Session,
|
||||
use_cache: bool = True,
|
||||
engine: str = "qwen",
|
||||
) -> dict:
|
||||
"""
|
||||
Create a combined voice prompt from all samples in a profile.
|
||||
@@ -352,17 +353,20 @@ async def create_voice_prompt_for_profile(
|
||||
profile_id: Profile ID
|
||||
db: Database session
|
||||
use_cache: Whether to use cached prompts
|
||||
engine: TTS engine to create prompt for ("qwen" or "luxtts")
|
||||
|
||||
Returns:
|
||||
Voice prompt dictionary
|
||||
"""
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
# Get all samples for profile
|
||||
samples = db.query(DBProfileSample).filter_by(profile_id=profile_id).all()
|
||||
|
||||
if not samples:
|
||||
raise ValueError(f"No samples found for profile {profile_id}")
|
||||
|
||||
tts_model = get_tts_model()
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
|
||||
if len(samples) == 1:
|
||||
# Single sample - use directly
|
||||
|
||||
@@ -9,11 +9,18 @@ alembic>=1.13.0
|
||||
|
||||
# ML models
|
||||
torch>=2.1.0
|
||||
transformers>=4.36.0
|
||||
transformers>=4.36.0,<=4.57.6
|
||||
accelerate>=0.26.0
|
||||
huggingface_hub>=0.20.0
|
||||
qwen-tts>=0.0.5
|
||||
|
||||
# LuxTTS (voice cloning engine)
|
||||
# piper-phonemize needs custom index (no PyPI wheels)
|
||||
--find-links https://k2-fsa.github.io/icefall/piper_phonemize.html
|
||||
# linacodec is a git-only dep of Zipvoice (uv-only source, pip can't resolve it)
|
||||
linacodec @ git+https://github.com/ysharma3501/LinaCodec.git
|
||||
Zipvoice @ git+https://github.com/ysharma3501/LuxTTS.git
|
||||
|
||||
# Audio processing
|
||||
librosa>=0.10.0
|
||||
soundfile>=0.12.0
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# Voicebox development commands
|
||||
# Install: brew install just (or cargo install just)
|
||||
# Usage: just --list
|
||||
|
||||
# Directories
|
||||
backend_dir := "backend"
|
||||
tauri_dir := "tauri"
|
||||
app_dir := "app"
|
||||
web_dir := "web"
|
||||
venv := backend_dir / "venv"
|
||||
venv_bin := venv / "bin"
|
||||
python := venv_bin / "python"
|
||||
pip := venv_bin / "pip"
|
||||
|
||||
# Detect best python for venv creation
|
||||
system_python := `command -v python3.12 2>/dev/null || command -v python3.13 2>/dev/null || echo python3`
|
||||
|
||||
# ─── Setup ────────────────────────────────────────────────────────────
|
||||
|
||||
# Full project setup (python venv + JS deps + dev sidecar)
|
||||
setup: setup-python setup-js
|
||||
@echo ""
|
||||
@echo "Setup complete! Run: just dev"
|
||||
|
||||
# Create venv and install Python dependencies
|
||||
setup-python:
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if [ ! -d "{{ venv }}" ]; then
|
||||
echo "Creating Python virtual environment..."
|
||||
PY_MINOR=$({{ system_python }} -c "import sys; print(sys.version_info[1])")
|
||||
if [ "$PY_MINOR" -gt 13 ]; then
|
||||
echo "Warning: Python 3.$PY_MINOR detected. ML packages may not be compatible."
|
||||
echo "Recommended: brew install [email protected]"
|
||||
fi
|
||||
{{ system_python }} -m venv {{ venv }}
|
||||
fi
|
||||
echo "Installing Python dependencies..."
|
||||
{{ pip }} install --upgrade pip -q
|
||||
{{ pip }} install -r {{ backend_dir }}/requirements.txt
|
||||
# Apple Silicon: install MLX backend
|
||||
if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
|
||||
echo "Detected Apple Silicon — installing MLX dependencies..."
|
||||
{{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt
|
||||
fi
|
||||
{{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
echo "Python environment ready."
|
||||
|
||||
# Install JavaScript dependencies
|
||||
setup-js:
|
||||
bun install
|
||||
|
||||
# ─── Development ──────────────────────────────────────────────────────
|
||||
|
||||
# Start backend + frontend for development (two processes, one terminal)
|
||||
dev: _ensure-venv _ensure-sidecar
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
trap 'kill 0' EXIT
|
||||
|
||||
echo "Starting backend on http://localhost:17493 ..."
|
||||
{{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 &
|
||||
sleep 2
|
||||
|
||||
echo "Starting Tauri desktop app..."
|
||||
cd {{ tauri_dir }} && bun run tauri dev &
|
||||
|
||||
wait
|
||||
|
||||
# Start backend only
|
||||
dev-backend: _ensure-venv
|
||||
{{ venv_bin }}/uvicorn backend.main:app --reload --port 17493
|
||||
|
||||
# Start Tauri desktop app only (backend must be running separately)
|
||||
dev-frontend: _ensure-sidecar
|
||||
cd {{ tauri_dir }} && bun run tauri dev
|
||||
|
||||
# Start backend + web app (no Tauri)
|
||||
dev-web: _ensure-venv
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
trap 'kill 0' EXIT
|
||||
{{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 &
|
||||
sleep 2
|
||||
cd {{ web_dir }} && bun run dev &
|
||||
wait
|
||||
|
||||
# Kill all dev processes
|
||||
kill:
|
||||
-pkill -f "uvicorn backend.main:app" 2>/dev/null || true
|
||||
-pkill -f "vite" 2>/dev/null || true
|
||||
@echo "Dev processes killed."
|
||||
|
||||
# ─── Build ────────────────────────────────────────────────────────────
|
||||
|
||||
# Build everything (server binary + desktop app)
|
||||
build: build-server build-tauri
|
||||
|
||||
# Build Python server binary
|
||||
build-server: _ensure-venv
|
||||
PATH="{{ venv_bin }}:$PATH" ./scripts/build-server.sh
|
||||
|
||||
# Build Tauri desktop app
|
||||
build-tauri:
|
||||
cd {{ tauri_dir }} && bun run tauri build
|
||||
|
||||
# Build web app
|
||||
build-web:
|
||||
cd {{ web_dir }} && bun run build
|
||||
|
||||
# ─── Code Quality ────────────────────────────────────────────────────
|
||||
|
||||
# Run all checks (lint + format + typecheck)
|
||||
check:
|
||||
bun run check
|
||||
|
||||
# Lint with Biome
|
||||
lint:
|
||||
bun run lint
|
||||
|
||||
# Format with Biome
|
||||
format:
|
||||
bun run format
|
||||
|
||||
# Fix lint + format issues
|
||||
fix:
|
||||
bun run check:fix
|
||||
|
||||
# ─── Database ─────────────────────────────────────────────────────────
|
||||
|
||||
# Initialize SQLite database
|
||||
db-init: _ensure-venv
|
||||
cd {{ backend_dir }} && {{ python }} -c "from database import init_db; init_db()"
|
||||
|
||||
# Reset database (delete + reinit)
|
||||
db-reset:
|
||||
rm -f {{ backend_dir }}/data/voicebox.db
|
||||
just db-init
|
||||
|
||||
# ─── Utilities ────────────────────────────────────────────────────────
|
||||
|
||||
# Generate TypeScript API client (backend must be running)
|
||||
generate-api:
|
||||
./scripts/generate-api.sh
|
||||
|
||||
# Open API docs in browser
|
||||
docs:
|
||||
open http://localhost:17493/docs 2>/dev/null || xdg-open http://localhost:17493/docs
|
||||
|
||||
# Tail backend logs
|
||||
logs:
|
||||
tail -f {{ backend_dir }}/logs/*.log 2>/dev/null || echo "No log files found"
|
||||
|
||||
# ─── Clean ────────────────────────────────────────────────────────────
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
rm -rf {{ tauri_dir }}/src-tauri/target/release
|
||||
rm -rf {{ web_dir }}/dist
|
||||
rm -rf {{ app_dir }}/dist
|
||||
|
||||
# Clean Python venv and cache
|
||||
clean-python:
|
||||
rm -rf {{ venv }}
|
||||
find {{ backend_dir }} -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
# Nuclear clean (everything including node_modules)
|
||||
clean-all: clean clean-python
|
||||
rm -rf node_modules
|
||||
rm -rf {{ app_dir }}/node_modules
|
||||
rm -rf {{ tauri_dir }}/node_modules
|
||||
rm -rf {{ web_dir }}/node_modules
|
||||
cd {{ tauri_dir }}/src-tauri && cargo clean
|
||||
|
||||
# ─── Internal ─────────────────────────────────────────────────────────
|
||||
|
||||
# Ensure venv exists (prompt to run setup if not)
|
||||
[private]
|
||||
_ensure-venv:
|
||||
#!/usr/bin/env bash
|
||||
if [ ! -d "{{ venv }}" ]; then
|
||||
echo "Python venv not found. Run: just setup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure Tauri dev sidecar placeholder exists
|
||||
[private]
|
||||
_ensure-sidecar:
|
||||
bun run setup:dev
|
||||
+271
-56
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Creates placeholder sidecar binaries for development mode.
|
||||
*
|
||||
@@ -9,10 +10,10 @@
|
||||
* The actual server should be started separately with `bun run dev:server`.
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, writeFileSync, statSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { execSync } from 'child_process';
|
||||
import { existsSync, mkdirSync, statSync, writeFileSync } from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -55,7 +56,9 @@ function createPlaceholderBinary(targetTriple) {
|
||||
try {
|
||||
const stats = statSync(binaryPath);
|
||||
if (stats.size > MIN_REAL_BINARY_SIZE) {
|
||||
console.log(`Real binary already exists: ${binaryName} (${(stats.size / 1024 / 1024).toFixed(1)} MB)`);
|
||||
console.log(
|
||||
`Real binary already exists: ${binaryName} (${(stats.size / 1024 / 1024).toFixed(1)} MB)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
@@ -73,52 +76,275 @@ function createPlaceholderBinary(targetTriple) {
|
||||
// This is the smallest valid PE that Windows will accept
|
||||
const minimalPE = Buffer.from([
|
||||
// DOS Header
|
||||
0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00,
|
||||
0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
|
||||
0x4d,
|
||||
0x5a,
|
||||
0x90,
|
||||
0x00,
|
||||
0x03,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x04,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0xff,
|
||||
0xff,
|
||||
0x00,
|
||||
0x00,
|
||||
0xb8,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x40,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x80,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
// DOS Stub
|
||||
0x0E, 0x1F, 0xBA, 0x0E, 0x00, 0xB4, 0x09, 0xCD, 0x21, 0xB8, 0x01, 0x4C, 0xCD, 0x21, 0x54, 0x68,
|
||||
0x69, 0x73, 0x20, 0x70, 0x72, 0x6F, 0x67, 0x72, 0x61, 0x6D, 0x20, 0x63, 0x61, 0x6E, 0x6E, 0x6F,
|
||||
0x74, 0x20, 0x62, 0x65, 0x20, 0x72, 0x75, 0x6E, 0x20, 0x69, 0x6E, 0x20, 0x44, 0x4F, 0x53, 0x20,
|
||||
0x6D, 0x6F, 0x64, 0x65, 0x2E, 0x0D, 0x0D, 0x0A, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x0e,
|
||||
0x1f,
|
||||
0xba,
|
||||
0x0e,
|
||||
0x00,
|
||||
0xb4,
|
||||
0x09,
|
||||
0xcd,
|
||||
0x21,
|
||||
0xb8,
|
||||
0x01,
|
||||
0x4c,
|
||||
0xcd,
|
||||
0x21,
|
||||
0x54,
|
||||
0x68,
|
||||
0x69,
|
||||
0x73,
|
||||
0x20,
|
||||
0x70,
|
||||
0x72,
|
||||
0x6f,
|
||||
0x67,
|
||||
0x72,
|
||||
0x61,
|
||||
0x6d,
|
||||
0x20,
|
||||
0x63,
|
||||
0x61,
|
||||
0x6e,
|
||||
0x6e,
|
||||
0x6f,
|
||||
0x74,
|
||||
0x20,
|
||||
0x62,
|
||||
0x65,
|
||||
0x20,
|
||||
0x72,
|
||||
0x75,
|
||||
0x6e,
|
||||
0x20,
|
||||
0x69,
|
||||
0x6e,
|
||||
0x20,
|
||||
0x44,
|
||||
0x4f,
|
||||
0x53,
|
||||
0x20,
|
||||
0x6d,
|
||||
0x6f,
|
||||
0x64,
|
||||
0x65,
|
||||
0x2e,
|
||||
0x0d,
|
||||
0x0d,
|
||||
0x0a,
|
||||
0x24,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
// PE Signature
|
||||
0x50, 0x45, 0x00, 0x00,
|
||||
0x50,
|
||||
0x45,
|
||||
0x00,
|
||||
0x00,
|
||||
// COFF Header (x64)
|
||||
0x64, 0x86, // Machine: AMD64
|
||||
0x01, 0x00, // NumberOfSections: 1
|
||||
0x00, 0x00, 0x00, 0x00, // TimeDateStamp
|
||||
0x00, 0x00, 0x00, 0x00, // PointerToSymbolTable
|
||||
0x00, 0x00, 0x00, 0x00, // NumberOfSymbols
|
||||
0xF0, 0x00, // SizeOfOptionalHeader
|
||||
0x22, 0x00, // Characteristics: EXECUTABLE_IMAGE | LARGE_ADDRESS_AWARE
|
||||
0x64,
|
||||
0x86, // Machine: AMD64
|
||||
0x01,
|
||||
0x00, // NumberOfSections: 1
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // TimeDateStamp
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // PointerToSymbolTable
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // NumberOfSymbols
|
||||
0xf0,
|
||||
0x00, // SizeOfOptionalHeader
|
||||
0x22,
|
||||
0x00, // Characteristics: EXECUTABLE_IMAGE | LARGE_ADDRESS_AWARE
|
||||
// Optional Header (PE32+)
|
||||
0x0B, 0x02, // Magic: PE32+
|
||||
0x00, 0x00, // Linker version
|
||||
0x00, 0x00, 0x00, 0x00, // SizeOfCode
|
||||
0x00, 0x00, 0x00, 0x00, // SizeOfInitializedData
|
||||
0x00, 0x00, 0x00, 0x00, // SizeOfUninitializedData
|
||||
0x00, 0x10, 0x00, 0x00, // AddressOfEntryPoint
|
||||
0x00, 0x00, 0x00, 0x00, // BaseOfCode
|
||||
0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0x00, // ImageBase
|
||||
0x00, 0x10, 0x00, 0x00, // SectionAlignment
|
||||
0x00, 0x02, 0x00, 0x00, // FileAlignment
|
||||
0x06, 0x00, 0x00, 0x00, // OS version
|
||||
0x00, 0x00, 0x00, 0x00, // Image version
|
||||
0x06, 0x00, 0x00, 0x00, // Subsystem version
|
||||
0x00, 0x00, 0x00, 0x00, // Win32VersionValue
|
||||
0x00, 0x20, 0x00, 0x00, // SizeOfImage
|
||||
0x00, 0x02, 0x00, 0x00, // SizeOfHeaders
|
||||
0x00, 0x00, 0x00, 0x00, // CheckSum
|
||||
0x03, 0x00, // Subsystem: CONSOLE
|
||||
0x60, 0x01, // DllCharacteristics
|
||||
0x0b,
|
||||
0x02, // Magic: PE32+
|
||||
0x00,
|
||||
0x00, // Linker version
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // SizeOfCode
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // SizeOfInitializedData
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // SizeOfUninitializedData
|
||||
0x00,
|
||||
0x10,
|
||||
0x00,
|
||||
0x00, // AddressOfEntryPoint
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // BaseOfCode
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x40,
|
||||
0x01,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // ImageBase
|
||||
0x00,
|
||||
0x10,
|
||||
0x00,
|
||||
0x00, // SectionAlignment
|
||||
0x00,
|
||||
0x02,
|
||||
0x00,
|
||||
0x00, // FileAlignment
|
||||
0x06,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // OS version
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // Image version
|
||||
0x06,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // Subsystem version
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // Win32VersionValue
|
||||
0x00,
|
||||
0x20,
|
||||
0x00,
|
||||
0x00, // SizeOfImage
|
||||
0x00,
|
||||
0x02,
|
||||
0x00,
|
||||
0x00, // SizeOfHeaders
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // CheckSum
|
||||
0x03,
|
||||
0x00, // Subsystem: CONSOLE
|
||||
0x60,
|
||||
0x01, // DllCharacteristics
|
||||
// Stack/Heap sizes (8 bytes each for PE32+)
|
||||
0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, // LoaderFlags
|
||||
0x10, 0x00, 0x00, 0x00, // NumberOfRvaAndSizes
|
||||
0x00,
|
||||
0x00,
|
||||
0x10,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x10,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // LoaderFlags
|
||||
0x10,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00, // NumberOfRvaAndSizes
|
||||
]);
|
||||
|
||||
// Pad to 512 bytes minimum for valid PE
|
||||
@@ -138,19 +364,8 @@ exit 1
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('Setting up development sidecar...');
|
||||
console.log('');
|
||||
|
||||
const targetTriple = getTargetTriple();
|
||||
console.log(`Platform: ${targetTriple}`);
|
||||
|
||||
createPlaceholderBinary(targetTriple);
|
||||
|
||||
console.log('');
|
||||
console.log('Sidecar setup complete.');
|
||||
console.log('For development, start the Python server in a separate terminal:');
|
||||
console.log(' bun run dev:server');
|
||||
console.log('');
|
||||
}
|
||||
|
||||
main();
|
||||
|
||||
Reference in New Issue
Block a user