mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-28 06:35:18 -07:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
219cfb1605 | ||
|
|
bf728a780c | ||
|
|
3e6513c0fb | ||
|
|
c54ee14173 | ||
|
|
cc07d4d3c9 | ||
|
|
9beb9d7fec | ||
|
|
76bb207b2b | ||
|
|
3576521d62 | ||
|
|
2df4ece388 | ||
|
|
cbb4979ed6 | ||
|
|
573f82a7e6 | ||
|
|
3d2506767d | ||
|
|
cdef2163c1 | ||
|
|
30ee07c2e3 | ||
|
|
d21c63b52c | ||
|
|
5ad67d7ecb | ||
|
|
6cc96c2614 |
@@ -5,6 +5,14 @@ All notable changes to Voicebox will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- **Profile Name Validation** - Added proper validation to prevent duplicate profile names ([#134](https://github.com/jamiepine/voicebox/issues/134))
|
||||
- Users now receive clear error messages when attempting to create or update profiles with duplicate names
|
||||
- Improved error handling in create and update profile API endpoints
|
||||
- Added comprehensive test suite for duplicate name validation
|
||||
|
||||
## [0.1.0] - 2026-01-25
|
||||
|
||||
### Added
|
||||
@@ -55,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
- Audio export failing when Tauri save dialog returns object instead of string path
|
||||
- OpenAPI client generator script now documents the local backend port and avoids an unused loop variable warning
|
||||
|
||||
### Added
|
||||
- **Makefile** - Comprehensive development workflow automation with commands for setup, development, building, testing, and code quality checks
|
||||
|
||||
+1
-1
@@ -426,7 +426,7 @@ See [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) for common issues and sol
|
||||
|
||||
- **Backend won't start:** Check Python version (3.11+), ensure venv is activated, install dependencies
|
||||
- **Tauri build fails:** Ensure Rust is installed, clean build with `cd tauri/src-tauri && cargo clean`
|
||||
- **OpenAPI client generation fails:** Ensure backend is running, check `curl http://localhost:8000/openapi.json`
|
||||
- **OpenAPI client generation fails:** Ensure backend is running, check `curl http://localhost:17493/openapi.json`
|
||||
|
||||
## Questions?
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ setup-python: $(VENV)/bin/activate ## Set up Python virtual environment and depe
|
||||
@echo -e "$(BLUE)Installing Python dependencies...$(NC)"
|
||||
$(PIP) install --upgrade pip
|
||||
$(PIP) install -r $(BACKEND_DIR)/requirements.txt
|
||||
$(PIP) install --no-deps chatterbox-tts
|
||||
@if [ "$$(uname -m)" = "arm64" ] && [ "$$(uname)" = "Darwin" ]; then \
|
||||
echo -e "$(BLUE)Detected Apple Silicon - installing MLX dependencies...$(NC)"; \
|
||||
$(PIP) install -r $(BACKEND_DIR)/requirements-mlx.txt; \
|
||||
@@ -79,7 +80,11 @@ dev: ## Start backend + desktop app (parallel)
|
||||
@echo -e "$(YELLOW)Note: If Tauri fails, run 'make build-server' first or use separate terminals$(NC)"
|
||||
@trap 'kill 0' EXIT; \
|
||||
$(MAKE) dev-backend & \
|
||||
sleep 2 && $(MAKE) dev-frontend & \
|
||||
sleep 2 && if [ "$$(uname)" = "Linux" ] && lspci 2>/dev/null | grep -qi nvidia; then \
|
||||
WEBKIT_DISABLE_DMABUF_RENDERER=1 $(MAKE) dev-frontend; \
|
||||
else \
|
||||
$(MAKE) dev-frontend; \
|
||||
fi & \
|
||||
wait
|
||||
|
||||
dev-backend: ## Start FastAPI backend server
|
||||
|
||||
@@ -147,17 +147,20 @@ Create multi-voice narratives, podcasts, and conversations with a timeline-based
|
||||
|
||||
Voicebox exposes a full REST API, so you can integrate voice synthesis into your own apps.
|
||||
|
||||
For the current local app and development workflow, the backend is typically available at `http://localhost:17493`.
|
||||
If you launch the backend manually with a different host or port, use that address instead.
|
||||
|
||||
```bash
|
||||
# Generate speech
|
||||
curl -X POST http://localhost:8000/generate \
|
||||
curl -X POST http://localhost:17493/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"text": "Hello world", "profile_id": "abc123", "language": "en"}'
|
||||
|
||||
# List voice profiles
|
||||
curl http://localhost:8000/profiles
|
||||
curl http://localhost:17493/profiles
|
||||
|
||||
# Create a profile
|
||||
curl -X POST http://localhost:8000/profiles \
|
||||
curl -X POST http://localhost:17493/profiles \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "My Voice", "language": "en"}'
|
||||
```
|
||||
@@ -170,7 +173,7 @@ curl -X POST http://localhost:8000/profiles \
|
||||
- Voice assistants
|
||||
- Content creation automation
|
||||
|
||||
Full API documentation available at `http://localhost:8000/docs` when running.
|
||||
Full API documentation is available at `http://localhost:17493/docs` in the default local workflow, or at `/docs` on whatever server address you configured.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
} from '@/components/ui/select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { LANGUAGE_OPTIONS } from '@/lib/constants/languages';
|
||||
import { getLanguageOptionsForEngine } from '@/lib/constants/languages';
|
||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||
import { useProfile, useProfiles } from '@/lib/hooks/useProfiles';
|
||||
import { useAddStoryItem, useStory } from '@/lib/hooks/useStories';
|
||||
@@ -316,7 +316,7 @@ export function FloatingGenerateBox({
|
||||
</span>
|
||||
</div>
|
||||
<AnimatePresence>
|
||||
{isExpanded && form.watch('engine') !== 'luxtts' && (
|
||||
{isExpanded && form.watch('engine') === 'qwen' && (
|
||||
<motion.div
|
||||
initial={{ opacity: 0, scale: 0.8 }}
|
||||
animate={{ opacity: 1, scale: 1 }}
|
||||
@@ -381,25 +381,30 @@ export function FloatingGenerateBox({
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage className="text-xs" />
|
||||
</FormItem>
|
||||
)}
|
||||
render={({ field }) => {
|
||||
const engineLangs = getLanguageOptionsForEngine(
|
||||
form.watch('engine') || 'qwen',
|
||||
);
|
||||
return (
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{engineLangs.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage className="text-xs" />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
@@ -407,11 +412,21 @@ export function FloatingGenerateBox({
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
@@ -434,6 +449,15 @@ export function FloatingGenerateBox({
|
||||
<SelectItem value="luxtts" className="text-xs text-muted-foreground">
|
||||
LuxTTS
|
||||
</SelectItem>
|
||||
<SelectItem value="chatterbox" className="text-xs text-muted-foreground">
|
||||
Chatterbox
|
||||
</SelectItem>
|
||||
<SelectItem
|
||||
value="chatterbox_turbo"
|
||||
className="text-xs text-muted-foreground"
|
||||
>
|
||||
Chatterbox Turbo
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { LANGUAGE_OPTIONS } from '@/lib/constants/languages';
|
||||
import { getLanguageOptionsForEngine } from '@/lib/constants/languages';
|
||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||
import { useProfile } from '@/lib/hooks/useProfiles';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
@@ -76,7 +76,7 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
{form.watch('engine') !== 'luxtts' && (
|
||||
{form.watch('engine') === 'qwen' && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
@@ -107,11 +107,21 @@ export function GenerationForm() {
|
||||
value={
|
||||
form.watch('engine') === 'luxtts'
|
||||
? 'luxtts'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
@@ -128,38 +138,47 @@ export function GenerationForm() {
|
||||
<SelectItem value="qwen:1.7B">Qwen3-TTS 1.7B</SelectItem>
|
||||
<SelectItem value="qwen:0.6B">Qwen3-TTS 0.6B</SelectItem>
|
||||
<SelectItem value="luxtts">LuxTTS</SelectItem>
|
||||
<SelectItem value="chatterbox">Chatterbox</SelectItem>
|
||||
<SelectItem value="chatterbox_turbo">Chatterbox Turbo</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{form.watch('engine') === 'luxtts'
|
||||
? 'Fast, English-focused'
|
||||
: 'Multi-language, two sizes'}
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? '23 languages, incl. Hebrew'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'English, [laugh] [cough] tags'
|
||||
: 'Multi-language, two sizes'}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Language</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
render={({ field }) => {
|
||||
const engineLangs = getLanguageOptionsForEngine(form.watch('engine') || 'qwen');
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>Language</FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{engineLangs.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
|
||||
@@ -2,7 +2,7 @@ import { ModelManagement } from '@/components/ServerSettings/ModelManagement';
|
||||
|
||||
export function ModelsTab() {
|
||||
return (
|
||||
<div className="space-y-4 overflow-y-auto flex flex-col">
|
||||
<div className="h-full flex flex-col p-4">
|
||||
<ModelManagement />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -32,7 +32,7 @@ export function GpuAcceleration() {
|
||||
} = useQuery({
|
||||
queryKey: ['cuda-status', serverUrl],
|
||||
queryFn: () => apiClient.getCudaStatus(),
|
||||
refetchInterval: cudaStatusLoading ? false : 10000,
|
||||
refetchInterval: (query) => (query.state.status === 'pending' ? false : 10000),
|
||||
retry: 1,
|
||||
enabled: !!health, // Only fetch when backend is reachable
|
||||
});
|
||||
|
||||
@@ -1,6 +1,21 @@
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { ChevronDown, ChevronUp, Download, Loader2, RotateCcw, Trash2, X } from 'lucide-react';
|
||||
import { useCallback, useState } from 'react';
|
||||
import {
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
ChevronUp,
|
||||
CircleCheck,
|
||||
CircleX,
|
||||
Download,
|
||||
ExternalLink,
|
||||
HardDrive,
|
||||
Heart,
|
||||
Loader2,
|
||||
RotateCcw,
|
||||
Scale,
|
||||
Trash2,
|
||||
X,
|
||||
} from 'lucide-react';
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
@@ -13,12 +28,59 @@ import {
|
||||
} from '@/components/ui/alert-dialog';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { ActiveDownloadTask } from '@/lib/api/types';
|
||||
import type { ActiveDownloadTask, HuggingFaceModelInfo, ModelStatus } from '@/lib/api/types';
|
||||
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
|
||||
|
||||
async function fetchHuggingFaceModelInfo(repoId: string): Promise<HuggingFaceModelInfo> {
|
||||
const response = await fetch(`https://huggingface.co/api/models/${repoId}`);
|
||||
if (!response.ok) throw new Error(`Failed to fetch model info: ${response.status}`);
|
||||
return response.json();
|
||||
}
|
||||
|
||||
function formatDownloads(n: number): string {
|
||||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
||||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`;
|
||||
return n.toString();
|
||||
}
|
||||
|
||||
function formatLicense(license: string): string {
|
||||
const map: Record<string, string> = {
|
||||
'apache-2.0': 'Apache 2.0',
|
||||
mit: 'MIT',
|
||||
'cc-by-4.0': 'CC BY 4.0',
|
||||
'cc-by-sa-4.0': 'CC BY-SA 4.0',
|
||||
'cc-by-nc-4.0': 'CC BY-NC 4.0',
|
||||
'openrail++': 'OpenRAIL++',
|
||||
openrail: 'OpenRAIL',
|
||||
};
|
||||
return map[license] || license;
|
||||
}
|
||||
|
||||
function formatPipelineTag(tag: string): string {
|
||||
return tag
|
||||
.split('-')
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return `${(bytes / k ** i).toFixed(1)} ${sizes[i]}`;
|
||||
}
|
||||
|
||||
export function ModelManagement() {
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
@@ -28,36 +90,48 @@ export function ModelManagement() {
|
||||
const [dismissedErrors, setDismissedErrors] = useState<Set<string>>(new Set());
|
||||
const [localErrors, setLocalErrors] = useState<Map<string, string>>(new Map());
|
||||
|
||||
// Modal state
|
||||
const [selectedModel, setSelectedModel] = useState<ModelStatus | null>(null);
|
||||
const [detailOpen, setDetailOpen] = useState(false);
|
||||
|
||||
const { data: modelStatus, isLoading } = useQuery({
|
||||
queryKey: ['modelStatus'],
|
||||
queryFn: async () => {
|
||||
console.log('[Query] Fetching model status');
|
||||
const result = await apiClient.getModelStatus();
|
||||
console.log('[Query] Model status fetched:', result);
|
||||
return result;
|
||||
},
|
||||
refetchInterval: 5000, // Refresh every 5 seconds
|
||||
refetchInterval: 5000,
|
||||
});
|
||||
|
||||
const { data: activeTasks } = useQuery({
|
||||
queryKey: ['activeTasks'],
|
||||
queryFn: () => apiClient.getActiveTasks(),
|
||||
refetchInterval: 5000,
|
||||
refetchInterval: (query) => {
|
||||
const data = query.state.data;
|
||||
const hasActive = data?.downloads.some((d) => d.status === 'downloading');
|
||||
return hasActive ? 1000 : 5000;
|
||||
},
|
||||
});
|
||||
|
||||
// HuggingFace model card query - only fetches when modal is open and model has a repo ID
|
||||
const { data: hfModelInfo, isLoading: hfLoading } = useQuery({
|
||||
queryKey: ['hfModelInfo', selectedModel?.hf_repo_id],
|
||||
queryFn: () => fetchHuggingFaceModelInfo(selectedModel!.hf_repo_id!),
|
||||
enabled: detailOpen && !!selectedModel?.hf_repo_id,
|
||||
staleTime: 1000 * 60 * 30, // Cache for 30 minutes
|
||||
retry: 1,
|
||||
});
|
||||
|
||||
// Build a map of errored downloads for quick lookup, excluding dismissed ones
|
||||
// Merge server errors with locally captured SSE errors
|
||||
const erroredDownloads = new Map<string, ActiveDownloadTask>();
|
||||
if (activeTasks?.downloads) {
|
||||
for (const dl of activeTasks.downloads) {
|
||||
if (dl.status === 'error' && !dismissedErrors.has(dl.model_name)) {
|
||||
// Prefer locally captured error (from SSE) over server error
|
||||
const localErr = localErrors.get(dl.model_name);
|
||||
erroredDownloads.set(dl.model_name, localErr ? { ...dl, error: localErr } : dl);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also add locally captured errors that aren't in server response yet
|
||||
for (const [modelName, error] of localErrors) {
|
||||
if (!erroredDownloads.has(modelName) && !dismissedErrors.has(modelName)) {
|
||||
erroredDownloads.set(modelName, {
|
||||
@@ -71,9 +145,20 @@ export function ModelManagement() {
|
||||
|
||||
const errorCount = erroredDownloads.size;
|
||||
|
||||
// Callbacks for download completion
|
||||
// Build progress map from active tasks for inline display
|
||||
const downloadProgressMap = useMemo(() => {
|
||||
const map = new Map<string, ActiveDownloadTask>();
|
||||
if (activeTasks?.downloads) {
|
||||
for (const dl of activeTasks.downloads) {
|
||||
if (dl.status === 'downloading') {
|
||||
map.set(dl.model_name, dl);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}, [activeTasks]);
|
||||
|
||||
const handleDownloadComplete = useCallback(() => {
|
||||
console.log('[ModelManagement] Download complete, clearing state');
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
queryClient.invalidateQueries({ queryKey: ['modelStatus'] });
|
||||
@@ -82,7 +167,6 @@ export function ModelManagement() {
|
||||
|
||||
const handleDownloadError = useCallback(
|
||||
(error: string) => {
|
||||
console.log('[ModelManagement] Download error, clearing state');
|
||||
if (downloadingModel) {
|
||||
setLocalErrors((prev) => new Map(prev).set(downloadingModel, error));
|
||||
setConsoleOpen(true);
|
||||
@@ -94,7 +178,6 @@ export function ModelManagement() {
|
||||
[queryClient, downloadingModel],
|
||||
);
|
||||
|
||||
// Use progress toast hook for the downloading model
|
||||
useModelDownloadToast({
|
||||
modelName: downloadingModel || '',
|
||||
displayName: downloadingDisplayName || '',
|
||||
@@ -111,36 +194,24 @@ export function ModelManagement() {
|
||||
} | null>(null);
|
||||
|
||||
const handleDownload = async (modelName: string) => {
|
||||
console.log('[Download] Button clicked for:', modelName, 'at', new Date().toISOString());
|
||||
// Clear any previous dismissal so fresh errors can appear
|
||||
setDismissedErrors((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(modelName);
|
||||
return next;
|
||||
});
|
||||
|
||||
// Find display name
|
||||
const model = modelStatus?.models.find((m) => m.model_name === modelName);
|
||||
const displayName = model?.display_name || modelName;
|
||||
|
||||
try {
|
||||
// IMPORTANT: Call the API FIRST before setting state
|
||||
// Setting state enables the SSE EventSource in useModelDownloadToast,
|
||||
// which can block/delay the download fetch due to HTTP/1.1 connection limits
|
||||
console.log('[Download] Calling download API for:', modelName);
|
||||
const result = await apiClient.triggerModelDownload(modelName);
|
||||
console.log('[Download] Download API responded:', result);
|
||||
await apiClient.triggerModelDownload(modelName);
|
||||
|
||||
// NOW set state to enable SSE tracking (after download has started on backend)
|
||||
setDownloadingModel(modelName);
|
||||
setDownloadingDisplayName(displayName);
|
||||
|
||||
// Download initiated successfully - state will be cleared when SSE reports completion
|
||||
// or by the polling interval detecting the model is downloaded
|
||||
queryClient.invalidateQueries({ queryKey: ['modelStatus'] });
|
||||
queryClient.invalidateQueries({ queryKey: ['activeTasks'] });
|
||||
} catch (error) {
|
||||
console.error('[Download] Download failed:', error);
|
||||
setDownloadingModel(null);
|
||||
setDownloadingDisplayName(null);
|
||||
toast({
|
||||
@@ -160,13 +231,11 @@ export function ModelManagement() {
|
||||
});
|
||||
|
||||
const handleCancel = (modelName: string) => {
|
||||
// Snapshot previous state for rollback
|
||||
const prevDismissed = dismissedErrors;
|
||||
const prevLocalErrors = localErrors;
|
||||
const prevDownloadingModel = downloadingModel;
|
||||
const prevDownloadingDisplayName = downloadingDisplayName;
|
||||
|
||||
// Optimistically hide the error and suppress downloading state in UI
|
||||
setDismissedErrors((prev) => new Set(prev).add(modelName));
|
||||
setLocalErrors((prev) => {
|
||||
const next = new Map(prev);
|
||||
@@ -180,7 +249,6 @@ export function ModelManagement() {
|
||||
|
||||
cancelMutation.mutate(modelName, {
|
||||
onError: () => {
|
||||
// Rollback optimistic updates on failure
|
||||
setDismissedErrors(prevDismissed);
|
||||
setLocalErrors(prevLocalErrors);
|
||||
setDownloadingModel(prevDownloadingModel);
|
||||
@@ -208,30 +276,22 @@ export function ModelManagement() {
|
||||
|
||||
const deleteMutation = useMutation({
|
||||
mutationFn: async (modelName: string) => {
|
||||
console.log('[Delete] Deleting model:', modelName);
|
||||
const result = await apiClient.deleteModel(modelName);
|
||||
console.log('[Delete] Model deleted successfully:', modelName);
|
||||
return result;
|
||||
},
|
||||
onSuccess: async (_data, _modelName) => {
|
||||
console.log('[Delete] onSuccess - showing toast and invalidating queries');
|
||||
onSuccess: async () => {
|
||||
toast({
|
||||
title: 'Model deleted',
|
||||
description: `${modelToDelete?.displayName || 'Model'} has been deleted successfully.`,
|
||||
});
|
||||
setDeleteDialogOpen(false);
|
||||
setModelToDelete(null);
|
||||
console.log('[Delete] Invalidating modelStatus query');
|
||||
await queryClient.invalidateQueries({
|
||||
queryKey: ['modelStatus'],
|
||||
refetchType: 'all',
|
||||
});
|
||||
console.log('[Delete] Explicitly refetching modelStatus query');
|
||||
setDetailOpen(false);
|
||||
setSelectedModel(null);
|
||||
await queryClient.invalidateQueries({ queryKey: ['modelStatus'], refetchType: 'all' });
|
||||
await queryClient.refetchQueries({ queryKey: ['modelStatus'] });
|
||||
console.log('[Delete] Query refetched');
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
console.log('[Delete] onError:', error);
|
||||
toast({
|
||||
title: 'Delete failed',
|
||||
description: error.message,
|
||||
@@ -241,185 +301,438 @@ export function ModelManagement() {
|
||||
});
|
||||
|
||||
const formatSize = (sizeMb?: number): string => {
|
||||
if (!sizeMb) return 'Unknown';
|
||||
if (!sizeMb) return 'Unknown size';
|
||||
if (sizeMb < 1024) return `${sizeMb.toFixed(1)} MB`;
|
||||
return `${(sizeMb / 1024).toFixed(2)} GB`;
|
||||
};
|
||||
|
||||
const getModelState = (model: ModelStatus) => {
|
||||
const isDownloading =
|
||||
(model.downloading || downloadingModel === model.model_name) &&
|
||||
!erroredDownloads.has(model.model_name) &&
|
||||
!dismissedErrors.has(model.model_name);
|
||||
const hasError = erroredDownloads.has(model.model_name);
|
||||
return { isDownloading, hasError };
|
||||
};
|
||||
|
||||
const openModelDetail = (model: ModelStatus) => {
|
||||
setSelectedModel(model);
|
||||
setDetailOpen(true);
|
||||
};
|
||||
|
||||
const ttsModels = modelStatus?.models.filter((m) => m.model_name.startsWith('qwen-tts')) ?? [];
|
||||
const otherTtsModels =
|
||||
modelStatus?.models.filter(
|
||||
(m) => m.model_name.startsWith('luxtts') || m.model_name.startsWith('chatterbox'),
|
||||
) ?? [];
|
||||
const whisperModels = modelStatus?.models.filter((m) => m.model_name.startsWith('whisper')) ?? [];
|
||||
|
||||
// Build sections
|
||||
const sections: { label: string; models: ModelStatus[] }[] = [
|
||||
{ label: 'Voice Generation', models: ttsModels },
|
||||
...(otherTtsModels.length > 0 ? [{ label: 'Other Voice Models', models: otherTtsModels }] : []),
|
||||
{ label: 'Transcription', models: whisperModels },
|
||||
];
|
||||
|
||||
// Get detail modal state for selected model
|
||||
const selectedState = selectedModel ? getModelState(selectedModel) : null;
|
||||
const selectedError = selectedModel ? erroredDownloads.get(selectedModel.model_name) : undefined;
|
||||
|
||||
// Keep selectedModel data fresh from query results
|
||||
const freshSelectedModel =
|
||||
selectedModel && modelStatus
|
||||
? modelStatus.models.find((m) => m.model_name === selectedModel.model_name) || selectedModel
|
||||
: selectedModel;
|
||||
|
||||
// Derive license from HF data
|
||||
const license =
|
||||
hfModelInfo?.cardData?.license ||
|
||||
hfModelInfo?.tags?.find((t) => t.startsWith('license:'))?.replace('license:', '');
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Model Management</CardTitle>
|
||||
<CardDescription>
|
||||
<div className="flex flex-col h-full">
|
||||
{/* Header */}
|
||||
<div className="shrink-0 pb-4">
|
||||
<h1 className="text-lg font-semibold">Models</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Download and manage AI models for voice generation and transcription
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
{isLoading ? (
|
||||
<div className="flex items-center justify-center py-8">
|
||||
<Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
) : modelStatus ? (
|
||||
<div className="space-y-4">
|
||||
{/* TTS Models */}
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">
|
||||
Voice Generation Models
|
||||
</h3>
|
||||
<div className="space-y-2">
|
||||
{modelStatus.models
|
||||
.filter((m) => m.model_name.startsWith('qwen-tts'))
|
||||
.map((model) => (
|
||||
<ModelItem
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Model list */}
|
||||
{isLoading ? (
|
||||
<div className="flex items-center justify-center py-16">
|
||||
<Loader2 className="h-5 w-5 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
) : modelStatus ? (
|
||||
<div className="flex-1 min-h-0 overflow-y-auto space-y-6">
|
||||
{sections.map((section) => (
|
||||
<div key={section.label}>
|
||||
<h2 className="text-xs font-medium text-muted-foreground uppercase tracking-wider mb-1 px-1">
|
||||
{section.label}
|
||||
</h2>
|
||||
<div className="border rounded-lg divide-y overflow-hidden">
|
||||
{section.models.map((model) => {
|
||||
const { isDownloading, hasError } = getModelState(model);
|
||||
return (
|
||||
<button
|
||||
key={model.model_name}
|
||||
model={model}
|
||||
onDownload={() => handleDownload(model.model_name)}
|
||||
onDelete={() => {
|
||||
setModelToDelete({
|
||||
name: model.model_name,
|
||||
displayName: model.display_name,
|
||||
sizeMb: model.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* LuxTTS Models */}
|
||||
{modelStatus.models.some((m) => m.model_name.startsWith('luxtts')) && (
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">LuxTTS Models</h3>
|
||||
<div className="space-y-2">
|
||||
{modelStatus.models
|
||||
.filter((m) => m.model_name.startsWith('luxtts'))
|
||||
.map((model) => (
|
||||
<ModelItem
|
||||
key={model.model_name}
|
||||
model={model}
|
||||
onDownload={() => handleDownload(model.model_name)}
|
||||
onDelete={() => {
|
||||
setModelToDelete({
|
||||
name: model.model_name,
|
||||
displayName: model.display_name,
|
||||
sizeMb: model.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
formatSize={formatSize}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Whisper Models */}
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold mb-3 text-muted-foreground">
|
||||
Transcription Models
|
||||
</h3>
|
||||
<div className="space-y-2">
|
||||
{modelStatus.models
|
||||
.filter((m) => m.model_name.startsWith('whisper'))
|
||||
.map((model) => (
|
||||
<ModelItem
|
||||
key={model.model_name}
|
||||
model={model}
|
||||
onDownload={() => handleDownload(model.model_name)}
|
||||
onDelete={() => {
|
||||
setModelToDelete({
|
||||
name: model.model_name,
|
||||
displayName: model.display_name,
|
||||
sizeMb: model.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
onCancel={() => handleCancel(model.model_name)}
|
||||
isDownloading={downloadingModel === model.model_name}
|
||||
isCancelling={
|
||||
cancelMutation.isPending && cancelMutation.variables === model.model_name
|
||||
}
|
||||
isDismissed={dismissedErrors.has(model.model_name)}
|
||||
erroredDownload={erroredDownloads.get(model.model_name)}
|
||||
formatSize={formatSize}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Console Panel */}
|
||||
{errorCount > 0 && (
|
||||
<div className="border rounded-lg overflow-hidden">
|
||||
<div className="flex items-center justify-between px-3 py-1.5 bg-muted/50 text-xs font-medium text-muted-foreground">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setConsoleOpen((v) => !v)}
|
||||
className="flex items-center gap-2 hover:text-foreground transition-colors"
|
||||
>
|
||||
{consoleOpen ? (
|
||||
<ChevronUp className="h-3.5 w-3.5" />
|
||||
) : (
|
||||
<ChevronDown className="h-3.5 w-3.5" />
|
||||
)}
|
||||
<span>Problems</span>
|
||||
<Badge variant="destructive" className="text-[10px] h-4 px-1.5 rounded-full">
|
||||
{errorCount}
|
||||
</Badge>
|
||||
</button>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-6 px-2 text-xs text-muted-foreground hover:text-foreground"
|
||||
onClick={() => clearAllMutation.mutate()}
|
||||
disabled={clearAllMutation.isPending}
|
||||
>
|
||||
<RotateCcw className="h-3 w-3 mr-1" />
|
||||
Clear All
|
||||
</Button>
|
||||
</div>
|
||||
{consoleOpen && (
|
||||
<div className="bg-[#1e1e1e] text-[#d4d4d4] p-3 max-h-48 overflow-auto font-mono text-xs leading-relaxed">
|
||||
{Array.from(erroredDownloads.entries()).map(([modelName, dl]) => (
|
||||
<div key={modelName} className="mb-2 last:mb-0">
|
||||
<span className="text-[#f44747]">[error]</span>{' '}
|
||||
<span className="text-[#569cd6]">{modelName}</span>
|
||||
{dl.error ? (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">
|
||||
{dl.error}
|
||||
</span>
|
||||
</>
|
||||
type="button"
|
||||
onClick={() => openModelDetail(model)}
|
||||
className="w-full flex items-center gap-3 px-3 py-2.5 text-left hover:bg-muted/50 transition-colors group"
|
||||
>
|
||||
{/* Status indicator */}
|
||||
<div className="shrink-0">
|
||||
{hasError ? (
|
||||
<CircleX className="h-4 w-4 text-destructive" />
|
||||
) : isDownloading ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin text-muted-foreground" />
|
||||
) : model.loaded ? (
|
||||
<CircleCheck className="h-4 w-4 text-accent" />
|
||||
) : model.downloaded ? (
|
||||
<CircleCheck className="h-4 w-4 text-emerald-500" />
|
||||
) : (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#808080]">
|
||||
No error details available. Try downloading again.
|
||||
</span>
|
||||
</>
|
||||
<Download className="h-4 w-4 text-muted-foreground/50" />
|
||||
)}
|
||||
<div className="text-[#6a9955] mt-0.5">
|
||||
started at {new Date(dl.started_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Name + inline progress */}
|
||||
<div className="flex-1 min-w-0">
|
||||
<span className="text-sm font-medium">{model.display_name}</span>
|
||||
{isDownloading &&
|
||||
(() => {
|
||||
const dl = downloadProgressMap.get(model.model_name);
|
||||
const pct = dl?.progress ?? 0;
|
||||
const hasProgress = dl && dl.total && dl.total > 0;
|
||||
return (
|
||||
<div className="mt-1 space-y-0.5">
|
||||
<Progress value={hasProgress ? pct : undefined} className="h-1" />
|
||||
<div className="text-[10px] text-muted-foreground truncate">
|
||||
{hasProgress
|
||||
? `${formatBytes(dl.current ?? 0)} / ${formatBytes(dl.total!)} (${pct.toFixed(0)}%)`
|
||||
: dl?.filename || 'Connecting...'}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
|
||||
{/* Right side info */}
|
||||
<div className="shrink-0 flex items-center gap-2">
|
||||
{hasError && (
|
||||
<Badge variant="destructive" className="text-[10px] h-5">
|
||||
Error
|
||||
</Badge>
|
||||
)}
|
||||
{model.loaded && (
|
||||
<Badge className="text-[10px] h-5 bg-accent/15 text-accent border-accent/30 hover:bg-accent/15">
|
||||
Loaded
|
||||
</Badge>
|
||||
)}
|
||||
{model.downloaded && !isDownloading && !hasError && (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatSize(model.size_mb)}
|
||||
</span>
|
||||
)}
|
||||
{!model.downloaded && !isDownloading && !hasError && (
|
||||
<span className="text-xs text-muted-foreground/60">Not downloaded</span>
|
||||
)}
|
||||
<ChevronRight className="h-4 w-4 text-muted-foreground/40 group-hover:text-muted-foreground transition-colors" />
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Error console */}
|
||||
{errorCount > 0 && (
|
||||
<div className="border rounded-lg overflow-hidden">
|
||||
<div className="flex items-center justify-between px-3 py-1.5 bg-muted/50 text-xs font-medium text-muted-foreground">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setConsoleOpen((v) => !v)}
|
||||
className="flex items-center gap-2 hover:text-foreground transition-colors"
|
||||
>
|
||||
{consoleOpen ? (
|
||||
<ChevronUp className="h-3.5 w-3.5" />
|
||||
) : (
|
||||
<ChevronDown className="h-3.5 w-3.5" />
|
||||
)}
|
||||
<span>Problems</span>
|
||||
<Badge variant="destructive" className="text-[10px] h-4 px-1.5 rounded-full">
|
||||
{errorCount}
|
||||
</Badge>
|
||||
</button>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-6 px-2 text-xs text-muted-foreground hover:text-foreground"
|
||||
onClick={() => clearAllMutation.mutate()}
|
||||
disabled={clearAllMutation.isPending}
|
||||
>
|
||||
<RotateCcw className="h-3 w-3 mr-1" />
|
||||
Clear All
|
||||
</Button>
|
||||
</div>
|
||||
{consoleOpen && (
|
||||
<div className="bg-[#1e1e1e] text-[#d4d4d4] p-3 max-h-48 overflow-auto font-mono text-xs leading-relaxed">
|
||||
{Array.from(erroredDownloads.entries()).map(([modelName, dl]) => (
|
||||
<div key={modelName} className="mb-2 last:mb-0">
|
||||
<span className="text-[#f44747]">[error]</span>{' '}
|
||||
<span className="text-[#569cd6]">{modelName}</span>
|
||||
{dl.error ? (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#ce9178] whitespace-pre-wrap break-all">
|
||||
{dl.error}
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
{': '}
|
||||
<span className="text-[#808080]">
|
||||
No error details available. Try downloading again.
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
<div className="text-[#6a9955] mt-0.5">
|
||||
started at {new Date(dl.started_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{/* Model Detail Modal */}
|
||||
<Dialog open={detailOpen} onOpenChange={setDetailOpen}>
|
||||
<DialogContent className="sm:max-w-md">
|
||||
{freshSelectedModel && (
|
||||
<>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{freshSelectedModel.display_name}</DialogTitle>
|
||||
<DialogDescription className="flex items-center gap-1.5">
|
||||
{freshSelectedModel.hf_repo_id ? (
|
||||
<a
|
||||
href={`https://huggingface.co/${freshSelectedModel.hf_repo_id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 hover:underline"
|
||||
>
|
||||
{freshSelectedModel.hf_repo_id}
|
||||
<ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
) : (
|
||||
freshSelectedModel.model_name
|
||||
)}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-4 pt-2">
|
||||
{/* Status badges */}
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
{freshSelectedModel.loaded && (
|
||||
<Badge className="text-xs bg-accent/15 text-accent border-accent/30 hover:bg-accent/15">
|
||||
<CircleCheck className="h-3 w-3 mr-1" />
|
||||
Loaded
|
||||
</Badge>
|
||||
)}
|
||||
{freshSelectedModel.downloaded && !freshSelectedModel.loaded && (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
<CircleCheck className="h-3 w-3 mr-1" />
|
||||
Downloaded
|
||||
</Badge>
|
||||
)}
|
||||
{selectedState?.hasError && (
|
||||
<Badge variant="destructive" className="text-xs">
|
||||
<CircleX className="h-3 w-3 mr-1" />
|
||||
Error
|
||||
</Badge>
|
||||
)}
|
||||
{!freshSelectedModel.downloaded &&
|
||||
!selectedState?.isDownloading &&
|
||||
!selectedState?.hasError && (
|
||||
<Badge variant="outline" className="text-xs text-muted-foreground">
|
||||
Not downloaded
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* HuggingFace model card info */}
|
||||
{hfLoading && freshSelectedModel.hf_repo_id && (
|
||||
<div className="flex items-center gap-2 text-xs text-muted-foreground py-2">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
Loading model info...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{hfModelInfo && (
|
||||
<div className="space-y-3">
|
||||
{/* Stats row */}
|
||||
<div className="flex items-center gap-4 text-xs text-muted-foreground">
|
||||
<span className="flex items-center gap-1" title="Downloads">
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.downloads)}
|
||||
</span>
|
||||
<span className="flex items-center gap-1" title="Likes">
|
||||
<Heart className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.likes)}
|
||||
</span>
|
||||
{license && (
|
||||
<span className="flex items-center gap-1" title="License">
|
||||
<Scale className="h-3.5 w-3.5" />
|
||||
{formatLicense(license)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Pipeline tag + author */}
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{hfModelInfo.pipeline_tag && (
|
||||
<Badge variant="outline" className="text-[10px]">
|
||||
{formatPipelineTag(hfModelInfo.pipeline_tag)}
|
||||
</Badge>
|
||||
)}
|
||||
{hfModelInfo.library_name && (
|
||||
<Badge variant="outline" className="text-[10px]">
|
||||
{hfModelInfo.library_name}
|
||||
</Badge>
|
||||
)}
|
||||
{hfModelInfo.author && (
|
||||
<Badge variant="outline" className="text-[10px]">
|
||||
by {hfModelInfo.author}
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Languages */}
|
||||
{hfModelInfo.cardData?.language && hfModelInfo.cardData.language.length > 0 && (
|
||||
<div>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{hfModelInfo.cardData.language.length > 10
|
||||
? `${hfModelInfo.cardData.language.length} languages supported`
|
||||
: `Languages: ${hfModelInfo.cardData.language.join(', ')}`}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Disk size */}
|
||||
{freshSelectedModel.downloaded && freshSelectedModel.size_mb && (
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<HardDrive className="h-4 w-4" />
|
||||
<span>{formatSize(freshSelectedModel.size_mb)} on disk</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Error detail */}
|
||||
{selectedError?.error && (
|
||||
<div className="rounded-md bg-destructive/10 border border-destructive/20 p-3 text-xs text-destructive">
|
||||
{selectedError.error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Actions */}
|
||||
<div className="flex items-center gap-2 pt-2 border-t">
|
||||
{selectedState?.hasError ? (
|
||||
<>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => handleDownload(freshSelectedModel.model_name)}
|
||||
variant="outline"
|
||||
className="flex-1"
|
||||
>
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
Retry Download
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => handleCancel(freshSelectedModel.model_name)}
|
||||
variant="ghost"
|
||||
disabled={
|
||||
cancelMutation.isPending &&
|
||||
cancelMutation.variables === freshSelectedModel.model_name
|
||||
}
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : selectedState?.isDownloading ? (
|
||||
<>
|
||||
<div className="flex-1 space-y-2">
|
||||
{(() => {
|
||||
const dl = freshSelectedModel
|
||||
? downloadProgressMap.get(freshSelectedModel.model_name)
|
||||
: undefined;
|
||||
const pct = dl?.progress ?? 0;
|
||||
const hasProgress = dl && dl.total && dl.total > 0;
|
||||
return (
|
||||
<>
|
||||
<Progress value={hasProgress ? pct : undefined} className="h-2" />
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{hasProgress
|
||||
? `${formatBytes(dl.current ?? 0)} / ${formatBytes(dl.total!)} (${pct.toFixed(1)}%)`
|
||||
: dl?.filename || 'Connecting to HuggingFace...'}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
})()}
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => handleCancel(freshSelectedModel.model_name)}
|
||||
variant="ghost"
|
||||
disabled={
|
||||
cancelMutation.isPending &&
|
||||
cancelMutation.variables === freshSelectedModel.model_name
|
||||
}
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</>
|
||||
) : freshSelectedModel.downloaded ? (
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setModelToDelete({
|
||||
name: freshSelectedModel.model_name,
|
||||
displayName: freshSelectedModel.display_name,
|
||||
sizeMb: freshSelectedModel.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
variant="outline"
|
||||
disabled={freshSelectedModel.loaded}
|
||||
title={
|
||||
freshSelectedModel.loaded ? 'Unload model before deleting' : 'Delete model'
|
||||
}
|
||||
className="flex-1"
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
{freshSelectedModel.loaded ? 'Unload to Delete' : 'Delete Model'}
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => handleDownload(freshSelectedModel.model_name)}
|
||||
className="flex-1"
|
||||
>
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
Download
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : null}
|
||||
</CardContent>
|
||||
</>
|
||||
)}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* Delete Confirmation Dialog */}
|
||||
<AlertDialog open={deleteDialogOpen} onOpenChange={setDeleteDialogOpen}>
|
||||
@@ -460,126 +773,6 @@ export function ModelManagement() {
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
interface ModelItemProps {
|
||||
model: {
|
||||
model_name: string;
|
||||
display_name: string;
|
||||
downloaded: boolean;
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
size_mb?: number;
|
||||
loaded: boolean;
|
||||
};
|
||||
onDownload: () => void;
|
||||
onDelete: () => void;
|
||||
onCancel: () => void;
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
isCancelling: boolean;
|
||||
isDismissed: boolean;
|
||||
erroredDownload?: ActiveDownloadTask;
|
||||
formatSize: (sizeMb?: number) => string;
|
||||
}
|
||||
|
||||
function ModelItem({
|
||||
model,
|
||||
onDownload,
|
||||
onDelete,
|
||||
onCancel,
|
||||
isDownloading,
|
||||
isCancelling,
|
||||
isDismissed,
|
||||
erroredDownload,
|
||||
formatSize,
|
||||
}: ModelItemProps) {
|
||||
// Use server's downloading state OR local state (for immediate feedback before server updates)
|
||||
// Suppress downloading if user just dismissed/cancelled this model
|
||||
const showDownloading = (model.downloading || isDownloading) && !erroredDownload && !isDismissed;
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between p-3 border rounded-lg">
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium text-sm">{model.display_name}</span>
|
||||
{model.loaded && (
|
||||
<Badge variant="default" className="text-xs">
|
||||
Loaded
|
||||
</Badge>
|
||||
)}
|
||||
{model.downloaded && !model.loaded && !showDownloading && !erroredDownload && (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
Downloaded
|
||||
</Badge>
|
||||
)}
|
||||
{erroredDownload && (
|
||||
<Badge variant="destructive" className="text-xs">
|
||||
Error
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
{model.downloaded && model.size_mb && !showDownloading && !erroredDownload && (
|
||||
<div className="text-xs text-muted-foreground mt-1">
|
||||
Size: {formatSize(model.size_mb)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2 shrink-0 ml-2">
|
||||
{erroredDownload ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Button size="sm" onClick={onDownload} variant="outline">
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
Retry
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onCancel}
|
||||
variant="ghost"
|
||||
disabled={isCancelling}
|
||||
title="Dismiss error"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
) : model.downloaded && !showDownloading ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center gap-1 text-sm text-muted-foreground">
|
||||
<span>Ready</span>
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onDelete}
|
||||
variant="outline"
|
||||
disabled={model.loaded}
|
||||
title={model.loaded ? 'Unload model before deleting' : 'Delete model'}
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
) : showDownloading ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Button size="sm" variant="outline" disabled>
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
Downloading...
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onCancel}
|
||||
variant="ghost"
|
||||
disabled={isCancelling}
|
||||
title="Cancel download"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<Button size="sm" onClick={onDownload} variant="outline">
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
Download
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { FloatingGenerateBox } from '@/components/Generation/FloatingGenerateBox';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { StoryContent } from './StoryContent';
|
||||
import { StoryList } from './StoryList';
|
||||
|
||||
export function StoriesTab() {
|
||||
const audioUrl = usePlayerStore((state) => state.audioUrl);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full min-h-0 overflow-hidden">
|
||||
{/* Main content area */}
|
||||
@@ -18,7 +21,7 @@ export function StoriesTab() {
|
||||
</div>
|
||||
|
||||
{/* Floating Generate Box - position is managed via storyStore.trackEditorHeight */}
|
||||
<FloatingGenerateBox showVoiceSelector />
|
||||
<FloatingGenerateBox showVoiceSelector isPlayerOpen={!!audioUrl} />
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -34,7 +34,7 @@ export interface GenerationRequest {
|
||||
language: LanguageCode;
|
||||
seed?: number;
|
||||
model_size?: '1.7B' | '0.6B';
|
||||
engine?: 'qwen' | 'luxtts';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo';
|
||||
instruct?: string;
|
||||
}
|
||||
|
||||
@@ -119,12 +119,29 @@ export interface ModelProgress {
|
||||
export interface ModelStatus {
|
||||
model_name: string;
|
||||
display_name: string;
|
||||
hf_repo_id?: string; // HuggingFace repository ID
|
||||
downloaded: boolean;
|
||||
downloading: boolean; // True if download is in progress
|
||||
size_mb?: number;
|
||||
loaded: boolean;
|
||||
}
|
||||
|
||||
export interface HuggingFaceModelInfo {
|
||||
id: string;
|
||||
author: string;
|
||||
lastModified: string;
|
||||
pipeline_tag?: string;
|
||||
library_name?: string;
|
||||
downloads: number;
|
||||
likes: number;
|
||||
tags: string[];
|
||||
cardData?: {
|
||||
license?: string;
|
||||
language?: string[];
|
||||
pipeline_tag?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ModelStatusListResponse {
|
||||
models: ModelStatus[];
|
||||
}
|
||||
@@ -138,6 +155,10 @@ export interface ActiveDownloadTask {
|
||||
status: string;
|
||||
started_at: string;
|
||||
error?: string;
|
||||
progress?: number; // 0-100 percentage
|
||||
current?: number; // bytes downloaded
|
||||
total?: number; // total bytes
|
||||
filename?: string; // current file being downloaded
|
||||
}
|
||||
|
||||
export interface ActiveGenerationTask {
|
||||
|
||||
@@ -1,26 +1,86 @@
|
||||
/**
|
||||
* Supported languages for Qwen3-TTS
|
||||
* Based on: https://github.com/QwenLM/Qwen3-TTS
|
||||
* Supported languages for voice generation, per engine.
|
||||
*
|
||||
* Qwen3-TTS supports 10 languages.
|
||||
* LuxTTS is English-only.
|
||||
* Chatterbox Multilingual supports 23 languages.
|
||||
* Chatterbox Turbo is English-only.
|
||||
*/
|
||||
|
||||
export const SUPPORTED_LANGUAGES = {
|
||||
zh: 'Chinese',
|
||||
/** All languages that any engine supports. */
|
||||
export const ALL_LANGUAGES = {
|
||||
ar: 'Arabic',
|
||||
da: 'Danish',
|
||||
de: 'German',
|
||||
el: 'Greek',
|
||||
en: 'English',
|
||||
es: 'Spanish',
|
||||
fi: 'Finnish',
|
||||
fr: 'French',
|
||||
he: 'Hebrew',
|
||||
hi: 'Hindi',
|
||||
it: 'Italian',
|
||||
ja: 'Japanese',
|
||||
ko: 'Korean',
|
||||
de: 'German',
|
||||
fr: 'French',
|
||||
ru: 'Russian',
|
||||
ms: 'Malay',
|
||||
nl: 'Dutch',
|
||||
no: 'Norwegian',
|
||||
pl: 'Polish',
|
||||
pt: 'Portuguese',
|
||||
es: 'Spanish',
|
||||
it: 'Italian',
|
||||
ru: 'Russian',
|
||||
sv: 'Swedish',
|
||||
sw: 'Swahili',
|
||||
tr: 'Turkish',
|
||||
zh: 'Chinese',
|
||||
} as const;
|
||||
|
||||
export type LanguageCode = keyof typeof SUPPORTED_LANGUAGES;
|
||||
export type LanguageCode = keyof typeof ALL_LANGUAGES;
|
||||
|
||||
export const LANGUAGE_CODES = Object.keys(SUPPORTED_LANGUAGES) as LanguageCode[];
|
||||
/** Per-engine supported language codes. */
|
||||
export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
|
||||
qwen: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'],
|
||||
luxtts: ['en'],
|
||||
chatterbox: [
|
||||
'ar',
|
||||
'da',
|
||||
'de',
|
||||
'el',
|
||||
'en',
|
||||
'es',
|
||||
'fi',
|
||||
'fr',
|
||||
'he',
|
||||
'hi',
|
||||
'it',
|
||||
'ja',
|
||||
'ko',
|
||||
'ms',
|
||||
'nl',
|
||||
'no',
|
||||
'pl',
|
||||
'pt',
|
||||
'ru',
|
||||
'sv',
|
||||
'sw',
|
||||
'tr',
|
||||
'zh',
|
||||
],
|
||||
chatterbox_turbo: ['en'],
|
||||
} as const;
|
||||
|
||||
/** Helper: get language options for a given engine. */
|
||||
export function getLanguageOptionsForEngine(engine: string) {
|
||||
const codes = ENGINE_LANGUAGES[engine] ?? ENGINE_LANGUAGES.qwen;
|
||||
return codes.map((code) => ({
|
||||
value: code,
|
||||
label: ALL_LANGUAGES[code],
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Backwards-compatible exports used elsewhere ──────────────────────
|
||||
export const SUPPORTED_LANGUAGES = ALL_LANGUAGES;
|
||||
export const LANGUAGE_CODES = Object.keys(ALL_LANGUAGES) as LanguageCode[];
|
||||
export const LANGUAGE_OPTIONS = LANGUAGE_CODES.map((code) => ({
|
||||
value: code,
|
||||
label: SUPPORTED_LANGUAGES[code],
|
||||
label: ALL_LANGUAGES[code],
|
||||
}));
|
||||
|
||||
@@ -16,7 +16,7 @@ const generationSchema = z.object({
|
||||
seed: z.number().int().optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B']).optional(),
|
||||
instruct: z.string().max(500).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts']).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo']).optional(),
|
||||
});
|
||||
|
||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||
@@ -70,13 +70,24 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
setIsGenerating(true);
|
||||
|
||||
const engine = data.engine || 'qwen';
|
||||
const modelName = engine === 'luxtts' ? 'luxtts' : `qwen-tts-${data.modelSize}`;
|
||||
const modelName =
|
||||
engine === 'luxtts'
|
||||
? 'luxtts'
|
||||
: engine === 'chatterbox'
|
||||
? 'chatterbox-tts'
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'chatterbox-turbo'
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
const displayName =
|
||||
engine === 'luxtts'
|
||||
? 'LuxTTS'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
: engine === 'chatterbox'
|
||||
? 'Chatterbox TTS'
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'Chatterbox Turbo'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
|
||||
try {
|
||||
const modelStatus = await apiClient.getModelStatus();
|
||||
@@ -90,14 +101,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
console.error('Failed to check model status:', error);
|
||||
}
|
||||
|
||||
const isQwen = engine === 'qwen';
|
||||
const result = await generation.mutateAsync({
|
||||
profile_id: selectedProfileId,
|
||||
text: data.text,
|
||||
language: data.language,
|
||||
seed: data.seed,
|
||||
model_size: engine === 'luxtts' ? undefined : data.modelSize,
|
||||
model_size: isQwen ? data.modelSize : undefined,
|
||||
engine,
|
||||
instruct: engine === 'luxtts' ? undefined : data.instruct || undefined,
|
||||
instruct: isQwen ? data.instruct || undefined : undefined,
|
||||
});
|
||||
|
||||
toast({
|
||||
|
||||
+12
-9
@@ -334,18 +334,21 @@ python -m backend.main --host 0.0.0.0 --port 8000
|
||||
|
||||
## Usage Examples
|
||||
|
||||
The desktop app, web client, and current development workflow use `http://localhost:17493` by default.
|
||||
If you launch the backend manually with a different host or port, substitute that address in the examples below.
|
||||
|
||||
### Creating a Voice Profile
|
||||
|
||||
```bash
|
||||
# 1. Create profile
|
||||
curl -X POST http://localhost:8000/profiles \
|
||||
curl -X POST http://localhost:17493/profiles \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "My Voice", "language": "en"}'
|
||||
|
||||
# Response: {"id": "abc-123", ...}
|
||||
|
||||
# 2. Add sample
|
||||
curl -X POST http://localhost:8000/profiles/abc-123/samples \
|
||||
curl -X POST http://localhost:17493/profiles/abc-123/samples \
|
||||
-F "[email protected]" \
|
||||
-F "reference_text=This is my voice sample"
|
||||
```
|
||||
@@ -353,7 +356,7 @@ curl -X POST http://localhost:8000/profiles/abc-123/samples \
|
||||
### Generating Speech
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/generate \
|
||||
curl -X POST http://localhost:17493/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"profile_id": "abc-123",
|
||||
@@ -365,13 +368,13 @@ curl -X POST http://localhost:8000/generate \
|
||||
# Response: {"id": "gen-456", "audio_path": "/path/to/audio.wav", ...}
|
||||
|
||||
# Download audio
|
||||
curl http://localhost:8000/audio/gen-456 -o output.wav
|
||||
curl http://localhost:17493/audio/gen-456 -o output.wav
|
||||
```
|
||||
|
||||
### Transcribing Audio
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/transcribe \
|
||||
curl -X POST http://localhost:17493/transcribe \
|
||||
-F "[email protected]" \
|
||||
-F "language=en"
|
||||
|
||||
@@ -386,12 +389,12 @@ Add multiple samples to a profile for better quality:
|
||||
|
||||
```bash
|
||||
# Add first sample
|
||||
curl -X POST http://localhost:8000/profiles/abc-123/samples \
|
||||
curl -X POST http://localhost:17493/profiles/abc-123/samples \
|
||||
-F "[email protected]" \
|
||||
-F "reference_text=First sample"
|
||||
|
||||
# Add second sample
|
||||
curl -X POST http://localhost:8000/profiles/abc-123/samples \
|
||||
curl -X POST http://localhost:17493/profiles/abc-123/samples \
|
||||
-F "[email protected]" \
|
||||
-F "reference_text=Second sample"
|
||||
|
||||
@@ -412,10 +415,10 @@ Models are lazy-loaded and can be manually unloaded:
|
||||
|
||||
```bash
|
||||
# Unload TTS model
|
||||
curl -X POST http://localhost:8000/models/unload
|
||||
curl -X POST http://localhost:17493/models/unload
|
||||
|
||||
# Load specific model size
|
||||
curl -X POST "http://localhost:8000/models/load?model_size=0.6B"
|
||||
curl -X POST "http://localhost:17493/models/load?model_size=0.6B"
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
@@ -121,6 +121,8 @@ _stt_backend: Optional[STTBackend] = None
|
||||
TTS_ENGINES = {
|
||||
"qwen": "Qwen TTS",
|
||||
"luxtts": "LuxTTS",
|
||||
"chatterbox": "Chatterbox TTS",
|
||||
"chatterbox_turbo": "Chatterbox Turbo",
|
||||
}
|
||||
|
||||
|
||||
@@ -167,6 +169,12 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
elif engine == "luxtts":
|
||||
from .luxtts_backend import LuxTTSBackend
|
||||
backend = LuxTTSBackend()
|
||||
elif engine == "chatterbox":
|
||||
from .chatterbox_backend import ChatterboxTTSBackend
|
||||
backend = ChatterboxTTSBackend()
|
||||
elif engine == "chatterbox_turbo":
|
||||
from .chatterbox_turbo_backend import ChatterboxTurboTTSBackend
|
||||
backend = ChatterboxTurboTTSBackend()
|
||||
else:
|
||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
"""
|
||||
Chatterbox TTS backend implementation.
|
||||
|
||||
Wraps ChatterboxMultilingualTTS from chatterbox-tts for zero-shot
|
||||
voice cloning. Supports 23 languages including Hebrew. Forces CPU
|
||||
on macOS due to known MPS tensor issues.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import TTSBackend
|
||||
from ..utils.audio import normalize_audio, load_audio
|
||||
from ..utils.progress import get_progress_manager
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHATTERBOX_HF_REPO = "ResembleAI/chatterbox"
|
||||
|
||||
# Files that must be present for the multilingual model
|
||||
_MTL_WEIGHT_FILES = [
|
||||
"t3_mtl23ls_v2.safetensors",
|
||||
"s3gen.pt",
|
||||
"ve.pt",
|
||||
]
|
||||
|
||||
|
||||
class ChatterboxTTSBackend:
|
||||
"""Chatterbox Multilingual TTS backend for voice cloning."""
|
||||
|
||||
# Class-level lock for torch.load monkey-patching
|
||||
_load_lock: ClassVar[threading.Lock] = threading.Lock()
|
||||
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
self.model_size = "default"
|
||||
self._device = None
|
||||
self._model_load_lock = asyncio.Lock()
|
||||
|
||||
def _get_device(self) -> str:
|
||||
"""Get the best available device. Forces CPU on macOS (MPS issue)."""
|
||||
if platform.system() == "Darwin":
|
||||
return "cpu"
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
except ImportError:
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
return self.model is not None
|
||||
|
||||
def _get_model_path(self, model_size: str = "default") -> str:
|
||||
return CHATTERBOX_HF_REPO
|
||||
|
||||
def _is_model_cached(self, model_size: str = "default") -> bool:
|
||||
"""Check if the Chatterbox multilingual model is cached locally."""
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
repo_cache = Path(hf_constants.HF_HUB_CACHE) / (
|
||||
"models--" + CHATTERBOX_HF_REPO.replace("/", "--")
|
||||
)
|
||||
|
||||
if not repo_cache.exists():
|
||||
return False
|
||||
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
|
||||
return False
|
||||
|
||||
# Check for multilingual weight files
|
||||
snapshots_dir = repo_cache / "snapshots"
|
||||
if snapshots_dir.exists():
|
||||
for fname in _MTL_WEIGHT_FILES:
|
||||
if not any(snapshots_dir.rglob(fname)):
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking Chatterbox cache: {e}")
|
||||
return False
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None:
|
||||
"""Load the Chatterbox multilingual model."""
|
||||
if self.model is not None:
|
||||
return
|
||||
async with self._model_load_lock:
|
||||
if self.model is not None:
|
||||
return
|
||||
await asyncio.to_thread(self._load_model_sync)
|
||||
|
||||
def _load_model_sync(self):
|
||||
"""Synchronous model loading."""
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
task_manager = get_task_manager()
|
||||
model_name = "chatterbox-tts"
|
||||
|
||||
is_cached = self._is_model_cached()
|
||||
|
||||
# Set up HF progress tracking (intercepts tqdm for file-level progress)
|
||||
progress_callback = create_hf_progress_callback(model_name, progress_manager)
|
||||
tracker = HFProgressTracker(progress_callback, filter_non_downloads=is_cached)
|
||||
tracker_context = tracker.patch_download()
|
||||
tracker_context.__enter__()
|
||||
|
||||
if not is_cached:
|
||||
task_manager.start_download(model_name)
|
||||
progress_manager.update_progress(
|
||||
model_name=model_name,
|
||||
current=0,
|
||||
total=0,
|
||||
filename="Connecting to HuggingFace...",
|
||||
status="downloading",
|
||||
)
|
||||
|
||||
try:
|
||||
device = self._get_device()
|
||||
self._device = device
|
||||
|
||||
logger.info(f"Loading Chatterbox Multilingual TTS on {device}...")
|
||||
|
||||
import torch
|
||||
from chatterbox.mtl_tts import ChatterboxMultilingualTTS
|
||||
|
||||
# Monkey-patch torch.load for CPU loading. The model's .pt files
|
||||
# were saved on CUDA; from_pretrained() doesn't pass map_location
|
||||
# so loading on CPU fails without this.
|
||||
try:
|
||||
if device == "cpu":
|
||||
_orig_torch_load = torch.load
|
||||
|
||||
def _patched_load(*args, **kwargs):
|
||||
kwargs.setdefault("map_location", "cpu")
|
||||
return _orig_torch_load(*args, **kwargs)
|
||||
|
||||
with ChatterboxTTSBackend._load_lock:
|
||||
torch.load = _patched_load
|
||||
try:
|
||||
self.model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
device=device,
|
||||
)
|
||||
finally:
|
||||
torch.load = _orig_torch_load
|
||||
else:
|
||||
self.model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
device=device,
|
||||
)
|
||||
finally:
|
||||
tracker_context.__exit__(None, None, None)
|
||||
|
||||
# Fix: transformers >= 4.36 defaults LlamaModel to sdpa attention
|
||||
# which doesn't support output_attentions=True (needed by
|
||||
# Chatterbox's AlignmentStreamAnalyzer). Force eager attention.
|
||||
t3_tfmr = self.model.t3.tfmr
|
||||
if hasattr(t3_tfmr, "config") and hasattr(
|
||||
t3_tfmr.config, "_attn_implementation"
|
||||
):
|
||||
t3_tfmr.config._attn_implementation = "eager"
|
||||
for layer in getattr(t3_tfmr, "layers", []):
|
||||
if hasattr(layer, "self_attn"):
|
||||
layer.self_attn._attn_implementation = "eager"
|
||||
|
||||
if not is_cached:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
logger.info("Chatterbox Multilingual TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"chatterbox-tts package not found. "
|
||||
"Install with: pip install chatterbox-tts"
|
||||
)
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load Chatterbox: {e}")
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
if self.model is not None:
|
||||
device = self._device
|
||||
del self.model
|
||||
self.model = None
|
||||
self._device = None
|
||||
if device == "cuda":
|
||||
import torch
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
logger.info("Chatterbox unloaded")
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
reference_text: str,
|
||||
use_cache: bool = True,
|
||||
) -> Tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt from reference audio.
|
||||
|
||||
Chatterbox processes reference audio at generation time, so the
|
||||
prompt just stores the file path. The actual audio is loaded by
|
||||
model.generate() via audio_prompt_path.
|
||||
"""
|
||||
voice_prompt = {
|
||||
"ref_audio": str(audio_path),
|
||||
"ref_text": reference_text,
|
||||
}
|
||||
return voice_prompt, False
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: List[str],
|
||||
reference_texts: List[str],
|
||||
) -> Tuple[np.ndarray, str]:
|
||||
"""Combine multiple reference samples."""
|
||||
combined_audio = []
|
||||
for path in audio_paths:
|
||||
audio, _sr = load_audio(path)
|
||||
audio = normalize_audio(audio)
|
||||
combined_audio.append(audio)
|
||||
|
||||
mixed = np.concatenate(combined_audio)
|
||||
mixed = normalize_audio(mixed)
|
||||
combined_text = " ".join(reference_texts)
|
||||
return mixed, combined_text
|
||||
|
||||
# Per-language generation defaults. Lower temp + higher cfg = clearer speech.
|
||||
_LANG_DEFAULTS: ClassVar[dict] = {
|
||||
"he": {
|
||||
"exaggeration": 0.4,
|
||||
"cfg_weight": 0.7,
|
||||
"temperature": 0.65,
|
||||
"repetition_penalty": 2.5,
|
||||
},
|
||||
}
|
||||
_GLOBAL_DEFAULTS: ClassVar[dict] = {
|
||||
"exaggeration": 0.5,
|
||||
"cfg_weight": 0.5,
|
||||
"temperature": 0.8,
|
||||
"repetition_penalty": 2.0,
|
||||
}
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: Optional[int] = None,
|
||||
instruct: Optional[str] = None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio using Chatterbox Multilingual TTS.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize
|
||||
voice_prompt: Dict with ref_audio path
|
||||
language: BCP-47 language code
|
||||
seed: Random seed for reproducibility
|
||||
instruct: Unused (protocol compatibility)
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
await self.load_model()
|
||||
|
||||
ref_audio = voice_prompt.get("ref_audio")
|
||||
if ref_audio and not Path(ref_audio).exists():
|
||||
logger.warning(f"Reference audio not found: {ref_audio}")
|
||||
ref_audio = None
|
||||
|
||||
# Merge language-specific defaults with global defaults
|
||||
lang_defaults = self._LANG_DEFAULTS.get(language, self._GLOBAL_DEFAULTS)
|
||||
|
||||
def _generate_sync():
|
||||
import torch
|
||||
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
|
||||
logger.info(f"[Chatterbox] Generating: lang={language}")
|
||||
|
||||
wav = self.model.generate(
|
||||
text,
|
||||
language_id=language,
|
||||
audio_prompt_path=ref_audio,
|
||||
exaggeration=lang_defaults["exaggeration"],
|
||||
cfg_weight=lang_defaults["cfg_weight"],
|
||||
temperature=lang_defaults["temperature"],
|
||||
repetition_penalty=lang_defaults["repetition_penalty"],
|
||||
)
|
||||
|
||||
# Convert tensor -> numpy
|
||||
if isinstance(wav, torch.Tensor):
|
||||
audio = wav.squeeze().cpu().numpy().astype(np.float32)
|
||||
else:
|
||||
audio = np.asarray(wav, dtype=np.float32)
|
||||
|
||||
sample_rate = (
|
||||
getattr(self.model, "sr", None)
|
||||
or getattr(self.model, "sample_rate", 24000)
|
||||
)
|
||||
|
||||
return audio, sample_rate
|
||||
|
||||
return await asyncio.to_thread(_generate_sync)
|
||||
@@ -0,0 +1,307 @@
|
||||
"""
|
||||
Chatterbox Turbo TTS backend implementation.
|
||||
|
||||
Wraps ChatterboxTurboTTS from chatterbox-tts for fast, English-only
|
||||
voice cloning with paralinguistic tag support ([laugh], [cough], etc.).
|
||||
Forces CPU on macOS due to known MPS tensor issues.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import TTSBackend
|
||||
from ..utils.audio import normalize_audio, load_audio
|
||||
from ..utils.progress import get_progress_manager
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHATTERBOX_TURBO_HF_REPO = "ResembleAI/chatterbox-turbo"
|
||||
|
||||
# Files that must be present for the turbo model
|
||||
_TURBO_WEIGHT_FILES = [
|
||||
"t3_turbo_v1.safetensors",
|
||||
"s3gen_meanflow.safetensors",
|
||||
"ve.safetensors",
|
||||
]
|
||||
|
||||
|
||||
class ChatterboxTurboTTSBackend:
|
||||
"""Chatterbox Turbo TTS backend — fast, English-only, with paralinguistic tags."""
|
||||
|
||||
# Class-level lock for torch.load monkey-patching
|
||||
_load_lock: ClassVar[threading.Lock] = threading.Lock()
|
||||
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
self.model_size = "default"
|
||||
self._device = None
|
||||
self._model_load_lock = asyncio.Lock()
|
||||
|
||||
def _get_device(self) -> str:
|
||||
"""Get the best available device. Forces CPU on macOS (MPS issue)."""
|
||||
if platform.system() == "Darwin":
|
||||
return "cpu"
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
except ImportError:
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
return self.model is not None
|
||||
|
||||
def _get_model_path(self, model_size: str = "default") -> str:
|
||||
return CHATTERBOX_TURBO_HF_REPO
|
||||
|
||||
def _is_model_cached(self, model_size: str = "default") -> bool:
|
||||
"""Check if the Chatterbox Turbo model is cached locally."""
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
repo_cache = Path(hf_constants.HF_HUB_CACHE) / (
|
||||
"models--" + CHATTERBOX_TURBO_HF_REPO.replace("/", "--")
|
||||
)
|
||||
|
||||
if not repo_cache.exists():
|
||||
return False
|
||||
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
|
||||
return False
|
||||
|
||||
# Check for turbo weight files
|
||||
snapshots_dir = repo_cache / "snapshots"
|
||||
if snapshots_dir.exists():
|
||||
for fname in _TURBO_WEIGHT_FILES:
|
||||
if not any(snapshots_dir.rglob(fname)):
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking Chatterbox Turbo cache: {e}")
|
||||
return False
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None:
|
||||
"""Load the Chatterbox Turbo model."""
|
||||
if self.model is not None:
|
||||
return
|
||||
async with self._model_load_lock:
|
||||
if self.model is not None:
|
||||
return
|
||||
await asyncio.to_thread(self._load_model_sync)
|
||||
|
||||
def _load_model_sync(self):
|
||||
"""Synchronous model loading."""
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
task_manager = get_task_manager()
|
||||
model_name = "chatterbox-turbo"
|
||||
|
||||
is_cached = self._is_model_cached()
|
||||
|
||||
# Set up HF progress tracking (intercepts tqdm for file-level progress)
|
||||
progress_callback = create_hf_progress_callback(model_name, progress_manager)
|
||||
tracker = HFProgressTracker(progress_callback, filter_non_downloads=is_cached)
|
||||
tracker_context = tracker.patch_download()
|
||||
tracker_context.__enter__()
|
||||
|
||||
if not is_cached:
|
||||
task_manager.start_download(model_name)
|
||||
progress_manager.update_progress(
|
||||
model_name=model_name,
|
||||
current=0,
|
||||
total=0,
|
||||
filename="Connecting to HuggingFace...",
|
||||
status="downloading",
|
||||
)
|
||||
|
||||
try:
|
||||
device = self._get_device()
|
||||
self._device = device
|
||||
|
||||
logger.info(f"Loading Chatterbox Turbo TTS on {device}...")
|
||||
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from chatterbox.tts_turbo import ChatterboxTurboTTS
|
||||
|
||||
# Download model files ourselves so we can pass token=None
|
||||
# (upstream from_pretrained passes token=True which requires
|
||||
# a stored HF token even though the repo is public).
|
||||
try:
|
||||
local_path = snapshot_download(
|
||||
repo_id=CHATTERBOX_TURBO_HF_REPO,
|
||||
token=None,
|
||||
allow_patterns=[
|
||||
"*.safetensors", "*.json", "*.txt", "*.pt", "*.model",
|
||||
],
|
||||
)
|
||||
finally:
|
||||
tracker_context.__exit__(None, None, None)
|
||||
|
||||
# Monkey-patch torch.load for CPU loading. The model's .pt files
|
||||
# were saved on CUDA; from_local() doesn't pass map_location
|
||||
# so loading on CPU fails without this.
|
||||
if device == "cpu":
|
||||
_orig_torch_load = torch.load
|
||||
|
||||
def _patched_load(*args, **kwargs):
|
||||
kwargs.setdefault("map_location", "cpu")
|
||||
return _orig_torch_load(*args, **kwargs)
|
||||
|
||||
with ChatterboxTurboTTSBackend._load_lock:
|
||||
torch.load = _patched_load
|
||||
try:
|
||||
self.model = ChatterboxTurboTTS.from_local(
|
||||
local_path, device,
|
||||
)
|
||||
finally:
|
||||
torch.load = _orig_torch_load
|
||||
else:
|
||||
self.model = ChatterboxTurboTTS.from_local(
|
||||
local_path, device,
|
||||
)
|
||||
|
||||
if not is_cached:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
logger.info("Chatterbox Turbo TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"chatterbox-tts package not found. "
|
||||
"Install with: pip install chatterbox-tts"
|
||||
)
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load Chatterbox Turbo: {e}")
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
if self.model is not None:
|
||||
device = self._device
|
||||
del self.model
|
||||
self.model = None
|
||||
self._device = None
|
||||
if device == "cuda":
|
||||
import torch
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
logger.info("Chatterbox Turbo unloaded")
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
reference_text: str,
|
||||
use_cache: bool = True,
|
||||
) -> Tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt from reference audio.
|
||||
|
||||
Chatterbox Turbo processes reference audio at generation time, so the
|
||||
prompt just stores the file path.
|
||||
"""
|
||||
voice_prompt = {
|
||||
"ref_audio": str(audio_path),
|
||||
"ref_text": reference_text,
|
||||
}
|
||||
return voice_prompt, False
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: List[str],
|
||||
reference_texts: List[str],
|
||||
) -> Tuple[np.ndarray, str]:
|
||||
"""Combine multiple reference samples."""
|
||||
combined_audio = []
|
||||
for path in audio_paths:
|
||||
audio, _sr = load_audio(path)
|
||||
audio = normalize_audio(audio)
|
||||
combined_audio.append(audio)
|
||||
|
||||
mixed = np.concatenate(combined_audio)
|
||||
mixed = normalize_audio(mixed)
|
||||
combined_text = " ".join(reference_texts)
|
||||
return mixed, combined_text
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: Optional[int] = None,
|
||||
instruct: Optional[str] = None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio using Chatterbox Turbo TTS.
|
||||
|
||||
Supports paralinguistic tags in text: [laugh], [cough], [chuckle], etc.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize (may include paralinguistic tags)
|
||||
voice_prompt: Dict with ref_audio path
|
||||
language: Ignored (Turbo is English-only)
|
||||
seed: Random seed for reproducibility
|
||||
instruct: Unused (protocol compatibility)
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
await self.load_model()
|
||||
|
||||
ref_audio = voice_prompt.get("ref_audio")
|
||||
if ref_audio and not Path(ref_audio).exists():
|
||||
logger.warning(f"Reference audio not found: {ref_audio}")
|
||||
ref_audio = None
|
||||
|
||||
def _generate_sync():
|
||||
import torch
|
||||
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
|
||||
logger.info("[Chatterbox Turbo] Generating (English)")
|
||||
|
||||
wav = self.model.generate(
|
||||
text,
|
||||
audio_prompt_path=ref_audio,
|
||||
temperature=0.8,
|
||||
top_k=1000,
|
||||
top_p=0.95,
|
||||
repetition_penalty=1.2,
|
||||
)
|
||||
|
||||
# Convert tensor -> numpy
|
||||
if isinstance(wav, torch.Tensor):
|
||||
audio = wav.squeeze().cpu().numpy().astype(np.float32)
|
||||
else:
|
||||
audio = np.asarray(wav, dtype=np.float32)
|
||||
|
||||
sample_rate = (
|
||||
getattr(self.model, "sr", None)
|
||||
or getattr(self.model, "sample_rate", 24000)
|
||||
)
|
||||
|
||||
return audio, sample_rate
|
||||
|
||||
return await asyncio.to_thread(_generate_sync)
|
||||
@@ -94,19 +94,27 @@ class LuxTTSBackend:
|
||||
|
||||
def _load_model_sync(self):
|
||||
"""Synchronous model loading."""
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
task_manager = get_task_manager()
|
||||
model_name = "luxtts"
|
||||
|
||||
is_cached = self._is_model_cached()
|
||||
|
||||
# Set up HF progress tracking (intercepts tqdm for file-level progress)
|
||||
progress_callback = create_hf_progress_callback(model_name, progress_manager)
|
||||
tracker = HFProgressTracker(progress_callback, filter_non_downloads=is_cached)
|
||||
tracker_context = tracker.patch_download()
|
||||
tracker_context.__enter__()
|
||||
|
||||
if not is_cached:
|
||||
task_manager.start_download(model_name)
|
||||
progress_manager.update_progress(
|
||||
model_name=model_name,
|
||||
current=0,
|
||||
total=0,
|
||||
filename="Downloading LuxTTS model...",
|
||||
filename="Connecting to HuggingFace...",
|
||||
status="downloading",
|
||||
)
|
||||
|
||||
@@ -117,19 +125,22 @@ class LuxTTSBackend:
|
||||
logger.info(f"Loading LuxTTS on {device}...")
|
||||
|
||||
# LuxTTS constructor downloads model and loads everything
|
||||
if device == "cpu":
|
||||
import os
|
||||
threads = os.cpu_count() or 4
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device="cpu",
|
||||
threads=min(threads, 8),
|
||||
)
|
||||
else:
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device=device,
|
||||
)
|
||||
try:
|
||||
if device == "cpu":
|
||||
import os
|
||||
threads = os.cpu_count() or 4
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device="cpu",
|
||||
threads=min(threads, 8),
|
||||
)
|
||||
else:
|
||||
self.model = LuxTTS(
|
||||
model_path=LUXTTS_HF_REPO,
|
||||
device=device,
|
||||
)
|
||||
finally:
|
||||
tracker_context.__exit__(None, None, None)
|
||||
|
||||
if not is_cached:
|
||||
progress_manager.mark_complete(model_name)
|
||||
|
||||
+157
-4
@@ -234,7 +234,10 @@ async def create_profile(
|
||||
"""Create a new voice profile."""
|
||||
try:
|
||||
return await profiles.create_profile(data, db)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
# Fallback for unexpected errors
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@@ -290,10 +293,13 @@ async def update_profile(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Update a voice profile."""
|
||||
profile = await profiles.update_profile(profile_id, data, db)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
return profile
|
||||
try:
|
||||
profile = await profiles.update_profile(profile_id, data, db)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
return profile
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@app.delete("/profiles/{profile_id}")
|
||||
@@ -670,6 +676,52 @@ async def generate_speech(
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox":
|
||||
if not tts_model._is_model_cached():
|
||||
model_name = "chatterbox-tts"
|
||||
|
||||
async def download_chatterbox_background():
|
||||
try:
|
||||
await tts_model.load_model()
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
asyncio.create_task(download_chatterbox_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": "Chatterbox model is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox_turbo":
|
||||
if not tts_model._is_model_cached():
|
||||
model_name = "chatterbox-turbo"
|
||||
|
||||
async def download_chatterbox_turbo_background():
|
||||
try:
|
||||
await tts_model.load_model()
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
asyncio.create_task(download_chatterbox_turbo_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": "Chatterbox Turbo model is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
|
||||
# Create voice prompt from profile
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
@@ -687,6 +739,11 @@ async def generate_speech(
|
||||
data.instruct,
|
||||
)
|
||||
|
||||
# Trim trailing silence/hallucination for Chatterbox output
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
from .utils.audio import trim_tts_output
|
||||
audio = trim_tts_output(audio, sample_rate)
|
||||
|
||||
# Calculate duration
|
||||
duration = len(audio) / sample_rate
|
||||
|
||||
@@ -757,6 +814,20 @@ async def stream_speech(
|
||||
detail="LuxTTS model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox_turbo":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Chatterbox Turbo model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id, db, engine=engine,
|
||||
@@ -770,6 +841,11 @@ async def stream_speech(
|
||||
data.instruct,
|
||||
)
|
||||
|
||||
# Trim trailing silence/hallucination for Chatterbox output
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
from .utils.audio import trim_tts_output
|
||||
audio = trim_tts_output(audio, sample_rate)
|
||||
|
||||
wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
|
||||
|
||||
async def _wav_stream():
|
||||
@@ -1378,6 +1454,24 @@ async def get_model_status():
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Check if Chatterbox backend is loaded
|
||||
def check_chatterbox_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Check if Chatterbox Turbo backend is loaded
|
||||
def check_chatterbox_turbo_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
model_configs = [
|
||||
{
|
||||
"model_name": "qwen-tts-1.7B",
|
||||
@@ -1400,6 +1494,20 @@ async def get_model_status():
|
||||
"model_size": "default",
|
||||
"check_loaded": check_luxtts_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "chatterbox-tts",
|
||||
"display_name": "Chatterbox TTS (Multilingual)",
|
||||
"hf_repo_id": "ResembleAI/chatterbox",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "chatterbox-turbo",
|
||||
"display_name": "Chatterbox Turbo (English, Tags)",
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_turbo_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-base",
|
||||
"display_name": "Whisper Base",
|
||||
@@ -1551,6 +1659,7 @@ async def get_model_status():
|
||||
statuses.append(models.ModelStatus(
|
||||
model_name=config["model_name"],
|
||||
display_name=config["display_name"],
|
||||
hf_repo_id=config["hf_repo_id"],
|
||||
downloaded=downloaded,
|
||||
downloading=is_downloading,
|
||||
size_mb=size_mb,
|
||||
@@ -1569,6 +1678,7 @@ async def get_model_status():
|
||||
statuses.append(models.ModelStatus(
|
||||
model_name=config["model_name"],
|
||||
display_name=config["display_name"],
|
||||
hf_repo_id=config["hf_repo_id"],
|
||||
downloaded=False, # Assume not downloaded if check failed
|
||||
downloading=is_downloading,
|
||||
size_mb=None,
|
||||
@@ -1600,6 +1710,14 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("luxtts").load_model(),
|
||||
},
|
||||
"chatterbox-tts": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(),
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox_turbo").load_model(),
|
||||
},
|
||||
"whisper-base": {
|
||||
"model_size": "base",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
|
||||
@@ -1717,6 +1835,16 @@ async def delete_model(model_name: str):
|
||||
"model_size": "default",
|
||||
"model_type": "luxtts",
|
||||
},
|
||||
"chatterbox-tts": {
|
||||
"hf_repo_id": "ResembleAI/chatterbox",
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox",
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox_turbo",
|
||||
},
|
||||
"whisper-base": {
|
||||
"hf_repo_id": "openai/whisper-base",
|
||||
"model_size": "base",
|
||||
@@ -1756,6 +1884,16 @@ async def delete_model(model_name: str):
|
||||
luxtts = get_tts_backend_for_engine("luxtts")
|
||||
if luxtts.is_loaded():
|
||||
luxtts.unload_model()
|
||||
elif config["model_type"] == "chatterbox":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
chatterbox = get_tts_backend_for_engine("chatterbox")
|
||||
if chatterbox.is_loaded():
|
||||
chatterbox.unload_model()
|
||||
elif config["model_type"] == "chatterbox_turbo":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
turbo = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
if turbo.is_loaded():
|
||||
turbo.unload_model()
|
||||
elif config["model_type"] == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:
|
||||
@@ -1834,11 +1972,22 @@ async def get_active_tasks():
|
||||
pm_data = progress_manager._progress.get(model_name)
|
||||
if pm_data:
|
||||
error = pm_data.get("error")
|
||||
# Include progress data if available
|
||||
prog = progress or {}
|
||||
if not prog:
|
||||
with progress_manager._lock:
|
||||
pm_data = progress_manager._progress.get(model_name)
|
||||
if pm_data:
|
||||
prog = pm_data
|
||||
active_downloads.append(models.ActiveDownloadTask(
|
||||
model_name=model_name,
|
||||
status=task.status,
|
||||
started_at=task.started_at,
|
||||
error=error,
|
||||
progress=prog.get("progress"),
|
||||
current=prog.get("current"),
|
||||
total=prog.get("total"),
|
||||
filename=prog.get("filename"),
|
||||
))
|
||||
elif progress:
|
||||
# Progress exists but no task - create from progress data
|
||||
@@ -1856,6 +2005,10 @@ async def get_active_tasks():
|
||||
status=progress.get("status", "downloading"),
|
||||
started_at=started_at,
|
||||
error=progress.get("error"),
|
||||
progress=progress.get("progress"),
|
||||
current=progress.get("current"),
|
||||
total=progress.get("total"),
|
||||
filename=progress.get("filename"),
|
||||
))
|
||||
|
||||
# Get active generations
|
||||
|
||||
+8
-3
@@ -11,7 +11,7 @@ class VoiceProfileCreate(BaseModel):
|
||||
"""Request model for creating a voice profile."""
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
description: Optional[str] = Field(None, max_length=500)
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it)$")
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$")
|
||||
|
||||
|
||||
class VoiceProfileResponse(BaseModel):
|
||||
@@ -53,11 +53,11 @@ class GenerationRequest(BaseModel):
|
||||
"""Request model for voice generation."""
|
||||
profile_id: str
|
||||
text: str = Field(..., min_length=1, max_length=5000)
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it)$")
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
|
||||
seed: Optional[int] = Field(None, ge=0)
|
||||
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
|
||||
instruct: Optional[str] = Field(None, max_length=500)
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts)$")
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo)$")
|
||||
|
||||
|
||||
class GenerationResponse(BaseModel):
|
||||
@@ -135,6 +135,7 @@ class ModelStatus(BaseModel):
|
||||
"""Response model for model status."""
|
||||
model_name: str
|
||||
display_name: str
|
||||
hf_repo_id: Optional[str] = None # HuggingFace repository ID
|
||||
downloaded: bool
|
||||
downloading: bool = False # True if download is in progress
|
||||
size_mb: Optional[float] = None
|
||||
@@ -157,6 +158,10 @@ class ActiveDownloadTask(BaseModel):
|
||||
status: str
|
||||
started_at: datetime
|
||||
error: Optional[str] = None
|
||||
progress: Optional[float] = None # 0-100 percentage
|
||||
current: Optional[int] = None # bytes downloaded
|
||||
total: Optional[int] = None # total bytes
|
||||
filename: Optional[str] = None # current file being downloaded
|
||||
|
||||
|
||||
class ActiveGenerationTask(BaseModel):
|
||||
|
||||
+27
-10
@@ -38,14 +38,22 @@ async def create_profile(
|
||||
) -> VoiceProfileResponse:
|
||||
"""
|
||||
Create a new voice profile.
|
||||
|
||||
|
||||
Args:
|
||||
data: Profile creation data
|
||||
db: Database session
|
||||
|
||||
|
||||
Returns:
|
||||
Created profile
|
||||
|
||||
Raises:
|
||||
ValueError: If a profile with the same name already exists
|
||||
"""
|
||||
# Check if profile name already exists
|
||||
existing_profile = db.query(DBVoiceProfile).filter_by(name=data.name).first()
|
||||
if existing_profile:
|
||||
raise ValueError(f"A profile with the name '{data.name}' already exists. Please choose a different name.")
|
||||
|
||||
# Create profile in database
|
||||
db_profile = DBVoiceProfile(
|
||||
id=str(uuid.uuid4()),
|
||||
@@ -55,15 +63,15 @@ async def create_profile(
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
|
||||
db.add(db_profile)
|
||||
db.commit()
|
||||
db.refresh(db_profile)
|
||||
|
||||
|
||||
# Create profile directory
|
||||
profile_dir = _get_profiles_dir() / db_profile.id
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
return VoiceProfileResponse.model_validate(db_profile)
|
||||
|
||||
|
||||
@@ -191,28 +199,37 @@ async def update_profile(
|
||||
) -> Optional[VoiceProfileResponse]:
|
||||
"""
|
||||
Update a voice profile.
|
||||
|
||||
|
||||
Args:
|
||||
profile_id: Profile ID
|
||||
data: Updated profile data
|
||||
db: Database session
|
||||
|
||||
|
||||
Returns:
|
||||
Updated profile or None if not found
|
||||
|
||||
Raises:
|
||||
ValueError: If a profile with the same name already exists (different profile)
|
||||
"""
|
||||
profile = db.query(DBVoiceProfile).filter_by(id=profile_id).first()
|
||||
if not profile:
|
||||
return None
|
||||
|
||||
|
||||
# Check if the new name conflicts with another profile
|
||||
if profile.name != data.name:
|
||||
existing_profile = db.query(DBVoiceProfile).filter_by(name=data.name).first()
|
||||
if existing_profile:
|
||||
raise ValueError(f"A profile with the name '{data.name}' already exists. Please choose a different name.")
|
||||
|
||||
# Update fields
|
||||
profile.name = data.name
|
||||
profile.description = data.description
|
||||
profile.language = data.language
|
||||
profile.updated_at = datetime.utcnow()
|
||||
|
||||
|
||||
db.commit()
|
||||
db.refresh(profile)
|
||||
|
||||
|
||||
return VoiceProfileResponse.model_validate(profile)
|
||||
|
||||
|
||||
|
||||
@@ -21,6 +21,18 @@ qwen-tts>=0.0.5
|
||||
linacodec @ git+https://github.com/ysharma3501/LinaCodec.git
|
||||
Zipvoice @ git+https://github.com/ysharma3501/LuxTTS.git
|
||||
|
||||
# Chatterbox TTS sub-dependencies (chatterbox-tts itself is installed
|
||||
# --no-deps in the setup script because it pins numpy<1.26 / torch==2.6
|
||||
# which are incompatible with Python 3.12+)
|
||||
conformer>=0.3.2
|
||||
diffusers>=0.29.0
|
||||
omegaconf
|
||||
pykakasi
|
||||
resemble-perth>=1.0.1
|
||||
s3tokenizer
|
||||
spacy-pkuseg
|
||||
pyloudnorm
|
||||
|
||||
# Audio processing
|
||||
librosa>=0.10.0
|
||||
soundfile>=0.12.0
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Tests for profile duplicate name validation.
|
||||
|
||||
This test suite verifies that the application correctly handles
|
||||
duplicate profile names and provides user-friendly error messages.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import tempfile
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
# Add parent directory to path to import backend modules
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from database import Base, VoiceProfile as DBVoiceProfile
|
||||
from models import VoiceProfileCreate
|
||||
from profiles import create_profile, update_profile
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_db():
|
||||
"""Create a temporary test database."""
|
||||
# Create temporary directory for test database
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test.db"
|
||||
|
||||
# Create engine and session
|
||||
engine = create_engine(f"sqlite:///{db_path}")
|
||||
Base.metadata.create_all(bind=engine)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
db = SessionLocal()
|
||||
|
||||
yield db
|
||||
|
||||
# Cleanup
|
||||
db.close()
|
||||
shutil.rmtree(temp_dir)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_profiles_dir(monkeypatch, tmp_path):
|
||||
"""Mock the profiles directory to use a temporary path."""
|
||||
import profiles
|
||||
monkeypatch.setattr(profiles, '_get_profiles_dir', lambda: tmp_path)
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_profile_duplicate_name_raises_error(test_db, mock_profiles_dir):
|
||||
"""Test that creating a profile with a duplicate name raises a ValueError."""
|
||||
# Create first profile
|
||||
profile_data_1 = VoiceProfileCreate(
|
||||
name="Test Profile",
|
||||
description="First profile",
|
||||
language="en"
|
||||
)
|
||||
|
||||
profile_1 = await create_profile(profile_data_1, test_db)
|
||||
assert profile_1.name == "Test Profile"
|
||||
|
||||
# Try to create second profile with same name
|
||||
profile_data_2 = VoiceProfileCreate(
|
||||
name="Test Profile",
|
||||
description="Second profile",
|
||||
language="en"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
await create_profile(profile_data_2, test_db)
|
||||
|
||||
# Verify error message is user-friendly
|
||||
assert "already exists" in str(exc_info.value)
|
||||
assert "Test Profile" in str(exc_info.value)
|
||||
assert "choose a different name" in str(exc_info.value).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_profile_different_names_succeeds(test_db, mock_profiles_dir):
|
||||
"""Test that creating profiles with different names succeeds."""
|
||||
# Create first profile
|
||||
profile_data_1 = VoiceProfileCreate(
|
||||
name="Profile One",
|
||||
description="First profile",
|
||||
language="en"
|
||||
)
|
||||
|
||||
profile_1 = await create_profile(profile_data_1, test_db)
|
||||
assert profile_1.name == "Profile One"
|
||||
|
||||
# Create second profile with different name
|
||||
profile_data_2 = VoiceProfileCreate(
|
||||
name="Profile Two",
|
||||
description="Second profile",
|
||||
language="en"
|
||||
)
|
||||
|
||||
profile_2 = await create_profile(profile_data_2, test_db)
|
||||
assert profile_2.name == "Profile Two"
|
||||
|
||||
# Verify both profiles exist
|
||||
assert profile_1.id != profile_2.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_profile_to_duplicate_name_raises_error(test_db, mock_profiles_dir):
|
||||
"""Test that updating a profile to a duplicate name raises a ValueError."""
|
||||
# Create two profiles with different names
|
||||
profile_data_1 = VoiceProfileCreate(
|
||||
name="Profile A",
|
||||
description="First profile",
|
||||
language="en"
|
||||
)
|
||||
profile_1 = await create_profile(profile_data_1, test_db)
|
||||
|
||||
profile_data_2 = VoiceProfileCreate(
|
||||
name="Profile B",
|
||||
description="Second profile",
|
||||
language="en"
|
||||
)
|
||||
profile_2 = await create_profile(profile_data_2, test_db)
|
||||
|
||||
# Try to update profile_2 to use profile_1's name
|
||||
update_data = VoiceProfileCreate(
|
||||
name="Profile A", # Duplicate name
|
||||
description="Updated description",
|
||||
language="en"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
await update_profile(profile_2.id, update_data, test_db)
|
||||
|
||||
# Verify error message is user-friendly
|
||||
assert "already exists" in str(exc_info.value)
|
||||
assert "Profile A" in str(exc_info.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_profile_keep_same_name_succeeds(test_db, mock_profiles_dir):
|
||||
"""Test that updating a profile while keeping the same name succeeds."""
|
||||
# Create profile
|
||||
profile_data = VoiceProfileCreate(
|
||||
name="My Profile",
|
||||
description="Original description",
|
||||
language="en"
|
||||
)
|
||||
profile = await create_profile(profile_data, test_db)
|
||||
|
||||
# Update profile with same name but different description
|
||||
update_data = VoiceProfileCreate(
|
||||
name="My Profile", # Same name
|
||||
description="Updated description",
|
||||
language="en"
|
||||
)
|
||||
|
||||
updated_profile = await update_profile(profile.id, update_data, test_db)
|
||||
|
||||
# Verify update succeeded
|
||||
assert updated_profile is not None
|
||||
assert updated_profile.id == profile.id
|
||||
assert updated_profile.name == "My Profile"
|
||||
assert updated_profile.description == "Updated description"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_profile_to_new_unique_name_succeeds(test_db, mock_profiles_dir):
|
||||
"""Test that updating a profile to a new unique name succeeds."""
|
||||
# Create profile
|
||||
profile_data = VoiceProfileCreate(
|
||||
name="Original Name",
|
||||
description="Profile description",
|
||||
language="en"
|
||||
)
|
||||
profile = await create_profile(profile_data, test_db)
|
||||
|
||||
# Update profile with new unique name
|
||||
update_data = VoiceProfileCreate(
|
||||
name="New Unique Name",
|
||||
description="Updated description",
|
||||
language="en"
|
||||
)
|
||||
|
||||
updated_profile = await update_profile(profile.id, update_data, test_db)
|
||||
|
||||
# Verify update succeeded
|
||||
assert updated_profile is not None
|
||||
assert updated_profile.id == profile.id
|
||||
assert updated_profile.name == "New Unique Name"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_case_sensitive_names_allowed(test_db, mock_profiles_dir):
|
||||
"""Test that profile names are case-sensitive (e.g., 'Test' and 'test' are different)."""
|
||||
# Create profile with lowercase name
|
||||
profile_data_1 = VoiceProfileCreate(
|
||||
name="test profile",
|
||||
description="Lowercase",
|
||||
language="en"
|
||||
)
|
||||
profile_1 = await create_profile(profile_data_1, test_db)
|
||||
|
||||
# Create profile with different case
|
||||
profile_data_2 = VoiceProfileCreate(
|
||||
name="Test Profile",
|
||||
description="Title case",
|
||||
language="en"
|
||||
)
|
||||
profile_2 = await create_profile(profile_data_2, test_db)
|
||||
|
||||
# Both should succeed since SQLite unique constraint is case-sensitive by default
|
||||
assert profile_1.name == "test profile"
|
||||
assert profile_2.name == "Test Profile"
|
||||
assert profile_1.id != profile_2.id
|
||||
@@ -80,6 +80,95 @@ def save_audio(
|
||||
sf.write(path, audio, sample_rate)
|
||||
|
||||
|
||||
def trim_tts_output(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = 24000,
|
||||
frame_ms: int = 20,
|
||||
silence_threshold_db: float = -40.0,
|
||||
min_silence_ms: int = 200,
|
||||
max_internal_silence_ms: int = 1000,
|
||||
fade_ms: int = 30,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Trim trailing silence and post-silence hallucination from TTS output.
|
||||
|
||||
Chatterbox sometimes produces ``[speech][silence][hallucinated noise]``.
|
||||
This detects internal silence gaps longer than *max_internal_silence_ms*
|
||||
and cuts the audio at that boundary, then trims trailing silence and
|
||||
applies a short cosine fade-out.
|
||||
|
||||
Args:
|
||||
audio: Input audio array (mono float32)
|
||||
sample_rate: Sample rate in Hz
|
||||
frame_ms: Frame size for RMS energy calculation
|
||||
silence_threshold_db: dB threshold below which a frame is silence
|
||||
min_silence_ms: Minimum trailing silence to keep
|
||||
max_internal_silence_ms: Cut after any silence gap longer than this
|
||||
fade_ms: Cosine fade-out duration in ms
|
||||
|
||||
Returns:
|
||||
Trimmed audio array
|
||||
"""
|
||||
frame_len = int(sample_rate * frame_ms / 1000)
|
||||
if frame_len == 0 or len(audio) < frame_len:
|
||||
return audio
|
||||
|
||||
n_frames = len(audio) // frame_len
|
||||
threshold_linear = 10 ** (silence_threshold_db / 20)
|
||||
|
||||
# Compute per-frame RMS
|
||||
rms = np.array(
|
||||
[
|
||||
np.sqrt(np.mean(audio[i * frame_len : (i + 1) * frame_len] ** 2))
|
||||
for i in range(n_frames)
|
||||
]
|
||||
)
|
||||
is_speech = rms >= threshold_linear
|
||||
|
||||
# Find first speech frame
|
||||
first_speech = 0
|
||||
for i, s in enumerate(is_speech):
|
||||
if s:
|
||||
first_speech = max(0, i - 1) # keep 1 frame padding
|
||||
break
|
||||
|
||||
# Walk forward from first speech; cut at long internal silence gaps
|
||||
max_silence_frames = int(max_internal_silence_ms / frame_ms)
|
||||
consecutive_silence = 0
|
||||
cut_frame = n_frames
|
||||
|
||||
for i in range(first_speech, n_frames):
|
||||
if is_speech[i]:
|
||||
consecutive_silence = 0
|
||||
else:
|
||||
consecutive_silence += 1
|
||||
if consecutive_silence >= max_silence_frames:
|
||||
cut_frame = i - consecutive_silence + 1
|
||||
break
|
||||
|
||||
# Trim trailing silence from the cut point
|
||||
min_silence_frames = int(min_silence_ms / frame_ms)
|
||||
end_frame = cut_frame
|
||||
while end_frame > first_speech and not is_speech[end_frame - 1]:
|
||||
end_frame -= 1
|
||||
# Keep a short tail
|
||||
end_frame = min(end_frame + min_silence_frames, cut_frame)
|
||||
|
||||
# Convert frames back to samples
|
||||
start_sample = first_speech * frame_len
|
||||
end_sample = min(end_frame * frame_len, len(audio))
|
||||
|
||||
trimmed = audio[start_sample:end_sample].copy()
|
||||
|
||||
# Cosine fade-out
|
||||
fade_samples = int(sample_rate * fade_ms / 1000)
|
||||
if fade_samples > 0 and len(trimmed) > fade_samples:
|
||||
fade = np.cos(np.linspace(0, np.pi / 2, fade_samples)) ** 2
|
||||
trimmed[-fade_samples:] *= fade
|
||||
|
||||
return trimmed
|
||||
|
||||
|
||||
def validate_reference_audio(
|
||||
audio_path: str,
|
||||
min_duration: float = 2.0,
|
||||
|
||||
@@ -162,7 +162,7 @@ chmod +x voicebox-*.AppImage
|
||||
**Solutions:**
|
||||
1. **Check server is running**
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:17493/health
|
||||
```
|
||||
|
||||
2. **Check remote mode**
|
||||
@@ -170,7 +170,7 @@ chmod +x voicebox-*.AppImage
|
||||
- Check firewall settings
|
||||
|
||||
3. **Check port availability**
|
||||
- Default port is 8000
|
||||
- The current local app and dev workflow uses port 17493 by default
|
||||
- Ensure no other service is using it
|
||||
|
||||
### CORS errors in browser
|
||||
@@ -276,7 +276,7 @@ chmod +x voicebox-*.AppImage
|
||||
|
||||
2. **Check OpenAPI endpoint**
|
||||
```bash
|
||||
curl http://localhost:8000/openapi.json
|
||||
curl http://localhost:17493/openapi.json
|
||||
```
|
||||
|
||||
3. **Regenerate client**
|
||||
|
||||
+200
-190
@@ -1,6 +1,6 @@
|
||||
# Voicebox Project Status & Roadmap
|
||||
|
||||
> Last updated: 2026-03-12 | Current version: **v0.1.13** | 13.1k stars | 176 open issues | 28 open PRs
|
||||
> Last updated: 2026-03-13 | Current version: **v0.1.13** | 13.1k stars | ~176 open issues | 25 open PRs
|
||||
|
||||
---
|
||||
|
||||
@@ -30,14 +30,18 @@
|
||||
│ │ HTTP :17493 │
|
||||
│ ┌──────────────────────▼────────────────────────┐ │
|
||||
│ │ FastAPI Backend (backend/) │ │
|
||||
│ │ ┌─────────────┐ ┌───────────┐ ┌─────────┐ │ │
|
||||
│ │ │ TTSBackend │ │ STTBackend│ │ Profiles│ │ │
|
||||
│ │ │ (Protocol) │ │ (Whisper) │ │ History │ │ │
|
||||
│ │ │ ┌────────┐ │ └───────────┘ │ Stories │ │ │
|
||||
│ │ │ │PyTorch │ │ └─────────┘ │ │
|
||||
│ │ │ │or MLX │ │ │ │
|
||||
│ │ │ └────────┘ │ │ │
|
||||
│ │ └─────────────┘ │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ TTSBackend Protocol │ │ │
|
||||
│ │ │ ┌──────────┐ ┌───────┐ ┌───────────┐ │ │ │
|
||||
│ │ │ │ Qwen3-TTS│ │LuxTTS │ │Chatterbox │ │ │ │
|
||||
│ │ │ │(Py/MLX) │ │ │ │(MTL+Turbo)│ │ │ │
|
||||
│ │ │ └──────────┘ └───────┘ └───────────┘ │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ ┌───────────┐ ┌─────────┐ │ │
|
||||
│ │ │ STTBackend│ │ Profiles│ │ │
|
||||
│ │ │ (Whisper) │ │ History │ │ │
|
||||
│ │ └───────────┘ │ Stories │ │ │
|
||||
│ │ └─────────┘ │ │
|
||||
│ └───────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
@@ -46,131 +50,180 @@
|
||||
|
||||
| Layer | File | Purpose |
|
||||
|-------|------|---------|
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~1700 lines) |
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2100 lines) |
|
||||
| TTS protocol | `backend/backends/__init__.py:14-81` | `TTSBackend` Protocol definition |
|
||||
| TTS factory | `backend/backends/__init__.py:118-137` | Singleton backend selection (MLX vs PyTorch) |
|
||||
| TTS factory | `backend/backends/__init__.py:138-178` | Thread-safe engine registry (double-checked locking) |
|
||||
| PyTorch TTS | `backend/backends/pytorch_backend.py` | Qwen3-TTS via `qwen_tts` package |
|
||||
| MLX TTS | `backend/backends/mlx_backend.py` | Qwen3-TTS via `mlx_audio.tts` |
|
||||
| LuxTTS | `backend/backends/luxtts_backend.py` | LuxTTS — fast, CPU-friendly |
|
||||
| Chatterbox MTL | `backend/backends/chatterbox_backend.py` | Chatterbox Multilingual — 23 languages |
|
||||
| Chatterbox Turbo | `backend/backends/chatterbox_turbo_backend.py` | Chatterbox Turbo — English, paralinguistic tags |
|
||||
| Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch |
|
||||
| API types | `backend/models.py` | Pydantic request/response models |
|
||||
| HF progress | `backend/utils/hf_progress.py` | HFProgressTracker (tqdm patching for download progress) |
|
||||
| Audio utils | `backend/utils/audio.py` | `trim_tts_output()`, normalize, load/save audio |
|
||||
| Frontend API | `app/src/lib/api/client.ts` | Hand-written fetch wrapper |
|
||||
| Frontend types | `app/src/lib/api/types.ts` | TypeScript API types |
|
||||
| Generation form | `app/src/components/Generation/GenerationForm.tsx` | TTS generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status UI |
|
||||
| Floating gen box | `app/src/components/Generation/FloatingGenerateBox.tsx` | Compact generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status/progress UI |
|
||||
| GPU acceleration | `app/src/components/ServerSettings/GpuAcceleration.tsx` | CUDA backend swap UI |
|
||||
| Gen form hook | `app/src/lib/hooks/useGenerationForm.ts` | Form validation + submission |
|
||||
| Language constants | `app/src/lib/constants/languages.ts` | Per-engine language maps |
|
||||
|
||||
### How TTS Generation Works (Current Flow)
|
||||
|
||||
```
|
||||
POST /generate
|
||||
1. Look up voice profile from DB
|
||||
2. Check model cache → if missing, trigger background download, return HTTP 202
|
||||
3. Load model (lazy): tts_backend.load_model(model_size)
|
||||
4. Create voice prompt: profiles.create_voice_prompt_for_profile()
|
||||
2. Resolve engine from request (qwen | luxtts | chatterbox | chatterbox_turbo)
|
||||
3. Get backend: get_tts_backend_for_engine(engine) # thread-safe singleton per engine
|
||||
4. Check model cache → if missing, trigger background download, return HTTP 202
|
||||
5. Load model (lazy): tts_backend.load_model(model_size)
|
||||
6. Create voice prompt: profiles.create_voice_prompt_for_profile(engine=engine)
|
||||
→ tts_backend.create_voice_prompt(audio_path, reference_text)
|
||||
5. Generate: tts_backend.generate(text, voice_prompt, language, seed, instruct)
|
||||
6. Save WAV → data/generations/{id}.wav
|
||||
7. Insert history record in SQLite
|
||||
8. Return GenerationResponse
|
||||
7. Generate: tts_backend.generate(text, voice_prompt, language, seed, instruct)
|
||||
8. Post-process: trim_tts_output() for Chatterbox engines
|
||||
9. Save WAV → data/generations/{id}.wav
|
||||
10. Insert history record in SQLite
|
||||
11. Return GenerationResponse
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Current State
|
||||
|
||||
### What's Shipped (v0.1.13)
|
||||
### What's Shipped (v0.1.13 + recent merges)
|
||||
|
||||
**Core TTS:**
|
||||
- Qwen3-TTS voice cloning (1.7B and 0.6B models)
|
||||
- MLX backend for Apple Silicon, PyTorch for everything else
|
||||
- Multi-engine TTS architecture with thread-safe backend registry (PR #254)
|
||||
- LuxTTS integration — fast, CPU-friendly English TTS (PR #254)
|
||||
- Chatterbox Multilingual TTS — 23 languages including Hebrew (PR #257)
|
||||
- Delivery instructions (instruct parameter, Qwen only)
|
||||
- Single flat model dropdown (Qwen 1.7B, Qwen 0.6B, LuxTTS, Chatterbox, Chatterbox Turbo)
|
||||
|
||||
**Infrastructure:**
|
||||
- CUDA backend swap via binary download and restart (PR #252)
|
||||
- GPU acceleration settings UI
|
||||
- Voice profiles with multi-sample support
|
||||
- Stories editor (multi-track DAW timeline)
|
||||
- Whisper transcription (base, small, medium, large variants)
|
||||
- Model management UI with download progress (SSE)
|
||||
- Model management UI with inline download progress bars (HFProgressTracker)
|
||||
- Download cancel/clear UI with error panel (PR #238)
|
||||
- Generation history with caching
|
||||
- Streaming generation endpoint (MLX only)
|
||||
- Delivery instructions (instruct parameter)
|
||||
- Duplicate profile name validation (PR #175)
|
||||
- Linux NVIDIA GBM buffer + WebKitGTK microphone fix (PR #210)
|
||||
|
||||
### What's NOT Shipped But Has Code
|
||||
### What's In-Flight
|
||||
|
||||
| Feature | Branch | Status |
|
||||
|---------|--------|--------|
|
||||
| External provider binaries (CUDA split) | `external-provider-binaries` | PR #33, significant work done, stale since Feb |
|
||||
| Dual server binaries | `feat/dual-server-binaries` | Branch exists, no PR |
|
||||
| Multi-sample fix | `fix-multi-sample` | Branch exists, no PR |
|
||||
| Model download notification fix | `fix-dl-notification-...` | Branch exists, no PR |
|
||||
| Feature | Branch/PR | Status |
|
||||
|---------|-----------|--------|
|
||||
| Chatterbox Turbo + per-engine language lists | `feat/chatterbox-turbo` / PR #258 | Open, ready for review |
|
||||
|
||||
### Hardcoded Qwen3-TTS Assumptions
|
||||
### TTS Engine Comparison
|
||||
|
||||
These are the specific coupling points that block multi-model support:
|
||||
| Engine | Model Name | Languages | Size | Key Features |
|
||||
|--------|-----------|-----------|------|-------------|
|
||||
| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Instruct mode, highest quality |
|
||||
| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster |
|
||||
| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast |
|
||||
| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual |
|
||||
| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency |
|
||||
|
||||
| Location | What's Hardcoded |
|
||||
|----------|-----------------|
|
||||
| `backend/models.py:58` | `model_size` regex: `^(1\.7B\|0\.6B)$` |
|
||||
| `backend/main.py:611` | Default: `model_size or "1.7B"` |
|
||||
| `backend/main.py:1322-1365` | Model status list (2 Qwen + 4 Whisper) |
|
||||
| `backend/main.py:1523-1548` | Download trigger map |
|
||||
| `backend/main.py:1597-1628` | Delete map |
|
||||
| `backend/backends/pytorch_backend.py:65-68` | HF repo ID map |
|
||||
| `backend/backends/mlx_backend.py:41-44` | MLX repo ID map |
|
||||
| `backend/backends/__init__.py:118-137` | Single global TTS backend |
|
||||
| `app/src/lib/hooks/useGenerationForm.ts:17` | `modelSize: z.enum(['1.7B', '0.6B'])` |
|
||||
| `app/src/lib/hooks/useGenerationForm.ts:70-71` | `modelName = "qwen-tts-${data.modelSize}"` |
|
||||
| `app/src/components/Generation/GenerationForm.tsx:140-141` | Hardcoded "Qwen TTS" labels |
|
||||
| `app/src/components/ServerSettings/ModelManagement.tsx:166-213` | Filters by `qwen-tts` and `whisper` prefix |
|
||||
| `backend/utils/cache.py` | Voice prompt cache uses `torch.save()` |
|
||||
### Multi-Engine Architecture (Shipped)
|
||||
|
||||
The singleton TTS backend blocker described in the previous version of this doc has been **resolved**. The architecture now supports:
|
||||
|
||||
- **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking
|
||||
- **Per-engine backend instances** — each engine gets its own singleton, loaded lazily
|
||||
- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo'`
|
||||
- **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages
|
||||
- **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend
|
||||
- **Trim post-processing** — `trim_tts_output()` for Chatterbox engines (cuts trailing silence/hallucination)
|
||||
|
||||
### Known Limitations
|
||||
|
||||
- **HF XET progress**: Large files downloaded via `hf-xet` (HuggingFace's new transfer backend) report `n=0` in tqdm updates. Progress bars may appear stuck for large `.safetensors` files even though the download is proceeding. This is a known upstream limitation.
|
||||
- **Chatterbox Turbo upstream token bug**: `from_pretrained()` passes `token=os.getenv("HF_TOKEN") or True` which fails without a stored HF token. Our backend works around this by calling `snapshot_download(token=None)` + `from_local()`.
|
||||
- **chatterbox-tts must install with `--no-deps`**: It pins `numpy<1.26`, `torch==2.6.0`, `transformers==4.46.3` — all incompatible with our stack (Python 3.12, torch 2.10, transformers 4.57.3). Sub-deps listed explicitly in `requirements.txt`.
|
||||
- **Streaming generation** only works for Qwen on MLX. Other engines use the non-streaming `/generate` endpoint.
|
||||
- **dicta-onnx** (Hebrew diacritization) not included — upstream Chatterbox bug requires `model_path` arg but calls `Dicta()` with none. Hebrew works fine without it.
|
||||
|
||||
---
|
||||
|
||||
## Open PRs — Triage & Analysis
|
||||
|
||||
### Recently Merged (Since Last Update)
|
||||
|
||||
| PR | Title | Merged |
|
||||
|----|-------|--------|
|
||||
| **#257** | feat: Chatterbox TTS engine with multilingual voice cloning | 2026-03-13 |
|
||||
| **#254** | feat: LuxTTS integration — multi-engine TTS support | 2026-03-13 |
|
||||
| **#252** | feat: CUDA backend swap via binary download and restart | 2026-03-13 |
|
||||
| **#238** | Download cancel/clear UI, fixed model downloading | 2026-03-13 |
|
||||
| **#250** | docs: align local API port examples | 2026-03-13 |
|
||||
| **#210** | fix: Linux NVIDIA GBM buffer crash | 2026-03-13 |
|
||||
| **#175** | Fix #134: duplicate profile name validation | 2026-03-13 |
|
||||
|
||||
### In-Flight (Our Work)
|
||||
|
||||
| PR | Title | Status | Notes |
|
||||
|----|-------|--------|-------|
|
||||
| **#258** | feat: Chatterbox Turbo engine + per-engine language lists | Open | Ready for review. Adds Turbo engine + dynamic language dropdown. |
|
||||
|
||||
### Merge-Ready / Near-Ready (Bug Fixes & Small Features)
|
||||
|
||||
| PR | Title | Risk | Notes |
|
||||
|----|-------|------|-------|
|
||||
| **#250** | docs: align local API port examples | None | Docs-only |
|
||||
| **#230** | docs: fix README grammar | None | Docs-only |
|
||||
| **#243** | a11y: screen reader and keyboard improvements | Low | Accessibility, no backend changes |
|
||||
| **#175** | Fix #134: duplicate profile name validation | Low | Simple validation |
|
||||
| **#178** | Fix #168 #140: generation error handling | Low | Error handling improvements |
|
||||
| **#152** | Fix: prevent crashes when HuggingFace unreachable | Medium | Monkey-patches HF hub; solves real offline bug (#150, #151) |
|
||||
| **#218** | fix: unify qwen tts cache dir on Windows | Low | Windows-specific path fix |
|
||||
| **#214** | fix: panic on launch from tokio::spawn | Low | Rust-side Tauri fix |
|
||||
| **#210** | fix: Linux NVIDIA GBM buffer crash | Low | Linux-specific, narrowly scoped |
|
||||
| **#88** | security: restrict CORS to known local origins | Low | Security hardening |
|
||||
| **#133** | feat: network access toggle | Low | Wires up existing plumbing |
|
||||
|
||||
### Significant Feature PRs
|
||||
|
||||
| PR | Title | Complexity | Dependencies | Notes |
|
||||
|----|-------|-----------|--------------|-------|
|
||||
| **#97** | fix: pass language parameter to TTS models | Medium | None | **Critical bug** — language param was silently dropped. Adds `LANGUAGE_CODE_TO_NAME` mapping to both backends. Should be high priority. |
|
||||
| **#133** | feat: network access toggle | Low | None | Wires up existing plumbing (`--host 0.0.0.0`). Clean, small. |
|
||||
| **#238** | download cancel/clear UI + error panel | Medium | None | Adds cancel buttons, VS Code-style Problems panel, fixes whisper-large repo. Quality-of-life win. |
|
||||
| **#99** | feat: chunked TTS with quality selector | Medium | None | Solves the 500-char/2048-token limit. Sentence-aware splitting, crossfade concat, 44.1kHz upsampling. Addresses #191, #203, #69, #111. |
|
||||
| **#154** | feat: Audiobook tab | Medium | Depends on #99 concepts | Full audiobook workflow — chunked gen, preview, auto-save to Stories. New route + tab. |
|
||||
| **#91** | fix: CoreAudio device enumeration | Medium | None | macOS audio device handling. |
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#253** | Enhance speech tokenizer with 48kHz version | Medium | Qwen tokenizer upgrade |
|
||||
| **#97** | fix: pass language parameter to TTS models | Medium | May be partially obsoleted by multi-engine work — needs review |
|
||||
| **#99** | feat: chunked TTS with quality selector | Medium | Solves 500-char limit. Addresses #191, #203, #69, #111. |
|
||||
| **#154** | feat: Audiobook tab | Medium | Full audiobook workflow. Depends on #99 concepts. |
|
||||
| **#91** | fix: CoreAudio device enumeration | Medium | macOS audio device handling |
|
||||
|
||||
### Architectural PRs (Need Careful Review)
|
||||
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#33** | CUDA GPU Support — External Provider Binaries | **Very High** | The big one. Splits monolithic backend into main app + downloadable provider executables (PyTorch CPU, CUDA). New provider management system, CI/CD for R2 uploads, provider settings UI. Created Feb 1, significant codebase. **This is the foundation for multi-model support** but is currently Qwen-only. |
|
||||
| **#225** | feat: custom HuggingFace model support | High | Adds `custom_models.py`, `custom:<slug>` model IDs, frontend model grouping (Built-in vs Custom). **Takes a different approach than #33** — keeps single backend but allows arbitrary HF repos. These two PRs may conflict architecturally. |
|
||||
| **#194** | feat: Hebrew + Chatterbox TTS | High | **First non-Qwen TTS model.** Adds `ChatterboxTTSBackend` alongside existing backends. Routes by language (`he` → Chatterbox, else → Qwen). Adds Hebrew Whisper models. Includes a lot of cleanup. Important precedent for multi-model. |
|
||||
| **#195** | feat: per-profile LoRA fine-tuning | **Very High** | Depends on #194. Training pipeline, adapter management, SSE progress, 15 new API endpoints. New DB tables. Forces PyTorch even on MLX systems for adapter inference. |
|
||||
| **#161** | feat: Docker + web deployment | High | 3-stage Dockerfile, SPA serving from FastAPI, docker-compose. Implements the Docker deployment plan. |
|
||||
| **#124** | Add Dockerfiles + docker-compose + docs | Medium | Earlier, simpler Docker attempt. Overlaps with #161. |
|
||||
| **#123** | added docker | Low | Minimal Docker PR. Overlaps with #161 and #124. |
|
||||
| **#227** | fix: harden input validation & file safety | Medium | Follow-up to #225. Atomic writes, threading locks, input validation. Good hardening but coupled to the custom models feature. |
|
||||
| **#225** | feat: custom HuggingFace model support | High | Arbitrary HF repo loading. May need rework given multi-engine arch is now shipped. |
|
||||
| **#194** | feat: Hebrew + Chatterbox TTS | High | **Superseded** by PR #257 which shipped Chatterbox multilingual (23 langs incl. Hebrew). May be closeable. |
|
||||
| **#195** | feat: per-profile LoRA fine-tuning | Very High | Training pipeline, adapter management, 15 new endpoints. Depends on #194 (now superseded). |
|
||||
| **#161** | feat: Docker + web deployment | High | 3-stage Dockerfile, SPA serving. Independent of TTS engine work. |
|
||||
| **#124** / **#123** | Docker (simpler attempts) | Low-Medium | Overlap with #161 |
|
||||
| **#227** | fix: harden input validation & file safety | Medium | Coupled to #225 (custom models) |
|
||||
|
||||
### PRs That Need Author Action / Are Stale
|
||||
|
||||
| PR | Title | Notes |
|
||||
|----|-------|-------|
|
||||
| **#237** | fix: bundle qwen_tts source files in PyInstaller | Solves #212 but needs review for build system impact |
|
||||
| **#237** | fix: bundle qwen_tts source files in PyInstaller | Build system, needs review |
|
||||
| **#215** | Update prerequisites with Tauri deps | Branch is `main` — will have conflicts |
|
||||
| **#89** | Linux Support | Branch is `main` — will have conflicts. Broad scope. |
|
||||
| **#83** | Update download links for v0.1.12 | Outdated (we're on v0.1.13) |
|
||||
|
||||
### PRs Likely Superseded
|
||||
|
||||
| PR | Superseded By | Notes |
|
||||
|----|--------------|-------|
|
||||
| **#194** (Hebrew + Chatterbox) | PR #257 (merged) | #257 ships Chatterbox multilingual with 23 languages including Hebrew. #194 took a different approach (route by language). Can likely be closed. |
|
||||
| **#33** (External provider binaries) | PR #252 (merged) | #252 shipped CUDA backend swap. #33's broader provider architecture may still have value but needs reassessment. |
|
||||
|
||||
---
|
||||
|
||||
## Open Issues — Categorized
|
||||
@@ -186,15 +239,15 @@ The single most reported category. Users on Windows with NVIDIA GPUs frequently
|
||||
|
||||
**Key issues:** #239, #222, #220, #217, #208, #198, #192, #167, #164, #141, #130, #127
|
||||
|
||||
**Fix path:** PR #33 (external provider binaries) is designed to solve this. Ship a small main app, let users download the CUDA provider separately.
|
||||
**Fix path:** PR #252 (CUDA backend swap) is now merged. Users can download the CUDA binary separately from the GPU acceleration settings. Many of these issues may now be resolvable — needs triage to confirm.
|
||||
|
||||
### Model Downloads (20 issues)
|
||||
|
||||
Second most reported. Users get stuck downloads, can't resume, no cancel button, no offline fallback.
|
||||
Second most reported. Users get stuck downloads, can't resume, no offline fallback.
|
||||
|
||||
**Key issues:** #249, #240, #221, #216, #212, #181, #180, #159, #150, #149, #145, #143, #135, #134
|
||||
|
||||
**Fix path:** PR #238 (cancel/clear UI), PR #152 (offline crash fix). Resume support not yet addressed.
|
||||
**Fix path:** PR #238 (cancel/clear UI) is now merged. PR #152 (offline crash fix) still open. Inline progress bars now show for all engines. Resume support not yet addressed.
|
||||
|
||||
### Language Requests (18 issues)
|
||||
|
||||
@@ -202,7 +255,7 @@ Strong demand for: Hindi (#245), Indonesian (#247), Dutch (#236), Hebrew (#199),
|
||||
|
||||
**Key issues:** #247, #245, #236, #211, #205, #199, #189, #188, #187, #183, #179, #162
|
||||
|
||||
**Fix path:** PR #97 (pass language param — currently silently dropped!) is the prerequisite. Qwen3-TTS already supports many languages; the bug is that the language code isn't forwarded. Multi-model (#194 Chatterbox for Hebrew) expands coverage further.
|
||||
**Fix path:** Chatterbox Multilingual (merged via #257) now supports 23 languages including many of the requested ones: Arabic, Danish, German, Greek, Finnish, Hebrew, Hindi, Dutch, Norwegian, Polish, Swedish, Swahili, Turkish. Per-engine language filtering (PR #258) ensures the UI shows correct options. Several of these issues may be closeable.
|
||||
|
||||
### New Model Requests (5 explicit issues)
|
||||
|
||||
@@ -214,7 +267,7 @@ Strong demand for: Hindi (#245), Indonesian (#247), Dutch (#236), Hebrew (#199),
|
||||
| #132 | LavaSR (transcription) |
|
||||
| #76 | (General model expansion) |
|
||||
|
||||
Community is also vocally requesting: LuxTTS, Chatterbox, XTTS-v2, Fish Speech, CosyVoice, Kokoro on social media and in issue comments.
|
||||
Community also requests: XTTS-v2, Fish Speech, CosyVoice, Kokoro. The multi-engine architecture is now in place, making new model integration significantly easier.
|
||||
|
||||
### Long-Form / Chunking (5 issues)
|
||||
|
||||
@@ -255,17 +308,14 @@ Notable requests:
|
||||
|
||||
| Document | Target Version | Status | Relevance |
|
||||
|----------|---------------|--------|-----------|
|
||||
| `TTS_PROVIDER_ARCHITECTURE.md` | v0.1.13 | **Partially implemented** in PR #33 | Core architecture for multi-model + CUDA distribution |
|
||||
| `EXTERNAL_PROVIDERS.md` | v0.2.0 | **Not started** | Remote server support. API path inconsistency with provider arch doc (`/v1/` vs `/tts/`) |
|
||||
| `MLX_AUDIO.md` | — | **Shipped** (the only one) | MLX backend is live. 0.6B MLX model still missing. |
|
||||
| `DOCKER_DEPLOYMENT.md` | v0.2.0 | **PR exists** (#161) | Waiting on review. No official images published. |
|
||||
| `OPENAI_SUPPORT.md` | v0.2.0 | **Not started** | OpenAI-compatible API layer. Linked to issue #10. Low complexity. |
|
||||
|
||||
### Cross-Document Conflicts
|
||||
|
||||
1. **API path inconsistency:** Provider arch uses `/tts/generate`, External providers uses `/v1/generate`, OpenAI compat uses `/v1/audio/speech`. Need to reconcile.
|
||||
2. **Docker vs. Provider split:** Docker doc assumes monolithic backend. Provider arch splits into separate binaries. Need to decide: does Docker run the monolith or individual providers?
|
||||
3. **Version targeting:** Provider arch targets v0.1.13 (current!) but isn't merged. Everything else targets v0.2.0.
|
||||
| `TTS_PROVIDER_ARCHITECTURE.md` | v0.1.13 | **Partially superseded** by multi-engine arch + CUDA swap | Core concepts implemented differently than planned |
|
||||
| `CUDA_BACKEND_SWAP.md` | — | **Shipped** (PR #252) | CUDA binary download + backend restart |
|
||||
| `CUDA_BACKEND_SWAP_FINAL.md` | — | **Shipped** (PR #252) | Final implementation plan |
|
||||
| `EXTERNAL_PROVIDERS.md` | v0.2.0 | **Not started** | Remote server support |
|
||||
| `MLX_AUDIO.md` | — | **Shipped** | MLX backend is live |
|
||||
| `DOCKER_DEPLOYMENT.md` | v0.2.0 | **PR exists** (#161) | Waiting on review |
|
||||
| `OPENAI_SUPPORT.md` | v0.2.0 | **Not started** | OpenAI-compatible API layer |
|
||||
| `PR33_CUDA_PROVIDER_REVIEW.md` | — | **Reference** | Analysis of the original provider approach |
|
||||
|
||||
---
|
||||
|
||||
@@ -273,135 +323,95 @@ Notable requests:
|
||||
|
||||
### Models Worth Supporting (2026 SOTA)
|
||||
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Repo |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|------|
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English-first | <1 GB | Easy | `ysharma3501/LuxTTS` |
|
||||
| **Chatterbox** | 5s zero-shot | Sub-200ms streaming | 24-48 kHz | 23+ | Low | Medium | `resemble-ai/chatterbox` |
|
||||
| **XTTS-v2** | 6s zero-shot | Fast mid-GPU | 24 kHz | 17+ | Medium | Medium | `coqui/XTTS-v2` |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Medium | `fishaudio/fish-speech` |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Easy | Alibaba HF org |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny | Medium | Kokoro repo |
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Status |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|--------|
|
||||
| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | **Shipped** | v0.1.13 |
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | **Shipped** | PR #254 |
|
||||
| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | **Shipped** | PR #257 |
|
||||
| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | **PR #258** | In review |
|
||||
| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Ready | Multi-engine arch in place |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny | Ready | Multi-engine arch in place |
|
||||
|
||||
### What's Needed Architecturally for Multi-Model
|
||||
### Adding a New Engine (Now Straightforward)
|
||||
|
||||
The current codebase assumes one TTS model family (Qwen3-TTS). Adding any new model requires:
|
||||
With the multi-engine architecture shipped, adding a new TTS engine requires:
|
||||
|
||||
1. **Model type concept** — A `model_type` field (e.g. `qwen`, `luxtts`, `chatterbox`) alongside `model_size`. The `GenerationRequest` schema, frontend form, and all model config dicts need updating.
|
||||
1. **Create `backend/backends/<engine>_backend.py`** — implement `TTSBackend` protocol (~200-300 lines)
|
||||
2. **Register in `backend/backends/__init__.py`** — add to `TTS_ENGINES` dict + factory function
|
||||
3. **Update `backend/models.py`** — add engine name to regex
|
||||
4. **Update `backend/main.py`** — add engine cases in generate, stream, model-status, download, delete (5 dispatch points)
|
||||
5. **Update frontend** — add to engine union type, form schema, model dropdown, language map (5-6 files)
|
||||
|
||||
2. **Multiple backend instances** — The singleton `get_tts_backend()` needs to become a registry. Different models have different voice prompt formats, different inference APIs, different sample rates.
|
||||
|
||||
3. **Voice prompt format abstraction** — Qwen uses `torch.save()`-serialized tensors. LuxTTS uses `encode_prompt()` returning its own format. Chatterbox uses audio-path-based cloning. The cache system (`backend/utils/cache.py`) needs to handle heterogeneous formats.
|
||||
|
||||
4. **Sample rate normalization** — Qwen outputs 24 kHz. LuxTTS outputs 48 kHz. The Stories editor and audio pipeline need to handle mixed rates.
|
||||
|
||||
5. **Per-model capabilities** — Not all models support `instruct` (delivery instructions), not all support streaming, not all support the same languages. The UI needs to adapt.
|
||||
|
||||
### PR #194 as Precedent
|
||||
|
||||
The Hebrew/Chatterbox PR (#194) is the first attempt at multi-model. It takes a pragmatic approach: route by language (`he` → Chatterbox, else → Qwen). This works for one extra model but doesn't scale — what happens when you want Chatterbox for English too?
|
||||
|
||||
### PR #225 as Alternative Approach
|
||||
|
||||
The custom HuggingFace models PR (#225) takes a different angle: let users register arbitrary HF repos and attempt to load them through the existing Qwen backend. This is flexible but fragile — it assumes all models have the same API as Qwen3-TTS.
|
||||
|
||||
### PR #33 as Foundation
|
||||
|
||||
The external provider binaries PR (#33) has the most robust architecture for multi-model, since each provider is a separate process with its own dependencies. But it's complex, currently Qwen-only, and has been stale since early February.
|
||||
Total effort: **~1 day** for a well-documented model with a PyPI package.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Bottlenecks
|
||||
|
||||
### 1. Single Backend Singleton
|
||||
### ~~1. Single Backend Singleton~~ — RESOLVED
|
||||
|
||||
**File:** `backend/backends/__init__.py:118-137`
|
||||
The singleton TTS backend was replaced with a thread-safe per-engine registry in PR #254. Multiple engines can now be loaded simultaneously.
|
||||
|
||||
The entire TTS system runs through one global `_tts_backend` instance. You literally cannot have two models loaded. This is the #1 blocker for multi-model support.
|
||||
### 2. `main.py` is 2100+ Lines
|
||||
|
||||
### 2. `main.py` is 1700+ Lines
|
||||
All API routes, all model configs, all business logic in one file. Five separate dispatch points for each engine. Any new engine touches this file in 5 places. A model config registry pattern would reduce duplication.
|
||||
|
||||
All API routes, all model configs, all business logic in one file. Three separate hardcoded model config dicts that must stay in sync. Any multi-model change touches this file heavily.
|
||||
### 3. Model Config is Scattered (Improved)
|
||||
|
||||
### 3. Model Config is Scattered
|
||||
|
||||
Model identifiers, HF repo IDs, display names, and download logic are duplicated across:
|
||||
- `main.py` (3 separate dicts)
|
||||
- `pytorch_backend.py` (HF repo map)
|
||||
- `mlx_backend.py` (MLX repo map)
|
||||
- `GenerationForm.tsx` (UI labels)
|
||||
- `useGenerationForm.ts` (validation schema)
|
||||
- `ModelManagement.tsx` (prefix filters)
|
||||
|
||||
There is no single source of truth for "what models does Voicebox support."
|
||||
Model identifiers are still duplicated across `main.py` (3 dicts), backend files, frontend components, and the languages constant. However, the pattern is now consistent and well-understood. A centralized model registry would help but isn't blocking.
|
||||
|
||||
### 4. Voice Prompt Cache Assumes PyTorch Tensors
|
||||
|
||||
`backend/utils/cache.py` uses `torch.save()` / `torch.load()` for caching voice prompts. Models that don't use PyTorch tensors (LuxTTS, MLX-native models) can't use this cache.
|
||||
`backend/utils/cache.py` uses `torch.save()` / `torch.load()`. LuxTTS and Chatterbox backends work around this by storing reference audio paths instead of tensors in their voice prompt dicts. Not ideal but functional.
|
||||
|
||||
### 5. Frontend Assumes Qwen Model Sizes
|
||||
### 5. ~~Frontend Assumes Qwen Model Sizes~~ — RESOLVED
|
||||
|
||||
The generation form schema (`useGenerationForm.ts:17`) validates `model_size` as `'1.7B' | '0.6B'`. The model management UI filters by string prefix `qwen-tts`. Adding any model requires touching 3-4 frontend files.
|
||||
The generation form now uses a flat model dropdown with engine-based routing. Per-engine language filtering is in place. Model size is only sent for Qwen.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Priorities
|
||||
|
||||
### Tier 1 — Ship Now (Bug Fixes & Critical Improvements)
|
||||
### Tier 1 — Ship Now (Low Risk)
|
||||
|
||||
These PRs fix real user pain with low risk. Can be reviewed and merged quickly.
|
||||
| Priority | PR/Item | Impact | Effort |
|
||||
|----------|---------|--------|--------|
|
||||
| 1 | **#258** — Chatterbox Turbo + per-engine languages | Paralinguistic tags, proper language filtering | Review only |
|
||||
| 2 | **#152** — Offline mode crash fix | Fixes #150, #151 | Low |
|
||||
| 3 | **#99** — Chunked TTS + quality selector | Removes 500-char limit, addresses 5 issues | Medium |
|
||||
| 4 | **#218** — Windows HF cache dir fix | Windows-specific pain | Low |
|
||||
| 5 | **#178** — Generation error handling | Error UX | Low |
|
||||
| 6 | **#230** — Docs fixes | Zero risk | None |
|
||||
| 7 | **#133** — Network access toggle | Wires up existing code | Low |
|
||||
| 8 | **#88** — CORS restriction | Security improvement | Low |
|
||||
| 9 | **#214** — Tauri window close panic fix | Stability | Low |
|
||||
| 10 | Triage GPU issues | Many may be resolved by CUDA swap (#252) | Low |
|
||||
| 11 | Close superseded PRs | #194 (superseded by #257), #83 (outdated) | None |
|
||||
|
||||
| Priority | PR | Impact | Effort |
|
||||
|----------|-----|--------|--------|
|
||||
| 1 | **#97** — Pass language param to TTS | Fixes all non-English generation (18 language issues) | Low |
|
||||
| 2 | **#238** — Download cancel/clear UI | Addresses 20 download-related issues | Low |
|
||||
| 3 | **#152** — Offline mode crash fix | Fixes #150, #151 | Low |
|
||||
| 4 | **#99** — Chunked TTS + quality selector | Removes 500-char limit, addresses 5 issues | Medium |
|
||||
| 5 | **#218** — Windows HF cache dir fix | Windows-specific pain | Low |
|
||||
| 6 | **#175, #178** — Profile validation + error handling | Small fixes | Low |
|
||||
| 7 | **#250, #230** — Docs fixes | Zero risk | None |
|
||||
| 8 | **#133** — Network access toggle | Wires up existing code | Low |
|
||||
| 9 | **#88** — CORS restriction | Security improvement | Low |
|
||||
| 10 | **#214** — Tauri window close panic fix | Stability | Low |
|
||||
### Tier 2 — Next Release (v0.2.0)
|
||||
|
||||
### Tier 2 — Next Release (v0.2.0 Foundations)
|
||||
|
||||
These require more review but unlock major capabilities.
|
||||
|
||||
| Priority | Item | Impact | Effort | Dependencies |
|
||||
|----------|------|--------|--------|-------------|
|
||||
| 1 | **PR #33** — External provider binaries | Solves GPU distribution (19 issues), foundation for multi-model | Very High | Needs rebase, thorough review |
|
||||
| 2 | **Multi-model abstraction layer** | Required before adding LuxTTS/Chatterbox/etc. | High | Informed by #33, #194, #225 |
|
||||
| 3 | **PR #161** — Docker deployment | Server/headless users | Medium | Independent of #33 |
|
||||
| 4 | **PR #194** — Hebrew + Chatterbox | First non-Qwen model, language expansion | High | Should align with multi-model abstraction |
|
||||
| 5 | **PR #154** — Audiobook tab | Significant feature for long-form users | Medium | Benefits from #99 (chunking) |
|
||||
| Priority | Item | Impact | Effort |
|
||||
|----------|------|--------|--------|
|
||||
| 1 | **#253** — 48kHz speech tokenizer | Quality improvement | Medium |
|
||||
| 2 | **#161** — Docker deployment | Server/headless users | Medium |
|
||||
| 3 | **#154** — Audiobook tab | Long-form users | Medium |
|
||||
| 4 | **Model config registry** | Reduce 5-dispatch-point duplication in main.py | Medium |
|
||||
| 5 | **#225** — Custom HuggingFace models | User-supplied models | High (needs rework for multi-engine) |
|
||||
|
||||
### Tier 3 — Future (v0.3.0+)
|
||||
|
||||
| Item | Notes |
|
||||
|------|-------|
|
||||
| LuxTTS integration | 48 kHz, low VRAM, but needs multi-model arch first |
|
||||
| XTTS-v2 / Fish Speech | Multilingual powerhouses |
|
||||
| XTTS-v2 / Fish Speech / CosyVoice | Multi-engine arch is ready; just needs backend implementation |
|
||||
| OpenAI-compatible API (plan doc exists) | Low effort once API is stable |
|
||||
| LoRA fine-tuning (PR #195) | Complex, depends on #194 |
|
||||
| External/remote providers (plan doc exists) | Depends on provider architecture |
|
||||
| LoRA fine-tuning (PR #195) | Complex, needs rework for multi-engine |
|
||||
| External/remote providers | Depends on use case demand |
|
||||
| GGUF support (#226) | Depends on model ecosystem maturity |
|
||||
| Queue system (#234) | Batch generation |
|
||||
| Real-time streaming synthesis | MLX-only currently, needs PyTorch path |
|
||||
|
||||
### Decision Point: Multi-Model Architecture
|
||||
|
||||
Before adding any new TTS model, a decision is needed on *how*:
|
||||
|
||||
**Option A — Provider Binary Split (PR #33 approach)**
|
||||
Each model family is a separate executable/process. Most isolated, most flexible, but most complex. Solves the CUDA distribution problem simultaneously.
|
||||
|
||||
**Option B — In-Process Model Registry**
|
||||
Keep everything in one process but replace the singleton with a registry that can instantiate multiple `TTSBackend` implementations. Simpler, but doesn't solve binary size / CUDA distribution.
|
||||
|
||||
**Option C — Hybrid (Recommended)**
|
||||
Use Option B for lightweight models (LuxTTS, Kokoro — small, CPU-friendly) that can coexist in-process. Use Option A for heavy models (CUDA Qwen3-TTS, Fish Speech) that need their own process/dependencies. The provider architecture from PR #33 becomes the escape hatch for heavy models, while light models are built-in.
|
||||
|
||||
This matches how PR #194 already works (Chatterbox loaded in-process alongside Qwen) while keeping the door open for PR #33's provider split.
|
||||
| Streaming for non-MLX engines | Currently MLX-only |
|
||||
| Kokoro-82M | Tiny model, great for CPU-only machines |
|
||||
|
||||
---
|
||||
|
||||
@@ -409,24 +419,20 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
|
||||
| Branch | PR | Status | Notes |
|
||||
|--------|-----|--------|-------|
|
||||
| `external-provider-binaries` | #33 | Open, stale | Major architecture work |
|
||||
| `feat/dual-server-binaries` | — | No PR | Related to provider split? |
|
||||
| `feat/chatterbox-turbo` | #258 | Open | Chatterbox Turbo + per-engine languages |
|
||||
| `feat/chatterbox` | #257 | **Merged** | Chatterbox Multilingual |
|
||||
| `feat/luxtts` | #254 | **Merged** | LuxTTS + multi-engine arch |
|
||||
| `external-provider-binaries` | #33 | Superseded by #252 | Original CUDA provider approach |
|
||||
| `feat/dual-server-binaries` | — | No PR | Related to provider split |
|
||||
| `fix-multi-sample` | — | No PR | Voice profile multi-sample fix |
|
||||
| `fix-dl-notification-...` | — | No PR | Model download UX |
|
||||
| `improvements` | — | No PR | Unknown scope |
|
||||
| `stories` | — | No PR | Stories editor work? |
|
||||
| `windows-server-shutdown` | — | No PR | Windows lifecycle |
|
||||
| `model-dl-fix` | — | No PR | Model download fix |
|
||||
| `channels` | — | No PR | Audio channels |
|
||||
| `audio-export-entitlement-fix` | — | No PR | macOS entitlements |
|
||||
| `better-docs` | — | No PR | Documentation |
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference: API Endpoints
|
||||
|
||||
<details>
|
||||
<summary>All current endpoints (v0.1.13)</summary>
|
||||
<summary>All current endpoints</summary>
|
||||
|
||||
| Endpoint | Method | Purpose |
|
||||
|----------|--------|---------|
|
||||
@@ -437,20 +443,21 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
| `/profiles/{id}/avatar` | POST, GET, DELETE | Avatar management |
|
||||
| `/profiles/{id}/export` | GET | Export profile as ZIP |
|
||||
| `/profiles/import` | POST | Import profile from ZIP |
|
||||
| `/generate` | POST | Generate speech |
|
||||
| `/generate/stream` | POST | Stream speech (SSE) |
|
||||
| `/generate` | POST | Generate speech (engine param selects TTS backend) |
|
||||
| `/generate/stream` | POST | Stream speech (MLX only) |
|
||||
| `/history` | GET | List generation history |
|
||||
| `/history/{id}` | GET, DELETE | Get/delete generation |
|
||||
| `/history/{id}/export` | GET | Export generation ZIP |
|
||||
| `/history/{id}/export-audio` | GET | Export audio only |
|
||||
| `/transcribe` | POST | Transcribe audio (Whisper) |
|
||||
| `/models/status` | GET | All model statuses |
|
||||
| `/models/status` | GET | All model statuses (Qwen, LuxTTS, Chatterbox, Chatterbox Turbo, Whisper) |
|
||||
| `/models/download` | POST | Trigger model download |
|
||||
| `/models/download/cancel` | POST | Cancel/dismiss download |
|
||||
| `/models/{name}` | DELETE | Delete downloaded model |
|
||||
| `/models/load` | POST | Load model into memory |
|
||||
| `/models/unload` | POST | Unload model |
|
||||
| `/models/progress/{name}` | GET | SSE download progress |
|
||||
| `/tasks/active` | GET | Active downloads/generations |
|
||||
| `/tasks/active` | GET | Active downloads/generations (with inline progress) |
|
||||
| `/stories` | POST, GET | Create/list stories |
|
||||
| `/stories/{id}` | GET, PUT, DELETE | Story CRUD |
|
||||
| `/stories/{id}/items` | POST, GET | Story items CRUD |
|
||||
@@ -458,5 +465,8 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
| `/channels` | POST, GET | Audio channel CRUD |
|
||||
| `/channels/{id}` | PUT, DELETE | Channel update/delete |
|
||||
| `/cache/clear` | POST | Clear voice prompt cache |
|
||||
| `/server/cuda/status` | GET | CUDA binary availability |
|
||||
| `/server/cuda/download` | POST | Download CUDA binary |
|
||||
| `/server/cuda/switch` | POST | Switch to CUDA backend |
|
||||
|
||||
</details>
|
||||
|
||||
@@ -38,6 +38,8 @@ setup-python:
|
||||
echo "Installing Python dependencies..."
|
||||
{{ pip }} install --upgrade pip -q
|
||||
{{ pip }} install -r {{ backend_dir }}/requirements.txt
|
||||
# Chatterbox pins numpy<1.26 / torch==2.6 which break on Python 3.12+
|
||||
{{ pip }} install --no-deps chatterbox-tts
|
||||
# Apple Silicon: install MLX backend
|
||||
if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
|
||||
echo "Detected Apple Silicon — installing MLX dependencies..."
|
||||
|
||||
@@ -6,7 +6,7 @@ set -e
|
||||
echo "Generating OpenAPI client..."
|
||||
|
||||
# Check if backend is running
|
||||
if ! curl -s http://localhost:8000/openapi.json > /dev/null 2>&1; then
|
||||
if ! curl -s http://localhost:17493/openapi.json > /dev/null 2>&1; then
|
||||
echo "Backend not running. Starting backend..."
|
||||
cd backend
|
||||
|
||||
@@ -26,19 +26,19 @@ if ! curl -s http://localhost:8000/openapi.json > /dev/null 2>&1; then
|
||||
|
||||
# Start backend in background
|
||||
echo "Starting backend server..."
|
||||
uvicorn main:app --port 8000 &
|
||||
uvicorn main:app --port 17493 & # Keep the generator on the app's documented local backend port.
|
||||
BACKEND_PID=$!
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8000/openapi.json > /dev/null 2>&1; then
|
||||
for _ in {1..30}; do
|
||||
if curl -s http://localhost:17493/openapi.json > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if ! curl -s http://localhost:8000/openapi.json > /dev/null 2>&1; then
|
||||
if ! curl -s http://localhost:17493/openapi.json > /dev/null 2>&1; then
|
||||
echo "Error: Backend failed to start"
|
||||
kill $BACKEND_PID 2>/dev/null || true
|
||||
exit 1
|
||||
@@ -52,7 +52,7 @@ fi
|
||||
|
||||
# Download OpenAPI schema
|
||||
echo "Downloading OpenAPI schema..."
|
||||
curl -s http://localhost:8000/openapi.json > app/openapi.json
|
||||
curl -s http://localhost:17493/openapi.json > app/openapi.json
|
||||
|
||||
# Check if openapi-typescript-codegen is installed
|
||||
if ! bunx --bun openapi-typescript-codegen --version > /dev/null 2>&1; then
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to observe exactly how HuggingFace reports download progress
|
||||
for each TTS model. Doesn't load models — just downloads and tracks tqdm.
|
||||
|
||||
Usage:
|
||||
backend/venv/bin/python scripts/test_download_progress.py qwen
|
||||
backend/venv/bin/python scripts/test_download_progress.py luxtts
|
||||
backend/venv/bin/python scripts/test_download_progress.py chatterbox
|
||||
|
||||
Add --delete to clear cache first and force a real download:
|
||||
backend/venv/bin/python scripts/test_download_progress.py chatterbox --delete
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager
|
||||
|
||||
# ─── Configuration ────────────────────────────────────────────────────────────
|
||||
|
||||
MODELS = {
|
||||
"qwen": {
|
||||
"repo_id": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
|
||||
"method": "from_pretrained",
|
||||
"description": "Qwen TTS 1.7B (uses transformers from_pretrained)",
|
||||
},
|
||||
"luxtts": {
|
||||
"repo_id": "YatharthS/LuxTTS",
|
||||
"method": "snapshot_download",
|
||||
"description": "LuxTTS (uses snapshot_download)",
|
||||
},
|
||||
"chatterbox": {
|
||||
"repo_id": "ResembleAI/chatterbox",
|
||||
"method": "snapshot_download",
|
||||
"allow_patterns": [
|
||||
"ve.pt",
|
||||
"t3_mtl23ls_v2.safetensors",
|
||||
"s3gen.pt",
|
||||
"grapheme_mtl_merged_expanded_v1.json",
|
||||
"conds.pt",
|
||||
"Cangjie5_TC.json",
|
||||
],
|
||||
"description": "Chatterbox Multilingual (uses snapshot_download with allow_patterns)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ─── Progress tracking (mirrors our HFProgressTracker) ────────────────────────
|
||||
|
||||
class ProgressSpy:
|
||||
"""Intercepts tqdm to see exactly what HF reports."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self.events = [] # List of dicts: {time, type, ...}
|
||||
self._original_tqdm_class = None
|
||||
self._original_tqdm_auto = None
|
||||
self._patched_modules = {}
|
||||
self._hf_tqdm_original_update = None
|
||||
self._start_time = None
|
||||
|
||||
def _elapsed(self):
|
||||
return time.time() - self._start_time if self._start_time else 0
|
||||
|
||||
def _log(self, event_type, **kwargs):
|
||||
entry = {"time": f"{self._elapsed():.1f}s", "type": event_type, **kwargs}
|
||||
self.events.append(entry)
|
||||
|
||||
# Live print
|
||||
parts = [f"[{entry['time']:>7s}] {event_type:>10s}"]
|
||||
for k, v in kwargs.items():
|
||||
if k in ("current", "total") and isinstance(v, (int, float)) and v > 1_000_000:
|
||||
parts.append(f"{k}={v / 1_000_000:.1f}MB")
|
||||
else:
|
||||
parts.append(f"{k}={v}")
|
||||
print(" ".join(parts), flush=True)
|
||||
|
||||
def _create_tracked_tqdm_class(self):
|
||||
spy = self
|
||||
original_tqdm = self._original_tqdm_class
|
||||
|
||||
class SpyTqdm(original_tqdm):
|
||||
def __init__(self, *args, **kwargs):
|
||||
desc = kwargs.get("desc", "")
|
||||
if not desc and args:
|
||||
first_arg = args[0]
|
||||
if isinstance(first_arg, str):
|
||||
desc = first_arg
|
||||
|
||||
filename = ""
|
||||
if desc:
|
||||
if ":" in desc:
|
||||
filename = desc.split(":")[0].strip()
|
||||
else:
|
||||
filename = desc.strip()
|
||||
|
||||
# Filter out non-standard kwargs
|
||||
tqdm_kwargs = {
|
||||
'iterable', 'desc', 'total', 'leave', 'file', 'ncols',
|
||||
'mininterval', 'maxinterval', 'miniters', 'ascii', 'disable',
|
||||
'unit', 'unit_scale', 'dynamic_ncols', 'smoothing',
|
||||
'bar_format', 'initial', 'position', 'postfix',
|
||||
'unit_divisor', 'write_bytes', 'lock_args', 'nrows',
|
||||
'colour', 'color', 'delay', 'gui', 'disable_default', 'pos',
|
||||
}
|
||||
filtered_kwargs = {k: v for k, v in kwargs.items() if k in tqdm_kwargs}
|
||||
|
||||
try:
|
||||
super().__init__(*args, **filtered_kwargs)
|
||||
except TypeError:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self._spy_filename = filename or "unknown"
|
||||
total = getattr(self, "total", None)
|
||||
|
||||
spy._log(
|
||||
"INIT",
|
||||
filename=self._spy_filename,
|
||||
total=total or 0,
|
||||
unit=kwargs.get("unit", "?"),
|
||||
unit_scale=kwargs.get("unit_scale", False),
|
||||
disable=kwargs.get("disable", False),
|
||||
)
|
||||
|
||||
def update(self, n=1):
|
||||
result = super().update(n)
|
||||
|
||||
current = getattr(self, "n", 0)
|
||||
total = getattr(self, "total", 0)
|
||||
filename = self._spy_filename
|
||||
|
||||
spy._log(
|
||||
"UPDATE",
|
||||
filename=filename,
|
||||
n=n,
|
||||
current=current,
|
||||
total=total or 0,
|
||||
pct=f"{100 * current / total:.1f}%" if total else "?",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def close(self):
|
||||
spy._log("CLOSE", filename=self._spy_filename)
|
||||
return super().close()
|
||||
|
||||
return SpyTqdm
|
||||
|
||||
@contextmanager
|
||||
def patch(self):
|
||||
"""Context manager that patches tqdm globally — same as HFProgressTracker."""
|
||||
self._start_time = time.time()
|
||||
|
||||
try:
|
||||
import tqdm as tqdm_module
|
||||
self._original_tqdm_class = tqdm_module.tqdm
|
||||
except ImportError:
|
||||
yield
|
||||
return
|
||||
|
||||
tracked_tqdm = self._create_tracked_tqdm_class()
|
||||
|
||||
# Patch tqdm.tqdm
|
||||
tqdm_module.tqdm = tracked_tqdm
|
||||
|
||||
# Patch tqdm.auto.tqdm
|
||||
self._original_tqdm_auto = None
|
||||
if hasattr(tqdm_module, "auto") and hasattr(tqdm_module.auto, "tqdm"):
|
||||
self._original_tqdm_auto = tqdm_module.auto.tqdm
|
||||
tqdm_module.auto.tqdm = tracked_tqdm
|
||||
|
||||
# Patch in sys.modules (same as HFProgressTracker)
|
||||
tqdm_attr_names = ['tqdm', 'base_tqdm', 'old_tqdm']
|
||||
patched_count = 0
|
||||
|
||||
for module_name in list(sys.modules.keys()):
|
||||
if "huggingface" in module_name or module_name.startswith("tqdm"):
|
||||
try:
|
||||
module = sys.modules[module_name]
|
||||
for attr_name in tqdm_attr_names:
|
||||
if hasattr(module, attr_name):
|
||||
attr = getattr(module, attr_name)
|
||||
is_tqdm_class = (
|
||||
attr is self._original_tqdm_class
|
||||
or (self._original_tqdm_auto and attr is self._original_tqdm_auto)
|
||||
or (
|
||||
hasattr(attr, "__name__")
|
||||
and attr.__name__ == "tqdm"
|
||||
and hasattr(attr, "update")
|
||||
)
|
||||
)
|
||||
if is_tqdm_class:
|
||||
key = f"{module_name}.{attr_name}"
|
||||
self._patched_modules[key] = (module, attr_name, attr)
|
||||
setattr(module, attr_name, tracked_tqdm)
|
||||
patched_count += 1
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
|
||||
# Monkey-patch HF's tqdm.update (same as HFProgressTracker)
|
||||
try:
|
||||
from huggingface_hub.utils import tqdm as hf_tqdm_module
|
||||
if hasattr(hf_tqdm_module, 'tqdm'):
|
||||
hf_tqdm_class = hf_tqdm_module.tqdm
|
||||
self._hf_tqdm_original_update = hf_tqdm_class.update
|
||||
spy = self
|
||||
|
||||
def patched_update(tqdm_self, n=1):
|
||||
result = spy._hf_tqdm_original_update(tqdm_self, n)
|
||||
desc = getattr(tqdm_self, 'desc', '') or ''
|
||||
current = getattr(tqdm_self, 'n', 0)
|
||||
total = getattr(tqdm_self, 'total', 0) or 0
|
||||
|
||||
spy._log(
|
||||
"HF_UPDATE",
|
||||
desc=desc,
|
||||
current=current,
|
||||
total=total,
|
||||
pct=f"{100 * current / total:.1f}%" if total else "?",
|
||||
)
|
||||
return result
|
||||
|
||||
hf_tqdm_class.update = patched_update
|
||||
patched_count += 1
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
print(f"\n=== Patched {patched_count} tqdm references ===\n", flush=True)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# Restore everything
|
||||
import tqdm as tqdm_module
|
||||
tqdm_module.tqdm = self._original_tqdm_class
|
||||
if self._original_tqdm_auto:
|
||||
tqdm_module.auto.tqdm = self._original_tqdm_auto
|
||||
for key, (module, attr_name, original) in self._patched_modules.items():
|
||||
try:
|
||||
setattr(module, attr_name, original)
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
if self._hf_tqdm_original_update:
|
||||
try:
|
||||
from huggingface_hub.utils import tqdm as hf_tqdm_module
|
||||
if hasattr(hf_tqdm_module, 'tqdm'):
|
||||
hf_tqdm_module.tqdm.update = self._hf_tqdm_original_update
|
||||
except (ImportError, AttributeError):
|
||||
pass
|
||||
|
||||
def summary(self):
|
||||
print("\n" + "=" * 70)
|
||||
print("SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
inits = [e for e in self.events if e["type"] == "INIT"]
|
||||
updates = [e for e in self.events if e["type"] in ("UPDATE", "HF_UPDATE")]
|
||||
|
||||
print(f"\ntqdm bars created: {len(inits)}")
|
||||
for e in inits:
|
||||
print(f" - {e.get('filename', '?'):40s} total={e.get('total', '?')}")
|
||||
|
||||
print(f"\nTotal update calls: {len(updates)}")
|
||||
|
||||
# Group updates by filename
|
||||
by_file = {}
|
||||
for e in updates:
|
||||
fn = e.get("filename") or e.get("desc", "unknown")
|
||||
if fn not in by_file:
|
||||
by_file[fn] = []
|
||||
by_file[fn].append(e)
|
||||
|
||||
for fn, evts in by_file.items():
|
||||
max_current = max(e.get("current", 0) for e in evts)
|
||||
max_total = max(e.get("total", 0) for e in evts)
|
||||
print(f"\n {fn}:")
|
||||
print(f" updates: {len(evts)}")
|
||||
print(f" max current: {max_current:,}")
|
||||
print(f" max total: {max_total:,}")
|
||||
if max_total > 0 and max_current > 0:
|
||||
print(f" final pct: {100 * max_current / max_total:.1f}%")
|
||||
else:
|
||||
print(f" final pct: NO PROGRESS REPORTED")
|
||||
|
||||
|
||||
# ─── Delete cache ─────────────────────────────────────────────────────────────
|
||||
|
||||
def delete_cache(repo_id: str):
|
||||
from huggingface_hub import constants as hf_constants
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
repo_cache = cache_dir / ("models--" + repo_id.replace("/", "--"))
|
||||
if repo_cache.exists():
|
||||
print(f"Deleting cache: {repo_cache}")
|
||||
shutil.rmtree(repo_cache)
|
||||
print("Deleted.")
|
||||
else:
|
||||
print(f"No cache found at {repo_cache}")
|
||||
|
||||
|
||||
# ─── Download functions ───────────────────────────────────────────────────────
|
||||
|
||||
def download_qwen(spy: ProgressSpy):
|
||||
"""Mirrors how pytorch_backend.py downloads Qwen."""
|
||||
from transformers import AutoModel
|
||||
repo_id = MODELS["qwen"]["repo_id"]
|
||||
|
||||
print(f"Downloading {repo_id} via AutoModel.from_pretrained...")
|
||||
with spy.patch():
|
||||
# This is what Qwen3TTSModel.from_pretrained does under the hood
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(repo_id)
|
||||
|
||||
|
||||
def download_luxtts(spy: ProgressSpy):
|
||||
"""Mirrors how luxtts_backend.py downloads LuxTTS."""
|
||||
from huggingface_hub import snapshot_download
|
||||
repo_id = MODELS["luxtts"]["repo_id"]
|
||||
|
||||
print(f"Downloading {repo_id} via snapshot_download...")
|
||||
with spy.patch():
|
||||
snapshot_download(repo_id)
|
||||
|
||||
|
||||
def download_chatterbox(spy: ProgressSpy):
|
||||
"""Mirrors how chatterbox_backend.py downloads Chatterbox."""
|
||||
from huggingface_hub import snapshot_download
|
||||
cfg = MODELS["chatterbox"]
|
||||
|
||||
print(f"Downloading {cfg['repo_id']} via snapshot_download with allow_patterns...")
|
||||
with spy.patch():
|
||||
snapshot_download(
|
||||
repo_id=cfg["repo_id"],
|
||||
repo_type="model",
|
||||
revision="main",
|
||||
allow_patterns=cfg["allow_patterns"],
|
||||
token=os.getenv("HF_TOKEN"),
|
||||
)
|
||||
|
||||
|
||||
# ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in MODELS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(MODELS.keys())}> [--delete]")
|
||||
sys.exit(1)
|
||||
|
||||
model_key = sys.argv[1]
|
||||
should_delete = "--delete" in sys.argv
|
||||
cfg = MODELS[model_key]
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"Testing download progress for: {cfg['description']}")
|
||||
print(f"Repo: {cfg['repo_id']}")
|
||||
print(f"Method: {cfg['method']}")
|
||||
print(f"{'=' * 70}\n")
|
||||
|
||||
if should_delete:
|
||||
delete_cache(cfg["repo_id"])
|
||||
print()
|
||||
|
||||
spy = ProgressSpy()
|
||||
|
||||
dispatch = {
|
||||
"qwen": download_qwen,
|
||||
"luxtts": download_luxtts,
|
||||
"chatterbox": download_chatterbox,
|
||||
}
|
||||
|
||||
try:
|
||||
dispatch[model_key](spy)
|
||||
except Exception as e:
|
||||
print(f"\n!!! Download failed: {e}")
|
||||
|
||||
spy.summary()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Generated
+2
-1
@@ -5041,7 +5041,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "voicebox"
|
||||
version = "0.1.12"
|
||||
version = "0.1.13"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"core-foundation-sys",
|
||||
@@ -5064,6 +5064,7 @@ dependencies = [
|
||||
"tauri-plugin-updater",
|
||||
"tokio",
|
||||
"wasapi",
|
||||
"webkit2gtk",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
|
||||
@@ -37,6 +37,9 @@ core-foundation-sys = "0.8"
|
||||
wasapi = "0.22"
|
||||
windows = { version = "0.62", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging", "Win32_System_Com"] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
webkit2gtk = "2.0"
|
||||
|
||||
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
|
||||
tauri-plugin-updater = "2.0"
|
||||
tauri-plugin-process = "2.0"
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -714,6 +714,43 @@ pub fn run() {
|
||||
}
|
||||
}
|
||||
|
||||
// Enable microphone access on Linux (WebKitGTK denies getUserMedia by default)
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use tauri::Manager;
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
let _ = window.with_webview(|webview| {
|
||||
use webkit2gtk::{WebViewExt, SettingsExt, PermissionRequestExt};
|
||||
use webkit2gtk::glib::ObjectExt;
|
||||
let wk_webview = webview.inner();
|
||||
|
||||
// Enable media stream support in WebKitGTK settings
|
||||
if let Some(settings) = WebViewExt::settings(&wk_webview) {
|
||||
settings.set_enable_media_stream(true);
|
||||
}
|
||||
|
||||
// Auto-grant UserMediaPermissionRequest (microphone access)
|
||||
// Only for trusted local origins (Tauri dev server or custom protocol)
|
||||
wk_webview.connect_permission_request(move |webview, request: &webkit2gtk::PermissionRequest| {
|
||||
if request.is::<webkit2gtk::UserMediaPermissionRequest>() {
|
||||
let uri = WebViewExt::uri(webview).unwrap_or_default();
|
||||
let is_trusted = uri.starts_with("tauri://")
|
||||
|| uri.starts_with("https://tauri.localhost")
|
||||
|| uri.starts_with("http://localhost")
|
||||
|| uri.starts_with("http://127.0.0.1");
|
||||
if is_trusted {
|
||||
request.allow();
|
||||
return true;
|
||||
}
|
||||
request.deny();
|
||||
return true;
|
||||
}
|
||||
false
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
|
||||
Reference in New Issue
Block a user