mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-27 14:15:16 -07:00
Compare commits
52
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49ebf6222e | ||
|
|
509b0e71cc | ||
|
|
81f8be1a94 | ||
|
|
655a60ca81 | ||
|
|
52285362ce | ||
|
|
3ea587797f | ||
|
|
325714bb83 | ||
|
|
9aa7080c51 | ||
|
|
97292ecef7 | ||
|
|
837f8525d8 | ||
|
|
70ca7f66cb | ||
|
|
c12b5d6f0a | ||
|
|
139fa38e3f | ||
|
|
0e9f5db40f | ||
|
|
2f535a772f | ||
|
|
b420637957 | ||
|
|
bfd7b815a5 | ||
|
|
cac80f6af0 | ||
|
|
47ce4cafdf | ||
|
|
bfe912e41a | ||
|
|
5ccf79a8f7 | ||
|
|
1d32170c2e | ||
|
|
ca74c155e2 | ||
|
|
1f770a157d | ||
|
|
d64e24d422 | ||
|
|
77d86ba835 | ||
|
|
986a748420 | ||
|
|
50e01d17f8 | ||
|
|
084c51b983 | ||
|
|
efbbbc7ec1 | ||
|
|
8e7f0cb9ad | ||
|
|
3357a06cba | ||
|
|
f58c7c1cf3 | ||
|
|
b5801891b8 | ||
|
|
8f77c041f5 | ||
|
|
5a3f3ba030 | ||
|
|
3c25ee6e2c | ||
|
|
b92b0dd508 | ||
|
|
670900bf5a | ||
|
|
219cfb1605 | ||
|
|
bf728a780c | ||
|
|
9955e1dcb7 | ||
|
|
19a28bf6c5 | ||
|
|
3f10a70d4c | ||
|
|
d0dfe78701 | ||
|
|
172addd918 | ||
|
|
ada309cfb9 | ||
|
|
edfc6e99fe | ||
|
|
d00e28ffda | ||
|
|
28a4fd4824 | ||
|
|
80c87c8e2c | ||
|
|
427d811954 |
@@ -0,0 +1,46 @@
|
||||
# Version control
|
||||
.git
|
||||
.github
|
||||
.gitignore
|
||||
|
||||
# Desktop-only (not needed in web container)
|
||||
tauri/
|
||||
landing/
|
||||
docs/
|
||||
mlx-test/
|
||||
scripts/
|
||||
|
||||
# Dependencies & build artifacts (rebuilt in Docker)
|
||||
node_modules/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
*.spec
|
||||
|
||||
# Data (will be bind-mounted)
|
||||
data/
|
||||
backend/data/
|
||||
|
||||
# IDE & OS
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Config files not needed in container
|
||||
biome.json
|
||||
.biomeignore
|
||||
.bumpversion.cfg
|
||||
.npmrc
|
||||
Makefile
|
||||
CHANGELOG.md
|
||||
CONTRIBUTING.md
|
||||
SECURITY.md
|
||||
LICENSE
|
||||
README.md
|
||||
backend/README.md
|
||||
@@ -22,10 +22,10 @@ jobs:
|
||||
args: "--target x86_64-apple-darwin"
|
||||
python-version: "3.12"
|
||||
backend: "pytorch"
|
||||
# - platform: 'ubuntu-22.04'
|
||||
# args: ''
|
||||
# python-version: '3.12'
|
||||
# backend: 'pytorch'
|
||||
- platform: "ubuntu-22.04"
|
||||
args: ""
|
||||
python-version: "3.12"
|
||||
backend: "pytorch"
|
||||
- platform: "windows-latest"
|
||||
args: ""
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -27,6 +27,7 @@ Thank you for your interest in contributing to Voicebox! This document provides
|
||||
```bash
|
||||
rustc --version # Check if installed
|
||||
```
|
||||
- **[Tauri Prerequisites](https://v2.tauri.app/start/prerequisites)** - Tauri-specific system dependencies (varies by OS).
|
||||
|
||||
- **Git** - Version control
|
||||
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
# ============================================================
|
||||
# Voicebox — Local TTS Server with Web UI (CPU)
|
||||
# 3-stage build: Frontend → Python deps → Runtime
|
||||
# ============================================================
|
||||
|
||||
# === Stage 1: Build frontend ===
|
||||
FROM oven/bun:1 AS frontend
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy workspace config and frontend source
|
||||
COPY package.json bun.lock ./
|
||||
COPY app/ ./app/
|
||||
COPY web/ ./web/
|
||||
|
||||
# Strip workspaces not needed for web build, and fix trailing comma
|
||||
RUN sed -i '/"tauri"/d; /"landing"/d' package.json && \
|
||||
sed -i -z 's/,\n ]/\n ]/' package.json
|
||||
RUN bun install --no-save
|
||||
# Build frontend (skip tsc — upstream has pre-existing type errors)
|
||||
RUN cd web && bunx --bun vite build
|
||||
|
||||
|
||||
# === Stage 2: Build Python dependencies ===
|
||||
FROM python:3.11-slim AS backend-builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/requirements.txt .
|
||||
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||
RUN pip install --no-cache-dir --prefix=/install \
|
||||
git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||
|
||||
|
||||
# === Stage 3: Runtime ===
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Create non-root user for security
|
||||
RUN groupadd -r voicebox && \
|
||||
useradd -r -g voicebox -m -s /bin/bash voicebox
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install only runtime system dependencies
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy installed Python packages from builder stage
|
||||
COPY --from=backend-builder /install /usr/local
|
||||
|
||||
# Copy backend application code
|
||||
COPY --chown=voicebox:voicebox backend/ /app/backend/
|
||||
|
||||
# Copy built frontend from frontend stage
|
||||
COPY --from=frontend --chown=voicebox:voicebox /build/web/dist /app/frontend/
|
||||
|
||||
# Create data directories owned by non-root user
|
||||
RUN mkdir -p /app/data/generations /app/data/profiles /app/data/cache \
|
||||
&& chown -R voicebox:voicebox /app/data
|
||||
|
||||
# Switch to non-root user
|
||||
USER voicebox
|
||||
|
||||
# Expose the API port
|
||||
EXPOSE 17493
|
||||
|
||||
# Health check — auto-restart if the server hangs
|
||||
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=60s \
|
||||
CMD curl -f http://localhost:17493/health || exit 1
|
||||
|
||||
# Start the FastAPI server
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "17493"]
|
||||
@@ -0,0 +1,58 @@
|
||||
# Voicebox Offline Mode Fix
|
||||
|
||||
## Problem
|
||||
Voicebox crashes when generating speech if HuggingFace is unreachable, even when models are fully cached locally.
|
||||
|
||||
**Root Cause:**
|
||||
- Voicebox downloads `mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16` (MLX optimized version)
|
||||
- But `mlx_audio.tts.load()` tries to fetch `config.json` from original repo `Qwen/Qwen3-TTS-12Hz-1.7B-Base`
|
||||
- This network request fails → server crashes with `RemoteDisconnected`
|
||||
|
||||
**Related Issues:**
|
||||
- Issue #150: "Internet connection required, even though models are downloaded?"
|
||||
- Issue #151: "API Stability Issues: Model Loading Hangs and Server Crashes"
|
||||
|
||||
## Solution
|
||||
Two-part fix:
|
||||
|
||||
### 1. Monkey-patch huggingface_hub (`backend/utils/hf_offline_patch.py`)
|
||||
- Intercepts cache lookup functions
|
||||
- Forces offline mode early (before mlx_audio imports)
|
||||
- Adds debug logging for cache hits/misses
|
||||
|
||||
### 2. Symlink original repo to MLX version (`ensure_original_qwen_config_cached()`)
|
||||
- When original `Qwen/Qwen3-TTS-12Hz-1.7B-Base` cache doesn't exist
|
||||
- But MLX `mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16` does exist
|
||||
- Creates a symlink so cache lookups succeed
|
||||
|
||||
## Files Changed
|
||||
- `backend/backends/mlx_backend.py` - Added patch imports at top
|
||||
- `backend/utils/hf_offline_patch.py` - New patch module
|
||||
|
||||
## Testing
|
||||
To test this fix:
|
||||
1. Build Voicebox from source: `make build`
|
||||
2. Disconnect from internet
|
||||
3. Try generating speech
|
||||
4. Should work without network requests
|
||||
|
||||
## Build Instructions
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Build the app
|
||||
make build
|
||||
|
||||
# Or build just the server
|
||||
make build-server
|
||||
```
|
||||
|
||||
## Notes
|
||||
- The patch is applied automatically when `mlx_backend.py` is imported
|
||||
- Set `VOICEBOX_OFFLINE_PATCH=0` to disable the patch
|
||||
- The symlink approach works because the config.json is compatible between versions
|
||||
|
||||
---
|
||||
*Patch contributed by community*
|
||||
@@ -98,12 +98,12 @@ Powered by Alibaba's **Qwen3-TTS** — a breakthrough model that achieves near-p
|
||||
- **Instant cloning** — Upload a sample, get a voice profile
|
||||
- **High fidelity** — Natural prosody, emotion, and cadence
|
||||
- **Multi-language** — English, Chinese, and more coming
|
||||
- **Lightning fast on Mac** — MLX backend leverages Apple Silicon's Neural Engine for super fast generation
|
||||
- **Lightning fast on Mac** — MLX backend leverages Apple Silicon's Neural Engine for super-fast generation
|
||||
|
||||
### Voice Profile Management
|
||||
|
||||
- **Create profiles** from audio files or record directly in-app
|
||||
- **Import/Export** profiles to share or backup
|
||||
- **Import/Export** profiles to share or back up
|
||||
- **Multi-sample support** — combine multiple samples for higher quality cloning
|
||||
- **Organize** with descriptions and language tags
|
||||
|
||||
@@ -242,7 +242,7 @@ Install [just](https://github.com/casey/just): `brew install just` or `cargo ins
|
||||
|
||||
Also available via Makefile: `make setup && make dev` (run `make help` for all commands).
|
||||
|
||||
**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org), [XCode on macOS](https://developer.apple.com/xcode/).
|
||||
**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org), [XCode on macOS](https://developer.apple.com/xcode/), [Tauri Prerequisites](https://v2.tauri.app/start/prerequisites/).
|
||||
|
||||
**Performance:**
|
||||
- **Apple Silicon (M1/M2/M3)**: Uses MLX backend with native Metal acceleration for 4-5x faster inference
|
||||
|
||||
+4
-2
@@ -93,10 +93,12 @@ function App() {
|
||||
}
|
||||
|
||||
serverStartingRef.current = true;
|
||||
console.log('Production mode: Starting bundled server...');
|
||||
const isRemote = useServerStore.getState().mode === 'remote';
|
||||
const customModelsDir = useServerStore.getState().customModelsDir;
|
||||
console.log(`Production mode: Starting bundled server... (remote: ${isRemote})`);
|
||||
|
||||
platform.lifecycle
|
||||
.startServer(false)
|
||||
.startServer(isRemote, customModelsDir)
|
||||
.then((serverUrl) => {
|
||||
console.log('Server is ready at:', serverUrl);
|
||||
// Update the server URL in the store with the dynamically assigned port
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Pause, Play, Repeat, Volume2, VolumeX, X } from 'lucide-react';
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useEffect, useId, useMemo, useRef, useState } from 'react';
|
||||
import WaveSurfer from 'wavesurfer.js';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Slider } from '@/components/ui/slider';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import { formatAudioDuration } from '@/lib/utils/audio';
|
||||
import { debug } from '@/lib/utils/debug';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
|
||||
export function AudioPlayer() {
|
||||
const platform = usePlatform();
|
||||
const volumeLabelId = useId();
|
||||
const {
|
||||
audioUrl,
|
||||
audioId,
|
||||
@@ -359,7 +360,7 @@ export function AudioPlayer() {
|
||||
if (shouldAutoPlayNow) {
|
||||
// Clear the flag first
|
||||
usePlayerStore.getState().clearAutoPlayFlag();
|
||||
|
||||
|
||||
// Use a small delay to ensure audio element is fully ready
|
||||
setTimeout(() => {
|
||||
wavesurfer.play().catch((error) => {
|
||||
@@ -664,7 +665,7 @@ export function AudioPlayer() {
|
||||
// Handle shouldAutoPlay flag - for story mode auto-advance
|
||||
const shouldAutoPlay = usePlayerStore((state) => state.shouldAutoPlay);
|
||||
const clearAutoPlayFlag = usePlayerStore((state) => state.clearAutoPlayFlag);
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
const wavesurfer = wavesurferRef.current;
|
||||
if (!wavesurfer || !shouldAutoPlay || duration === 0) {
|
||||
@@ -831,6 +832,9 @@ export function AudioPlayer() {
|
||||
disabled={isLoading || duration === 0}
|
||||
className="shrink-0"
|
||||
title={duration === 0 && !isLoading ? 'Audio not loaded' : ''}
|
||||
aria-label={
|
||||
duration === 0 && !isLoading ? 'Audio not loaded' : isPlaying ? 'Pause' : 'Play'
|
||||
}
|
||||
>
|
||||
{isPlaying ? <Pause className="h-5 w-5" /> : <Play className="h-5 w-5" />}
|
||||
</Button>
|
||||
@@ -845,6 +849,8 @@ export function AudioPlayer() {
|
||||
max={100}
|
||||
step={0.1}
|
||||
className="w-full"
|
||||
aria-label="Playback position"
|
||||
aria-valuetext={`${formatAudioDuration(currentTime)} of ${formatAudioDuration(duration)}`}
|
||||
/>
|
||||
)}
|
||||
{isLoading && (
|
||||
@@ -862,7 +868,9 @@ export function AudioPlayer() {
|
||||
|
||||
{/* Title */}
|
||||
{title && (
|
||||
<div className="text-sm font-medium truncate max-w-[200px] shrink-0">{title}</div>
|
||||
<div className="text-sm font-medium truncate max-w-[200px] shrink-0 hidden lg:block">
|
||||
{title}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Loop Button */}
|
||||
@@ -872,26 +880,37 @@ export function AudioPlayer() {
|
||||
onClick={toggleLoop}
|
||||
className={isLooping ? 'text-primary' : ''}
|
||||
title="Toggle loop"
|
||||
aria-label={isLooping ? 'Stop looping' : 'Loop'}
|
||||
>
|
||||
<Repeat className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
{/* Volume Control */}
|
||||
<div className="flex items-center gap-2 shrink-0 w-[120px]">
|
||||
<div
|
||||
className="flex items-center gap-2 shrink-0 w-[120px]"
|
||||
role="group"
|
||||
aria-label="Volume"
|
||||
>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => setVolume(volume > 0 ? 0 : 1)}
|
||||
className="h-8 w-8"
|
||||
aria-label={volume > 0 ? 'Mute' : 'Unmute'}
|
||||
>
|
||||
{volume > 0 ? <Volume2 className="h-4 w-4" /> : <VolumeX className="h-4 w-4" />}
|
||||
</Button>
|
||||
<span id={volumeLabelId} className="sr-only">
|
||||
Volume level, {Math.round(volume * 100)}%
|
||||
</span>
|
||||
<Slider
|
||||
value={[volume * 100]}
|
||||
onValueChange={handleVolumeChange}
|
||||
max={100}
|
||||
step={1}
|
||||
className="flex-1"
|
||||
aria-labelledby={volumeLabelId}
|
||||
aria-valuetext={`${Math.round(volume * 100)}%`}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -902,6 +921,7 @@ export function AudioPlayer() {
|
||||
onClick={handleClose}
|
||||
className="shrink-0"
|
||||
title="Close player"
|
||||
aria-label="Close player"
|
||||
>
|
||||
<X className="h-5 w-5" />
|
||||
</Button>
|
||||
|
||||
@@ -23,8 +23,8 @@ import {
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
|
||||
interface AudioDevice {
|
||||
id: string;
|
||||
@@ -129,7 +129,7 @@ export function AudioTab() {
|
||||
if (await confirm('Delete this channel?')) {
|
||||
deleteChannel.mutate(channelId);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const allChannels = channels || [];
|
||||
const allDevices = devices || [];
|
||||
@@ -168,7 +168,7 @@ export function AudioTab() {
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-3 p-2">
|
||||
<div className="space-y-3">
|
||||
{allChannels.map((channel) => {
|
||||
const isSelected = selectedChannelId === channel.id;
|
||||
return (
|
||||
@@ -343,7 +343,9 @@ export function AudioTab() {
|
||||
<div className="flex flex-col items-center justify-center py-12 border-2 border-dashed border-muted rounded-md">
|
||||
<CheckCircle2 className="h-12 w-12 text-muted-foreground mb-4" />
|
||||
<p className="text-muted-foreground text-center">
|
||||
{platform.metadata.isTauri ? 'No audio devices found' : 'Audio device selection requires Tauri'}
|
||||
{platform.metadata.isTauri
|
||||
? 'No audio devices found'
|
||||
: 'Audio device selection requires Tauri'}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -12,14 +12,15 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { LANGUAGE_OPTIONS } from '@/lib/constants/languages';
|
||||
import { getLanguageOptionsForEngine, type LanguageCode } from '@/lib/constants/languages';
|
||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||
import { useProfile, useProfiles } from '@/lib/hooks/useProfiles';
|
||||
import { useAddStoryItem, useStory } from '@/lib/hooks/useStories';
|
||||
import { useStory } from '@/lib/hooks/useStories';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
import { ParalinguisticInput } from './ParalinguisticInput';
|
||||
|
||||
interface FloatingGenerateBoxProps {
|
||||
isPlayerOpen?: boolean;
|
||||
@@ -43,8 +44,7 @@ export function FloatingGenerateBox({
|
||||
const selectedStoryId = useStoryStore((state) => state.selectedStoryId);
|
||||
const trackEditorHeight = useStoryStore((state) => state.trackEditorHeight);
|
||||
const { data: currentStory } = useStory(selectedStoryId);
|
||||
const addStoryItem = useAddStoryItem();
|
||||
const { toast } = useToast();
|
||||
const addPendingStoryAdd = useGenerationStore((s) => s.addPendingStoryAdd);
|
||||
|
||||
// Calculate if track editor is visible (on stories route with items)
|
||||
const hasTrackEditor = isStoriesRoute && currentStory && currentStory.items.length > 0;
|
||||
@@ -52,25 +52,9 @@ export function FloatingGenerateBox({
|
||||
const { form, handleSubmit, isPending } = useGenerationForm({
|
||||
onSuccess: async (generationId) => {
|
||||
setIsExpanded(false);
|
||||
// If on stories route and a story is selected, add generation to story
|
||||
// Defer the story add until TTS completes — useGenerationProgress handles it
|
||||
if (isStoriesRoute && selectedStoryId && generationId) {
|
||||
try {
|
||||
await addStoryItem.mutateAsync({
|
||||
storyId: selectedStoryId,
|
||||
data: { generation_id: generationId },
|
||||
});
|
||||
toast({
|
||||
title: 'Added to story',
|
||||
description: `Generation added to "${currentStory?.name || 'story'}"`,
|
||||
});
|
||||
} catch (error) {
|
||||
toast({
|
||||
title: 'Failed to add to story',
|
||||
description:
|
||||
error instanceof Error ? error.message : 'Could not add generation to story',
|
||||
variant: 'destructive',
|
||||
});
|
||||
}
|
||||
addPendingStoryAdd(generationId, selectedStoryId);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -112,6 +96,13 @@ export function FloatingGenerateBox({
|
||||
}
|
||||
}, [selectedProfileId, profiles, setSelectedProfileId]);
|
||||
|
||||
// Sync generation form language with selected profile's language
|
||||
useEffect(() => {
|
||||
if (selectedProfile?.language) {
|
||||
form.setValue('language', selectedProfile.language as LanguageCode);
|
||||
}
|
||||
}, [selectedProfile, form]);
|
||||
|
||||
// Auto-resize textarea based on content (only when expanded)
|
||||
useEffect(() => {
|
||||
if (!isExpanded) {
|
||||
@@ -174,7 +165,7 @@ export function FloatingGenerateBox({
|
||||
isStoriesRoute
|
||||
? // Position aligned with story list: after sidebar + padding, width 360px
|
||||
'left-[calc(5rem+2rem)] w-[360px]'
|
||||
: 'left-[calc(5rem+2rem)] w-[calc((100%-5rem-4rem)/2-1rem)]',
|
||||
: 'left-[calc(5rem+2rem)] right-8 lg:right-auto lg:w-[calc((100%-5rem-4rem)/2-1rem)]',
|
||||
)}
|
||||
style={{
|
||||
// On stories route: offset by track editor height when visible
|
||||
@@ -212,34 +203,57 @@ export function FloatingGenerateBox({
|
||||
transition={{ duration: 0.15, ease: 'easeOut' }}
|
||||
style={{ overflow: 'hidden' }}
|
||||
>
|
||||
<Textarea
|
||||
{...field}
|
||||
ref={(node: HTMLTextAreaElement | null) => {
|
||||
// Store ref for auto-resize (only for active field)
|
||||
if (!isInstructMode) {
|
||||
textareaRef.current = node;
|
||||
{form.watch('engine') === 'chatterbox_turbo' ? (
|
||||
<ParalinguisticInput
|
||||
value={field.value}
|
||||
onChange={field.onChange}
|
||||
placeholder={
|
||||
isStoriesRoute && currentStory
|
||||
? `Generate speech for "${currentStory.name}"... (type / for effects)`
|
||||
: selectedProfile
|
||||
? `Type / for effects like [laugh], [sigh]...`
|
||||
: 'Select a voice profile above...'
|
||||
}
|
||||
// Forward ref to react-hook-form
|
||||
if (typeof field.ref === 'function') {
|
||||
field.ref(node);
|
||||
className="px-3 py-2 resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm w-full"
|
||||
style={{
|
||||
minHeight: isExpanded ? '100px' : '32px',
|
||||
maxHeight: '300px',
|
||||
overflowY: 'auto',
|
||||
}}
|
||||
disabled={!selectedProfileId}
|
||||
onClick={() => setIsExpanded(true)}
|
||||
onFocus={() => setIsExpanded(true)}
|
||||
/>
|
||||
) : (
|
||||
<Textarea
|
||||
{...field}
|
||||
ref={(node: HTMLTextAreaElement | null) => {
|
||||
// Store ref for auto-resize (only for active field)
|
||||
if (!isInstructMode) {
|
||||
textareaRef.current = node;
|
||||
}
|
||||
// Forward ref to react-hook-form
|
||||
if (typeof field.ref === 'function') {
|
||||
field.ref(node);
|
||||
}
|
||||
}}
|
||||
placeholder={
|
||||
isStoriesRoute && currentStory
|
||||
? `Generate speech for "${currentStory.name}"...`
|
||||
: selectedProfile
|
||||
? `Generate speech using ${selectedProfile.name}...`
|
||||
: 'Select a voice profile above...'
|
||||
}
|
||||
}}
|
||||
placeholder={
|
||||
isStoriesRoute && currentStory
|
||||
? `Generate speech for "${currentStory.name}"...`
|
||||
: selectedProfile
|
||||
? `Generate speech using ${selectedProfile.name}...`
|
||||
: 'Select a voice profile above...'
|
||||
}
|
||||
className="resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm placeholder:text-muted-foreground/60 w-full"
|
||||
style={{
|
||||
minHeight: isExpanded ? '100px' : '32px',
|
||||
maxHeight: '300px',
|
||||
}}
|
||||
disabled={!selectedProfileId}
|
||||
onClick={() => setIsExpanded(true)}
|
||||
onFocus={() => setIsExpanded(true)}
|
||||
/>
|
||||
className="resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm placeholder:text-muted-foreground/60 w-full"
|
||||
style={{
|
||||
minHeight: isExpanded ? '100px' : '32px',
|
||||
maxHeight: '300px',
|
||||
}}
|
||||
disabled={!selectedProfileId}
|
||||
onClick={() => setIsExpanded(true)}
|
||||
onFocus={() => setIsExpanded(true)}
|
||||
/>
|
||||
)}
|
||||
</motion.div>
|
||||
</FormControl>
|
||||
<FormMessage className="text-xs" />
|
||||
@@ -300,6 +314,13 @@ export function FloatingGenerateBox({
|
||||
disabled={isPending || !selectedProfileId}
|
||||
className="h-10 w-10 rounded-full bg-accent hover:bg-accent/90 hover:scale-105 text-accent-foreground shadow-lg hover:shadow-accent/50 transition-all duration-200"
|
||||
size="icon"
|
||||
aria-label={
|
||||
isPending
|
||||
? 'Generating...'
|
||||
: !selectedProfileId
|
||||
? 'Select a voice profile first'
|
||||
: 'Generate speech'
|
||||
}
|
||||
>
|
||||
{isPending ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
@@ -336,6 +357,9 @@ export function FloatingGenerateBox({
|
||||
? 'bg-accent text-accent-foreground border border-accent hover:bg-accent/90'
|
||||
: 'bg-card border border-border hover:bg-background/50',
|
||||
)}
|
||||
aria-label={
|
||||
isInstructMode ? 'Fine tune instructions, on' : 'Fine tune instructions'
|
||||
}
|
||||
>
|
||||
<SlidersHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
@@ -381,25 +405,30 @@ export function FloatingGenerateBox({
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage className="text-xs" />
|
||||
</FormItem>
|
||||
)}
|
||||
render={({ field }) => {
|
||||
const engineLangs = getLanguageOptionsForEngine(
|
||||
form.watch('engine') || 'qwen',
|
||||
);
|
||||
return (
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{engineLangs.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage className="text-xs" />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
<FormItem className="flex-1 space-y-0">
|
||||
@@ -409,13 +438,19 @@ export function FloatingGenerateBox({
|
||||
? 'luxtts'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
@@ -441,6 +476,12 @@ export function FloatingGenerateBox({
|
||||
<SelectItem value="chatterbox" className="text-xs text-muted-foreground">
|
||||
Chatterbox
|
||||
</SelectItem>
|
||||
<SelectItem
|
||||
value="chatterbox_turbo"
|
||||
className="text-xs text-muted-foreground"
|
||||
>
|
||||
Chatterbox Turbo
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
@@ -19,10 +19,11 @@ import {
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { LANGUAGE_OPTIONS } from '@/lib/constants/languages';
|
||||
import { getLanguageOptionsForEngine } from '@/lib/constants/languages';
|
||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||
import { useProfile } from '@/lib/hooks/useProfiles';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
import { ParalinguisticInput } from './ParalinguisticInput';
|
||||
|
||||
export function GenerationForm() {
|
||||
const selectedProfileId = useUIStore((state) => state.selectedProfileId);
|
||||
@@ -64,13 +65,26 @@ export function GenerationForm() {
|
||||
<FormItem>
|
||||
<FormLabel>Text to Speak</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="Enter the text you want to generate..."
|
||||
className="min-h-[150px]"
|
||||
{...field}
|
||||
/>
|
||||
{form.watch('engine') === 'chatterbox_turbo' ? (
|
||||
<ParalinguisticInput
|
||||
value={field.value}
|
||||
onChange={field.onChange}
|
||||
placeholder="Enter text... type / for effects like [laugh], [sigh]"
|
||||
className="min-h-[150px] rounded-md border border-input bg-background px-3 py-2"
|
||||
/>
|
||||
) : (
|
||||
<Textarea
|
||||
placeholder="Enter the text you want to generate..."
|
||||
className="min-h-[150px]"
|
||||
{...field}
|
||||
/>
|
||||
)}
|
||||
</FormControl>
|
||||
<FormDescription>Max 5000 characters</FormDescription>
|
||||
<FormDescription>
|
||||
{form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'Max 5000 characters. Type / to insert sound effects.'
|
||||
: 'Max 5000 characters'}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
@@ -109,13 +123,19 @@ export function GenerationForm() {
|
||||
? 'luxtts'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'chatterbox'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'chatterbox_turbo'
|
||||
: `qwen:${form.watch('modelSize') || '1.7B'}`
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
if (value === 'luxtts') {
|
||||
form.setValue('engine', 'luxtts');
|
||||
form.setValue('language', 'en');
|
||||
} else if (value === 'chatterbox') {
|
||||
form.setValue('engine', 'chatterbox');
|
||||
} else if (value === 'chatterbox_turbo') {
|
||||
form.setValue('engine', 'chatterbox_turbo');
|
||||
form.setValue('language', 'en');
|
||||
} else {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
@@ -133,40 +153,46 @@ export function GenerationForm() {
|
||||
<SelectItem value="qwen:0.6B">Qwen3-TTS 0.6B</SelectItem>
|
||||
<SelectItem value="luxtts">LuxTTS</SelectItem>
|
||||
<SelectItem value="chatterbox">Chatterbox</SelectItem>
|
||||
<SelectItem value="chatterbox_turbo">Chatterbox Turbo</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{form.watch('engine') === 'luxtts'
|
||||
? 'Fast, English-focused'
|
||||
: form.watch('engine') === 'chatterbox'
|
||||
? 'Multilingual, incl. Hebrew'
|
||||
: 'Multi-language, two sizes'}
|
||||
? '23 languages, incl. Hebrew'
|
||||
: form.watch('engine') === 'chatterbox_turbo'
|
||||
? 'English, [laugh] [cough] tags'
|
||||
: 'Multi-language, two sizes'}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="language"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Language</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{LANGUAGE_OPTIONS.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
render={({ field }) => {
|
||||
const engineLangs = getLanguageOptionsForEngine(form.watch('engine') || 'qwen');
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>Language</FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{engineLangs.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value}>
|
||||
{lang.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
/**
|
||||
* ParalinguisticInput — a contentEditable rich text input that renders
|
||||
* Chatterbox Turbo paralinguistic tags (e.g. [laugh]) as inline badges.
|
||||
*
|
||||
* Trigger: typing "/" opens an autocomplete dropdown.
|
||||
* Paste: pasting text with [tag] patterns auto-converts to badges.
|
||||
* Output: serializes badges back to plain [tag] text for the API.
|
||||
*/
|
||||
|
||||
import { AnimatePresence, motion } from 'framer-motion';
|
||||
import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from 'react';
|
||||
import { createPortal } from 'react-dom';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
|
||||
// ── Tag definitions ─────────────────────────────────────────────────
|
||||
const PARALINGUISTIC_TAGS = [
|
||||
{ tag: '[laugh]', label: 'laugh', emoji: '\u{1F602}' },
|
||||
{ tag: '[chuckle]', label: 'chuckle', emoji: '\u{1F60F}' },
|
||||
{ tag: '[gasp]', label: 'gasp', emoji: '\u{1F62E}' },
|
||||
{ tag: '[cough]', label: 'cough', emoji: '\u{1F637}' },
|
||||
{ tag: '[sigh]', label: 'sigh', emoji: '\u{1F614}' },
|
||||
{ tag: '[groan]', label: 'groan', emoji: '\u{1F629}' },
|
||||
{ tag: '[sniff]', label: 'sniff', emoji: '\u{1F443}' },
|
||||
{ tag: '[shush]', label: 'shush', emoji: '\u{1F92B}' },
|
||||
{ tag: '[clear throat]', label: 'clear throat', emoji: '\u{1F64A}' },
|
||||
] as const;
|
||||
|
||||
const TAG_REGEX = /\[(laugh|chuckle|gasp|cough|sigh|groan|sniff|shush|clear throat)\]/gi;
|
||||
|
||||
// Data attribute used to identify badge spans in the DOM
|
||||
const BADGE_ATTR = 'data-ptag';
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Build an inline badge <span> for a tag. */
|
||||
function makeBadgeHTML(tag: string): string {
|
||||
const entry = PARALINGUISTIC_TAGS.find((t) => t.tag.toLowerCase() === tag.toLowerCase());
|
||||
const label = entry?.label ?? tag.replace(/[[\]]/g, '');
|
||||
const emoji = entry?.emoji ?? '';
|
||||
// Non-editable inline badge. Zero-width spaces around it let the
|
||||
// caret sit on either side so the user can type before/after.
|
||||
return `\u200B<span ${BADGE_ATTR}="${tag}" contenteditable="false" class="ptag-badge">${emoji ? `${emoji}\u00A0` : ''}${label}</span>\u200B`;
|
||||
}
|
||||
|
||||
/** Convert plain text with [tag] patterns into HTML with badge spans. */
|
||||
function textToHTML(text: string): string {
|
||||
// Escape HTML entities first
|
||||
const escaped = text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
// Replace tag patterns with badge HTML
|
||||
return escaped.replace(TAG_REGEX, (match) => makeBadgeHTML(match));
|
||||
}
|
||||
|
||||
/** Serialize the contentEditable innerHTML back to plain text with [tag] syntax. */
|
||||
function htmlToText(container: HTMLElement): string {
|
||||
let result = '';
|
||||
for (const node of container.childNodes) {
|
||||
if (node.nodeType === Node.TEXT_NODE) {
|
||||
// Strip zero-width spaces we added around badges
|
||||
result += (node.textContent ?? '').replace(/\u200B/g, '');
|
||||
} else if (node.nodeType === Node.ELEMENT_NODE) {
|
||||
const el = node as HTMLElement;
|
||||
if (el.hasAttribute(BADGE_ATTR)) {
|
||||
result += el.getAttribute(BADGE_ATTR) ?? '';
|
||||
} else if (el.tagName === 'BR') {
|
||||
result += '\n';
|
||||
} else {
|
||||
// Recurse for nested elements (e.g. spans from paste)
|
||||
result += htmlToText(el);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Get the text content from the current caret position back to the last
|
||||
* whitespace or start of container, to detect the "/" trigger. */
|
||||
function getWordBeforeCaret(_container: HTMLElement): { word: string; range: Range | null } {
|
||||
const sel = window.getSelection();
|
||||
if (!sel || sel.rangeCount === 0) return { word: '', range: null };
|
||||
const range = sel.getRangeAt(0).cloneRange();
|
||||
range.collapse(true);
|
||||
|
||||
// Walk backwards from caret through the text node
|
||||
const textNode = range.startContainer;
|
||||
if (textNode.nodeType !== Node.TEXT_NODE) return { word: '', range: null };
|
||||
const text = textNode.textContent ?? '';
|
||||
const offset = range.startOffset;
|
||||
|
||||
let start = offset;
|
||||
while (
|
||||
start > 0 &&
|
||||
text[start - 1] !== ' ' &&
|
||||
text[start - 1] !== '\n' &&
|
||||
text[start - 1] !== '\u00A0'
|
||||
) {
|
||||
start--;
|
||||
}
|
||||
|
||||
const word = text.slice(start, offset);
|
||||
const wordRange = document.createRange();
|
||||
wordRange.setStart(textNode, start);
|
||||
wordRange.setEnd(textNode, offset);
|
||||
|
||||
return { word, range: wordRange };
|
||||
}
|
||||
|
||||
// ── Component ───────────────────────────────────────────────────────
|
||||
|
||||
export interface ParalinguisticInputProps {
|
||||
value?: string;
|
||||
onChange?: (value: string) => void;
|
||||
placeholder?: string;
|
||||
disabled?: boolean;
|
||||
className?: string;
|
||||
style?: React.CSSProperties;
|
||||
onClick?: () => void;
|
||||
onFocus?: () => void;
|
||||
}
|
||||
|
||||
export interface ParalinguisticInputRef {
|
||||
focus: () => void;
|
||||
element: HTMLDivElement | null;
|
||||
}
|
||||
|
||||
export const ParalinguisticInput = forwardRef<ParalinguisticInputRef, ParalinguisticInputProps>(
|
||||
function ParalinguisticInput(
|
||||
{ value, onChange, placeholder, disabled, className, style, onClick, onFocus },
|
||||
ref,
|
||||
) {
|
||||
const editorRef = useRef<HTMLDivElement>(null);
|
||||
const [showMenu, setShowMenu] = useState(false);
|
||||
const [menuFilter, setMenuFilter] = useState('');
|
||||
const [menuIndex, setMenuIndex] = useState(0);
|
||||
const [menuPosition, setMenuPosition] = useState<{ bottom: number; left: number }>({
|
||||
bottom: 0,
|
||||
left: 0,
|
||||
});
|
||||
const triggerRangeRef = useRef<Range | null>(null);
|
||||
const lastSerializedRef = useRef<string>('');
|
||||
const isComposingRef = useRef(false);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
focus: () => editorRef.current?.focus(),
|
||||
element: editorRef.current,
|
||||
}));
|
||||
|
||||
// Filtered tag list for the autocomplete menu
|
||||
const filteredTags = PARALINGUISTIC_TAGS.filter((t) =>
|
||||
t.label.toLowerCase().includes(menuFilter.toLowerCase()),
|
||||
);
|
||||
|
||||
// ── Sync external value → editor ──────────────────────────────
|
||||
useEffect(() => {
|
||||
const el = editorRef.current;
|
||||
if (!el) return;
|
||||
// Only update DOM if the external value differs from what we last emitted
|
||||
if (value !== undefined && value !== lastSerializedRef.current) {
|
||||
lastSerializedRef.current = value;
|
||||
el.innerHTML = value ? textToHTML(value) : '';
|
||||
}
|
||||
}, [value]);
|
||||
|
||||
// ── Emit plain-text value on input ────────────────────────────
|
||||
const emitChange = useCallback(() => {
|
||||
const el = editorRef.current;
|
||||
if (!el || !onChange) return;
|
||||
const text = htmlToText(el);
|
||||
lastSerializedRef.current = text;
|
||||
onChange(text);
|
||||
}, [onChange]);
|
||||
|
||||
// ── Insert a tag badge at the caret ───────────────────────────
|
||||
const insertTag = useCallback(
|
||||
(tag: string) => {
|
||||
const el = editorRef.current;
|
||||
if (!el) return;
|
||||
|
||||
// Delete the /filter text
|
||||
const wordRange = triggerRangeRef.current;
|
||||
if (wordRange) {
|
||||
wordRange.deleteContents();
|
||||
}
|
||||
|
||||
// Insert badge HTML
|
||||
const temp = document.createElement('span');
|
||||
temp.innerHTML = makeBadgeHTML(tag);
|
||||
const frag = document.createDocumentFragment();
|
||||
let lastNode: Node | null = null;
|
||||
while (temp.firstChild) {
|
||||
lastNode = frag.appendChild(temp.firstChild);
|
||||
}
|
||||
|
||||
const sel = window.getSelection();
|
||||
if (sel && sel.rangeCount > 0) {
|
||||
const range = sel.getRangeAt(0);
|
||||
range.deleteContents();
|
||||
range.insertNode(frag);
|
||||
|
||||
// Move caret after the badge
|
||||
if (lastNode) {
|
||||
const newRange = document.createRange();
|
||||
newRange.setStartAfter(lastNode);
|
||||
newRange.collapse(true);
|
||||
sel.removeAllRanges();
|
||||
sel.addRange(newRange);
|
||||
}
|
||||
}
|
||||
|
||||
setShowMenu(false);
|
||||
setMenuFilter('');
|
||||
emitChange();
|
||||
el.focus();
|
||||
},
|
||||
[emitChange],
|
||||
);
|
||||
|
||||
// ── Handle keydown for autocomplete navigation ────────────────
|
||||
const handleKeyDown = useCallback(
|
||||
(e: React.KeyboardEvent) => {
|
||||
if (showMenu) {
|
||||
if (filteredTags.length === 0) {
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault();
|
||||
setShowMenu(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (e.key === 'ArrowDown') {
|
||||
e.preventDefault();
|
||||
setMenuIndex((i) => (i + 1) % filteredTags.length);
|
||||
} else if (e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
setMenuIndex((i) => (i - 1 + filteredTags.length) % filteredTags.length);
|
||||
} else if (e.key === 'Enter' || e.key === 'Tab') {
|
||||
e.preventDefault();
|
||||
if (filteredTags[menuIndex]) {
|
||||
insertTag(filteredTags[menuIndex].tag);
|
||||
}
|
||||
} else if (e.key === 'Escape') {
|
||||
e.preventDefault();
|
||||
setShowMenu(false);
|
||||
}
|
||||
} else {
|
||||
// Prevent Enter from creating <div> blocks in contentEditable
|
||||
if (e.key === 'Enter' && !e.shiftKey) {
|
||||
// Let the form handle submit
|
||||
}
|
||||
}
|
||||
},
|
||||
[showMenu, filteredTags, menuIndex, insertTag],
|
||||
);
|
||||
|
||||
// ── Handle input (check for / trigger) ────────────────────────
|
||||
const handleInput = useCallback(() => {
|
||||
if (isComposingRef.current) return;
|
||||
const el = editorRef.current;
|
||||
if (!el) return;
|
||||
|
||||
const { word, range } = getWordBeforeCaret(el);
|
||||
|
||||
if (word.startsWith('/')) {
|
||||
const filter = word.slice(1); // strip the /
|
||||
setMenuFilter(filter);
|
||||
setMenuIndex(0);
|
||||
triggerRangeRef.current = range;
|
||||
|
||||
// Position the menu above the caret using viewport coords (portalled)
|
||||
const sel = window.getSelection();
|
||||
if (sel && sel.rangeCount > 0) {
|
||||
const rect = sel.getRangeAt(0).getBoundingClientRect();
|
||||
setMenuPosition({
|
||||
bottom: window.innerHeight - rect.top + 4,
|
||||
left: rect.left,
|
||||
});
|
||||
}
|
||||
|
||||
setShowMenu(true);
|
||||
} else {
|
||||
setShowMenu(false);
|
||||
}
|
||||
|
||||
emitChange();
|
||||
}, [emitChange]);
|
||||
|
||||
// ── Handle paste — convert [tag] patterns to badges ───────────
|
||||
const handlePaste = useCallback(
|
||||
(e: React.ClipboardEvent) => {
|
||||
e.preventDefault();
|
||||
const text = e.clipboardData.getData('text/plain');
|
||||
if (!text) return;
|
||||
|
||||
const el = editorRef.current;
|
||||
if (!el) return;
|
||||
|
||||
const html = textToHTML(text);
|
||||
|
||||
// Insert at caret
|
||||
const sel = window.getSelection();
|
||||
if (sel && sel.rangeCount > 0) {
|
||||
const range = sel.getRangeAt(0);
|
||||
range.deleteContents();
|
||||
const temp = document.createElement('div');
|
||||
temp.innerHTML = html;
|
||||
const frag = document.createDocumentFragment();
|
||||
let lastNode: Node | null = null;
|
||||
while (temp.firstChild) {
|
||||
lastNode = frag.appendChild(temp.firstChild);
|
||||
}
|
||||
range.insertNode(frag);
|
||||
if (lastNode) {
|
||||
const newRange = document.createRange();
|
||||
newRange.setStartAfter(lastNode);
|
||||
newRange.collapse(true);
|
||||
sel.removeAllRanges();
|
||||
sel.addRange(newRange);
|
||||
}
|
||||
}
|
||||
|
||||
emitChange();
|
||||
},
|
||||
[emitChange],
|
||||
);
|
||||
|
||||
// ── Show placeholder ──────────────────────────────────────────
|
||||
const isEmpty = !value || value.trim() === '';
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
{/* Placeholder */}
|
||||
{isEmpty && placeholder && (
|
||||
<div
|
||||
className="pointer-events-none absolute inset-0 text-sm text-muted-foreground/60 px-3 py-2 select-none"
|
||||
aria-hidden
|
||||
>
|
||||
{placeholder}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Editable area */}
|
||||
<div
|
||||
ref={editorRef}
|
||||
contentEditable={!disabled}
|
||||
suppressContentEditableWarning
|
||||
role={disabled ? undefined : 'textbox'}
|
||||
aria-multiline={disabled ? undefined : true}
|
||||
aria-placeholder={placeholder}
|
||||
aria-disabled={disabled}
|
||||
tabIndex={disabled ? -1 : 0}
|
||||
className={cn(
|
||||
'min-h-[32px] text-sm whitespace-pre-wrap break-words outline-none',
|
||||
'[&_.ptag-badge]:inline-flex [&_.ptag-badge]:items-center [&_.ptag-badge]:rounded-full',
|
||||
'[&_.ptag-badge]:bg-accent/20 [&_.ptag-badge]:text-accent [&_.ptag-badge]:border [&_.ptag-badge]:border-accent/30',
|
||||
'[&_.ptag-badge]:px-2 [&_.ptag-badge]:py-0 [&_.ptag-badge]:text-xs [&_.ptag-badge]:font-medium',
|
||||
'[&_.ptag-badge]:mx-0.5 [&_.ptag-badge]:select-none [&_.ptag-badge]:cursor-default',
|
||||
'[&_.ptag-badge]:align-baseline',
|
||||
disabled && 'opacity-50 cursor-not-allowed',
|
||||
className,
|
||||
)}
|
||||
style={style}
|
||||
onInput={!disabled ? handleInput : undefined}
|
||||
onKeyDown={!disabled ? handleKeyDown : undefined}
|
||||
onPaste={!disabled ? handlePaste : undefined}
|
||||
onClick={!disabled ? onClick : undefined}
|
||||
onFocus={!disabled ? onFocus : undefined}
|
||||
onBlur={() => {
|
||||
setShowMenu(false);
|
||||
triggerRangeRef.current = null;
|
||||
}}
|
||||
onCompositionStart={() => {
|
||||
isComposingRef.current = true;
|
||||
}}
|
||||
onCompositionEnd={() => {
|
||||
isComposingRef.current = false;
|
||||
handleInput();
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Autocomplete dropdown — portalled to body, positioned above the caret */}
|
||||
{showMenu &&
|
||||
filteredTags.length > 0 &&
|
||||
createPortal(
|
||||
<AnimatePresence>
|
||||
<motion.div
|
||||
initial={{ opacity: 0, y: 4 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
exit={{ opacity: 0, y: 4 }}
|
||||
transition={{ duration: 0.12 }}
|
||||
className="fixed z-[9999] min-w-[200px] max-h-[280px] overflow-y-auto rounded-lg border border-border bg-popover shadow-lg"
|
||||
style={{
|
||||
bottom: menuPosition.bottom,
|
||||
left: menuPosition.left,
|
||||
}}
|
||||
>
|
||||
{filteredTags.map((t, i) => (
|
||||
<button
|
||||
key={t.tag}
|
||||
type="button"
|
||||
className={cn(
|
||||
'flex items-center gap-2 w-full px-3 py-1.5 text-sm text-left transition-colors',
|
||||
i === menuIndex
|
||||
? 'bg-accent/20 text-accent-foreground'
|
||||
: 'text-popover-foreground hover:bg-muted/50',
|
||||
)}
|
||||
onMouseDown={(e) => {
|
||||
e.preventDefault(); // Keep focus in editor
|
||||
insertTag(t.tag);
|
||||
}}
|
||||
onMouseEnter={() => setMenuIndex(i)}
|
||||
>
|
||||
<span className="text-base leading-none">{t.emoji}</span>
|
||||
<span>{t.label}</span>
|
||||
<span className="ml-auto text-xs text-muted-foreground font-mono">{t.tag}</span>
|
||||
</button>
|
||||
))}
|
||||
</motion.div>
|
||||
</AnimatePresence>,
|
||||
document.body,
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,13 +1,15 @@
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import {
|
||||
AudioWaveform,
|
||||
Download,
|
||||
FileArchive,
|
||||
Loader2,
|
||||
MoreHorizontal,
|
||||
Play,
|
||||
RotateCcw,
|
||||
Trash2,
|
||||
} from 'lucide-react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import Loader from 'react-loaders';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
@@ -36,7 +38,8 @@ import {
|
||||
useImportGeneration,
|
||||
} from '@/lib/hooks/useHistory';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { formatDate, formatDuration } from '@/lib/utils/format';
|
||||
import { formatDate, formatDuration, formatEngineName } from '@/lib/utils/format';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
|
||||
// OLD TABLE-BASED COMPONENT - REMOVED (can be found in git history)
|
||||
@@ -54,9 +57,12 @@ export function HistoryTable() {
|
||||
const [importDialogOpen, setImportDialogOpen] = useState(false);
|
||||
const [selectedFile, setSelectedFile] = useState<File | null>(null);
|
||||
const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
|
||||
const [generationToDelete, setGenerationToDelete] = useState<{ id: string; name: string } | null>(null);
|
||||
const [generationToDelete, setGenerationToDelete] = useState<{ id: string; name: string } | null>(
|
||||
null,
|
||||
);
|
||||
const limit = 20;
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const {
|
||||
data: historyData,
|
||||
@@ -71,6 +77,7 @@ export function HistoryTable() {
|
||||
const exportGeneration = useExportGeneration();
|
||||
const exportGenerationAudio = useExportGenerationAudio();
|
||||
const importGeneration = useImportGeneration();
|
||||
const addPendingGeneration = useGenerationStore((state) => state.addPendingGeneration);
|
||||
const setAudioWithAutoPlay = usePlayerStore((state) => state.setAudioWithAutoPlay);
|
||||
const restartCurrentAudio = usePlayerStore((state) => state.restartCurrentAudio);
|
||||
const currentAudioId = usePlayerStore((state) => state.audioId);
|
||||
@@ -194,6 +201,20 @@ export function HistoryTable() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleRetry = async (generationId: string) => {
|
||||
try {
|
||||
const result = await apiClient.retryGeneration(generationId);
|
||||
addPendingGeneration(result.id);
|
||||
queryClient.invalidateQueries({ queryKey: ['history'] });
|
||||
} catch (error) {
|
||||
toast({
|
||||
title: 'Retry failed',
|
||||
description: error instanceof Error ? error.message : 'Could not retry generation',
|
||||
variant: 'destructive',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleImportConfirm = () => {
|
||||
if (selectedFile) {
|
||||
importGeneration.mutate(selectedFile, {
|
||||
@@ -250,25 +271,54 @@ export function HistoryTable() {
|
||||
>
|
||||
{history.map((gen) => {
|
||||
const isCurrentlyPlaying = currentAudioId === gen.id && isPlaying;
|
||||
const isGenerating = gen.status === 'generating';
|
||||
const isFailed = gen.status === 'failed';
|
||||
const isPlayable = !isGenerating && !isFailed;
|
||||
return (
|
||||
<div
|
||||
key={gen.id}
|
||||
role={isPlayable ? 'button' : undefined}
|
||||
tabIndex={isPlayable ? 0 : undefined}
|
||||
className={cn(
|
||||
'flex items-stretch gap-4 h-26 border rounded-md p-3 bg-card hover:bg-muted/70 transition-colors text-left w-full',
|
||||
'flex items-stretch gap-4 h-26 border rounded-md p-3 bg-card transition-colors text-left w-full',
|
||||
isPlayable && 'hover:bg-muted/70 cursor-pointer',
|
||||
isCurrentlyPlaying && 'bg-muted/70',
|
||||
)}
|
||||
aria-label={
|
||||
isGenerating
|
||||
? `Generating speech for ${gen.profile_name}...`
|
||||
: isFailed
|
||||
? `Generation failed for ${gen.profile_name}`
|
||||
: isCurrentlyPlaying
|
||||
? `Sample from ${gen.profile_name}, ${formatDuration(gen.duration ?? 0)}, ${formatDate(gen.created_at)}. Playing. Press Enter to restart.`
|
||||
: `Sample from ${gen.profile_name}, ${formatDuration(gen.duration ?? 0)}, ${formatDate(gen.created_at)}. Press Enter to play.`
|
||||
}
|
||||
onMouseDown={(e) => {
|
||||
// Don't trigger play if clicking on textarea or if text is selected
|
||||
if (!isPlayable) return;
|
||||
const target = e.target as HTMLElement;
|
||||
if (target.closest('textarea') || window.getSelection()?.toString()) {
|
||||
return;
|
||||
}
|
||||
handlePlay(gen.id, gen.text, gen.profile_id);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (!isPlayable) return;
|
||||
const target = e.target as HTMLElement;
|
||||
if (target.closest('textarea') || target.closest('button')) return;
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
handlePlay(gen.id, gen.text, gen.profile_id);
|
||||
}
|
||||
}}
|
||||
>
|
||||
{/* Waveform icon */}
|
||||
<div className="flex items-center shrink-0">
|
||||
<AudioWaveform className="h-5 w-5 text-muted-foreground" />
|
||||
{/* Status icon */}
|
||||
<div className="flex items-center shrink-0 w-10 justify-center overflow-hidden">
|
||||
<div className="scale-50">
|
||||
<Loader
|
||||
type={isGenerating ? 'line-scale' : 'line-scale-pulse-out-rapid'}
|
||||
active={isGenerating || isCurrentlyPlaying}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Left side - Meta information */}
|
||||
@@ -279,11 +329,22 @@ export function HistoryTable() {
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs text-muted-foreground">{gen.language}</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatDuration(gen.duration)}
|
||||
{formatEngineName(gen.engine, gen.model_size)}
|
||||
</span>
|
||||
{isFailed ? (
|
||||
<span className="text-xs text-destructive">Failed</span>
|
||||
) : !isGenerating ? (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatDuration(gen.duration ?? 0)}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{formatDate(gen.created_at)}
|
||||
{isGenerating ? (
|
||||
<span className="text-accent">Generating...</span>
|
||||
) : (
|
||||
formatDate(gen.created_at)
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -293,57 +354,70 @@ export function HistoryTable() {
|
||||
value={gen.text}
|
||||
className="flex-1 resize-none text-sm text-muted-foreground select-text"
|
||||
readOnly
|
||||
aria-label={`Transcript for sample from ${gen.profile_name}, ${formatDuration(gen.duration ?? 0)}`}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Far right - Ellipsis actions */}
|
||||
{/* Far right - Actions */}
|
||||
<div
|
||||
className="w-10 shrink-0 flex justify-end"
|
||||
className="w-10 shrink-0 flex justify-end items-center"
|
||||
onMouseDown={(e) => e.stopPropagation()}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8"
|
||||
aria-label="Actions"
|
||||
>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem
|
||||
onClick={() => handlePlay(gen.id, gen.text, gen.profile_id)}
|
||||
>
|
||||
<Play className="mr-2 h-4 w-4" />
|
||||
Play
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleDownloadAudio(gen.id, gen.text)}
|
||||
disabled={exportGenerationAudio.isPending}
|
||||
>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export Audio
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleExportPackage(gen.id, gen.text)}
|
||||
disabled={exportGeneration.isPending}
|
||||
>
|
||||
<FileArchive className="mr-2 h-4 w-4" />
|
||||
Export Package
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleDeleteClick(gen.id, gen.profile_name)}
|
||||
disabled={deleteGeneration.isPending}
|
||||
className="text-destructive focus:text-destructive"
|
||||
>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
Delete
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
{isFailed ? (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8"
|
||||
aria-label="Retry generation"
|
||||
onClick={() => handleRetry(gen.id)}
|
||||
>
|
||||
<RotateCcw className="h-4 w-4" />
|
||||
</Button>
|
||||
) : isPlayable ? (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8"
|
||||
aria-label="Actions"
|
||||
>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem
|
||||
onClick={() => handlePlay(gen.id, gen.text, gen.profile_id)}
|
||||
>
|
||||
<Play className="mr-2 h-4 w-4" />
|
||||
Play
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleDownloadAudio(gen.id, gen.text)}
|
||||
disabled={exportGenerationAudio.isPending}
|
||||
>
|
||||
<Download className="mr-2 h-4 w-4" />
|
||||
Export Audio
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleExportPackage(gen.id, gen.text)}
|
||||
disabled={exportGeneration.isPending}
|
||||
>
|
||||
<FileArchive className="mr-2 h-4 w-4" />
|
||||
Export Package
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={() => handleDeleteClick(gen.id, gen.profile_name)}
|
||||
disabled={deleteGeneration.isPending}
|
||||
className="text-destructive focus:text-destructive"
|
||||
>
|
||||
<Trash2 className="mr-2 h-4 w-4" />
|
||||
Delete
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
@@ -371,7 +445,8 @@ export function HistoryTable() {
|
||||
<DialogHeader>
|
||||
<DialogTitle>Delete Generation</DialogTitle>
|
||||
<DialogDescription>
|
||||
Are you sure you want to delete this generation from "{generationToDelete?.name}"? This action cannot be undone.
|
||||
Are you sure you want to delete this generation from "{generationToDelete?.name}"?
|
||||
This action cannot be undone.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<DialogFooter>
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
} from '@/components/ui/dialog';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { ProfileList } from '@/components/VoiceProfiles/ProfileList';
|
||||
import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui';
|
||||
|
||||
import { useImportProfile } from '@/lib/hooks/useProfiles';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
@@ -77,9 +77,9 @@ export function MainEditor() {
|
||||
|
||||
return (
|
||||
// Main view: Profiles top left, Generator bottom left, History right
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 gap-6 h-full min-h-0 overflow-hidden relative">
|
||||
<div className="grid grid-cols-1 lg:grid-cols-2 lg:gap-6 h-full min-h-0 overflow-hidden relative">
|
||||
{/* Left Column */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden relative">
|
||||
<div className="flex flex-col min-h-0 overflow-hidden relative lg:overflow-hidden">
|
||||
{/* Scroll Mask - Always visible, behind content */}
|
||||
<div className="absolute top-0 left-0 right-0 h-16 bg-gradient-to-b from-background to-transparent z-0 pointer-events-none" />
|
||||
|
||||
@@ -110,10 +110,7 @@ export function MainEditor() {
|
||||
{/* Scrollable Content */}
|
||||
<div
|
||||
ref={scrollRef}
|
||||
className={cn(
|
||||
'flex-1 min-h-0 overflow-y-auto pt-14',
|
||||
isPlayerVisible ? BOTTOM_SAFE_AREA_PADDING : 'pb-4',
|
||||
)}
|
||||
className={cn('flex-1 min-h-0 overflow-y-auto pt-14 pb-4', isPlayerVisible && 'lg:pb-32')}
|
||||
>
|
||||
<div className="flex flex-col gap-6">
|
||||
<div className="shrink-0 flex flex-col">
|
||||
@@ -123,6 +120,9 @@ export function MainEditor() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Divider - single column only */}
|
||||
{/* <div className="border-t border-border -my-3 lg:hidden" /> */}
|
||||
|
||||
{/* Right Column - History */}
|
||||
<div className="flex flex-col min-h-0 overflow-hidden">
|
||||
<HistoryTable />
|
||||
|
||||
@@ -2,7 +2,7 @@ import { ModelManagement } from '@/components/ServerSettings/ModelManagement';
|
||||
|
||||
export function ModelsTab() {
|
||||
return (
|
||||
<div className="h-full flex flex-col p-4">
|
||||
<div className="h-full flex flex-col">
|
||||
<ModelManagement />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import { zodResolver } from '@hookform/resolvers/zod';
|
||||
import { Loader2, XCircle } from 'lucide-react';
|
||||
import { useEffect } from 'react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import * as z from 'zod';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
@@ -14,10 +17,10 @@ import {
|
||||
FormMessage,
|
||||
} from '@/components/ui/form';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
import { useServerHealth } from '@/lib/hooks/useServer';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
const connectionSchema = z.object({
|
||||
serverUrl: z.string().url('Please enter a valid URL'),
|
||||
@@ -31,7 +34,10 @@ export function ConnectionForm() {
|
||||
const setServerUrl = useServerStore((state) => state.setServerUrl);
|
||||
const keepServerRunningOnClose = useServerStore((state) => state.keepServerRunningOnClose);
|
||||
const setKeepServerRunningOnClose = useServerStore((state) => state.setKeepServerRunningOnClose);
|
||||
const mode = useServerStore((state) => state.mode);
|
||||
const setMode = useServerStore((state) => state.setMode);
|
||||
const { toast } = useToast();
|
||||
const { data: health, isLoading, error: healthError } = useServerHealth();
|
||||
|
||||
const form = useForm<ConnectionFormValues>({
|
||||
resolver: zodResolver(connectionSchema),
|
||||
@@ -49,7 +55,7 @@ export function ConnectionForm() {
|
||||
|
||||
function onSubmit(data: ConnectionFormValues) {
|
||||
setServerUrl(data.serverUrl);
|
||||
form.reset(data); // Reset form state after successful submission
|
||||
form.reset(data);
|
||||
toast({
|
||||
title: 'Server URL updated',
|
||||
description: `Connected to ${data.serverUrl}`,
|
||||
@@ -57,7 +63,7 @@ export function ConnectionForm() {
|
||||
}
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Card role="region" aria-label="Server Connection" tabIndex={0}>
|
||||
<CardHeader>
|
||||
<CardTitle>Server Connection</CardTitle>
|
||||
</CardHeader>
|
||||
@@ -83,10 +89,42 @@ export function ConnectionForm() {
|
||||
</form>
|
||||
</Form>
|
||||
|
||||
{/* Connection status */}
|
||||
<div className="mt-4">
|
||||
{isLoading ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
<span className="text-sm text-muted-foreground">Checking connection...</span>
|
||||
</div>
|
||||
) : healthError ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<XCircle className="h-4 w-4 text-destructive" />
|
||||
<span className="text-sm text-destructive">
|
||||
Connection failed: {healthError.message}
|
||||
</span>
|
||||
</div>
|
||||
) : health ? (
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Badge
|
||||
variant={health.model_loaded || health.model_downloaded ? 'default' : 'secondary'}
|
||||
>
|
||||
{health.model_loaded || health.model_downloaded ? 'Model Ready' : 'No Model'}
|
||||
</Badge>
|
||||
<Badge variant={health.gpu_available ? 'default' : 'secondary'}>
|
||||
GPU: {health.gpu_available ? 'Available' : 'Not Available'}
|
||||
</Badge>
|
||||
{health.vram_used_mb && (
|
||||
<Badge variant="outline">VRAM: {health.vram_used_mb.toFixed(0)} MB</Badge>
|
||||
)}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="mt-6 pt-6 border-t">
|
||||
<div className="flex items-start space-x-3">
|
||||
<Checkbox
|
||||
id="keepServerRunning"
|
||||
className="mt-[6px]"
|
||||
checked={keepServerRunningOnClose}
|
||||
onCheckedChange={(checked: boolean) => {
|
||||
setKeepServerRunningOnClose(checked);
|
||||
@@ -115,6 +153,39 @@ export function ConnectionForm() {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{platform.metadata.isTauri && (
|
||||
<div className="mt-6 pt-6 border-t">
|
||||
<div className="flex items-start space-x-3">
|
||||
<Checkbox
|
||||
id="allowNetworkAccess"
|
||||
className="mt-[6px]"
|
||||
checked={mode === 'remote'}
|
||||
onCheckedChange={(checked: boolean) => {
|
||||
setMode(checked ? 'remote' : 'local');
|
||||
toast({
|
||||
title: 'Setting updated',
|
||||
description: checked
|
||||
? 'Network access enabled. Restart the app to apply.'
|
||||
: 'Network access disabled. Restart the app to apply.',
|
||||
});
|
||||
}}
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<label
|
||||
htmlFor="allowNetworkAccess"
|
||||
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 cursor-pointer"
|
||||
>
|
||||
Allow network access
|
||||
</label>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Makes the server accessible from other devices on your network. Restart the app
|
||||
after changing this setting.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import { Slider } from '@/components/ui/slider';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
export function GenerationSettings() {
|
||||
const maxChunkChars = useServerStore((state) => state.maxChunkChars);
|
||||
const setMaxChunkChars = useServerStore((state) => state.setMaxChunkChars);
|
||||
const crossfadeMs = useServerStore((state) => state.crossfadeMs);
|
||||
const setCrossfadeMs = useServerStore((state) => state.setCrossfadeMs);
|
||||
const normalizeAudio = useServerStore((state) => state.normalizeAudio);
|
||||
const setNormalizeAudio = useServerStore((state) => state.setNormalizeAudio);
|
||||
const autoplayOnGenerate = useServerStore((state) => state.autoplayOnGenerate);
|
||||
const setAutoplayOnGenerate = useServerStore((state) => state.setAutoplayOnGenerate);
|
||||
|
||||
return (
|
||||
<Card role="region" aria-label="Generation Settings" tabIndex={0}>
|
||||
<CardHeader>
|
||||
<CardTitle>Generation Settings</CardTitle>
|
||||
<CardDescription>
|
||||
Controls for long text generation. These settings apply to all engines.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="space-y-6">
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<label htmlFor="maxChunkChars" className="text-sm font-medium leading-none">
|
||||
Auto-chunking limit
|
||||
</label>
|
||||
<span className="text-sm tabular-nums text-muted-foreground">
|
||||
{maxChunkChars} chars
|
||||
</span>
|
||||
</div>
|
||||
<Slider
|
||||
id="maxChunkChars"
|
||||
value={[maxChunkChars]}
|
||||
onValueChange={([value]) => setMaxChunkChars(value)}
|
||||
min={100}
|
||||
max={5000}
|
||||
step={50}
|
||||
aria-label="Auto-chunking character limit"
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Long text is split into chunks at sentence boundaries before generating. Lower values
|
||||
can improve quality for long outputs.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<label htmlFor="crossfadeMs" className="text-sm font-medium leading-none">
|
||||
Chunk crossfade
|
||||
</label>
|
||||
<span className="text-sm tabular-nums text-muted-foreground">
|
||||
{crossfadeMs === 0 ? 'Cut' : `${crossfadeMs}ms`}
|
||||
</span>
|
||||
</div>
|
||||
<Slider
|
||||
id="crossfadeMs"
|
||||
value={[crossfadeMs]}
|
||||
onValueChange={([value]) => setCrossfadeMs(value)}
|
||||
min={0}
|
||||
max={200}
|
||||
step={10}
|
||||
aria-label="Chunk crossfade duration"
|
||||
/>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Blends audio between chunks to smooth transitions. Set to 0 for a hard cut.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start gap-3">
|
||||
<Checkbox
|
||||
id="normalizeAudio"
|
||||
checked={normalizeAudio}
|
||||
onCheckedChange={setNormalizeAudio}
|
||||
className="mt-[6px]"
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<label
|
||||
htmlFor="normalizeAudio"
|
||||
className="text-sm font-medium leading-none cursor-pointer"
|
||||
>
|
||||
Normalize audio
|
||||
</label>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Adjusts output volume to a consistent level across generations.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start gap-3">
|
||||
<Checkbox
|
||||
id="autoplayOnGenerate"
|
||||
checked={autoplayOnGenerate}
|
||||
onCheckedChange={setAutoplayOnGenerate}
|
||||
className="mt-[6px]"
|
||||
/>
|
||||
<div className="space-y-1">
|
||||
<label
|
||||
htmlFor="autoplayOnGenerate"
|
||||
className="text-sm font-medium leading-none cursor-pointer"
|
||||
>
|
||||
Autoplay on generate
|
||||
</label>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Automatically play audio when a generation completes.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { AlertCircle, Cpu, Download, Loader2, RotateCw, Trash2, Zap } from 'lucide-react';
|
||||
import { AlertCircle, Download, Loader2, RotateCw, Trash2 } from 'lucide-react';
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
@@ -216,31 +215,19 @@ export function GpuAcceleration() {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2">
|
||||
<Zap className="h-4 w-4" />
|
||||
GPU Acceleration
|
||||
</CardTitle>
|
||||
<CardTitle>GPU Acceleration</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
{/* Current status */}
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="space-y-1">
|
||||
<div className="text-sm font-medium">Backend</div>
|
||||
<div className="text-sm text-muted-foreground">
|
||||
{isCurrentlyCuda ? 'CUDA (GPU accelerated)' : 'CPU'}
|
||||
</div>
|
||||
<div className="space-y-1">
|
||||
<div className="text-sm font-medium">Backend</div>
|
||||
<div className="text-sm text-muted-foreground">
|
||||
{isCurrentlyCuda
|
||||
? 'CUDA (GPU accelerated)'
|
||||
: hasNativeGpu
|
||||
? `${health.backend_type === 'mlx' ? 'MLX' : 'PyTorch'} (GPU accelerated)`
|
||||
: 'CPU'}
|
||||
</div>
|
||||
<Badge variant={isCurrentlyCuda ? 'default' : 'secondary'}>
|
||||
{isCurrentlyCuda ? (
|
||||
<>
|
||||
<Zap className="h-3 w-3 mr-1" /> CUDA
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Cpu className="h-3 w-3 mr-1" /> CPU
|
||||
</>
|
||||
)}
|
||||
</Badge>
|
||||
</div>
|
||||
|
||||
{/* GPU info from health */}
|
||||
@@ -257,14 +244,6 @@ export function GpuAcceleration() {
|
||||
)}
|
||||
|
||||
{/* Native GPU detected - no CUDA download needed */}
|
||||
{hasNativeGpu && (
|
||||
<div className="p-3 rounded-lg bg-accent/10 border border-accent/20">
|
||||
<div className="text-sm">
|
||||
Your system uses <strong>{health.gpu_type}</strong> for acceleration. No additional
|
||||
downloads needed.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* CUDA download section - only show when native GPU is NOT detected (i.e., Windows/Linux NVIDIA users) */}
|
||||
{!hasNativeGpu && (
|
||||
|
||||
@@ -7,12 +7,14 @@ import {
|
||||
CircleX,
|
||||
Download,
|
||||
ExternalLink,
|
||||
FolderOpen,
|
||||
HardDrive,
|
||||
Heart,
|
||||
Loader2,
|
||||
RotateCcw,
|
||||
Scale,
|
||||
Trash2,
|
||||
Unplug,
|
||||
X,
|
||||
} from 'lucide-react';
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
@@ -40,6 +42,8 @@ import { useToast } from '@/components/ui/use-toast';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { ActiveDownloadTask, HuggingFaceModelInfo, ModelStatus } from '@/lib/api/types';
|
||||
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
async function fetchHuggingFaceModelInfo(repoId: string): Promise<HuggingFaceModelInfo> {
|
||||
const response = await fetch(`https://huggingface.co/api/models/${repoId}`);
|
||||
@@ -47,6 +51,29 @@ async function fetchHuggingFaceModelInfo(repoId: string): Promise<HuggingFaceMod
|
||||
return response.json();
|
||||
}
|
||||
|
||||
const MODEL_DESCRIPTIONS: Record<string, string> = {
|
||||
'qwen-tts-1.7B':
|
||||
'High-quality multilingual TTS by Alibaba. Supports 10 languages with natural prosody and voice cloning from short reference audio.',
|
||||
'qwen-tts-0.6B':
|
||||
'Lightweight version of Qwen TTS. Same language support with faster inference, ideal for lower-end hardware.',
|
||||
luxtts:
|
||||
'Lightweight ZipVoice-based TTS designed for high quality voice cloning and 48kHz speech generation at speeds exceeding 150x realtime.',
|
||||
'chatterbox-tts':
|
||||
'Production-grade open source TTS by Resemble AI. Supports 23 languages with voice cloning and emotion exaggeration control.',
|
||||
'chatterbox-turbo':
|
||||
'Streamlined 350M parameter TTS by Resemble AI. High-quality English speech with less compute and VRAM than larger models.',
|
||||
'whisper-base':
|
||||
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
|
||||
'whisper-small':
|
||||
'Whisper Small (244M parameters). Good balance of speed and accuracy for transcription.',
|
||||
'whisper-medium':
|
||||
'Whisper Medium (769M parameters). Higher accuracy transcription at moderate speed.',
|
||||
'whisper-large':
|
||||
'Whisper Large (1.5B parameters). Best accuracy for speech-to-text across multiple languages.',
|
||||
'whisper-turbo':
|
||||
'Whisper Large v3 Turbo. Pruned for significantly faster inference while maintaining near-large accuracy.',
|
||||
};
|
||||
|
||||
function formatDownloads(n: number): string {
|
||||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
||||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`;
|
||||
@@ -84,6 +111,18 @@ function formatBytes(bytes: number): string {
|
||||
export function ModelManagement() {
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
const platform = usePlatform();
|
||||
const customModelsDir = useServerStore((state) => state.customModelsDir);
|
||||
const setCustomModelsDir = useServerStore((state) => state.setCustomModelsDir);
|
||||
const [migrating, setMigrating] = useState(false);
|
||||
const [migrationProgress, setMigrationProgress] = useState<{
|
||||
current: number;
|
||||
total: number;
|
||||
progress: number;
|
||||
filename?: string;
|
||||
status: string;
|
||||
} | null>(null);
|
||||
const [pendingMigrateDir, setPendingMigrateDir] = useState<string | null>(null);
|
||||
const [downloadingModel, setDownloadingModel] = useState<string | null>(null);
|
||||
const [downloadingDisplayName, setDownloadingDisplayName] = useState<string | null>(null);
|
||||
const [consoleOpen, setConsoleOpen] = useState(false);
|
||||
@@ -103,6 +142,12 @@ export function ModelManagement() {
|
||||
refetchInterval: 5000,
|
||||
});
|
||||
|
||||
const { data: cacheDir } = useQuery({
|
||||
queryKey: ['modelsCacheDir'],
|
||||
queryFn: () => apiClient.getModelsCacheDir(),
|
||||
staleTime: 1000 * 60 * 5,
|
||||
});
|
||||
|
||||
const { data: activeTasks } = useQuery({
|
||||
queryKey: ['activeTasks'],
|
||||
queryFn: () => apiClient.getActiveTasks(),
|
||||
@@ -300,6 +345,27 @@ export function ModelManagement() {
|
||||
},
|
||||
});
|
||||
|
||||
const unloadMutation = useMutation({
|
||||
mutationFn: async (modelName: string) => {
|
||||
return await apiClient.unloadModel(modelName);
|
||||
},
|
||||
onSuccess: async (_data, modelName) => {
|
||||
toast({
|
||||
title: 'Model unloaded',
|
||||
description: `${modelName} has been unloaded from memory.`,
|
||||
});
|
||||
await queryClient.invalidateQueries({ queryKey: ['modelStatus'], refetchType: 'all' });
|
||||
await queryClient.refetchQueries({ queryKey: ['modelStatus'] });
|
||||
},
|
||||
onError: (error: Error) => {
|
||||
toast({
|
||||
title: 'Unload failed',
|
||||
description: error.message,
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const formatSize = (sizeMb?: number): string => {
|
||||
if (!sizeMb) return 'Unknown size';
|
||||
if (sizeMb < 1024) return `${sizeMb.toFixed(1)} MB`;
|
||||
@@ -320,17 +386,18 @@ export function ModelManagement() {
|
||||
setDetailOpen(true);
|
||||
};
|
||||
|
||||
const ttsModels = modelStatus?.models.filter((m) => m.model_name.startsWith('qwen-tts')) ?? [];
|
||||
const otherTtsModels =
|
||||
const voiceModels =
|
||||
modelStatus?.models.filter(
|
||||
(m) => m.model_name.startsWith('luxtts') || m.model_name.startsWith('chatterbox'),
|
||||
(m) =>
|
||||
m.model_name.startsWith('qwen-tts') ||
|
||||
m.model_name.startsWith('luxtts') ||
|
||||
m.model_name.startsWith('chatterbox'),
|
||||
) ?? [];
|
||||
const whisperModels = modelStatus?.models.filter((m) => m.model_name.startsWith('whisper')) ?? [];
|
||||
|
||||
// Build sections
|
||||
const sections: { label: string; models: ModelStatus[] }[] = [
|
||||
{ label: 'Voice Generation', models: ttsModels },
|
||||
...(otherTtsModels.length > 0 ? [{ label: 'Other Voice Models', models: otherTtsModels }] : []),
|
||||
{ label: 'Voice Generation', models: voiceModels },
|
||||
{ label: 'Transcription', models: whisperModels },
|
||||
];
|
||||
|
||||
@@ -359,6 +426,87 @@ export function ModelManagement() {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Model storage location */}
|
||||
{platform.metadata.isTauri && cacheDir && (
|
||||
<div className="shrink-0 pb-4 border-b mb-4">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="min-w-0">
|
||||
<span className="text-xs text-muted-foreground">Storage location</span>
|
||||
<p
|
||||
className="text-xs font-mono text-muted-foreground/70 truncate"
|
||||
title={cacheDir.path}
|
||||
>
|
||||
{cacheDir.path}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-1 shrink-0">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-xs text-muted-foreground h-7 px-2"
|
||||
onClick={async () => {
|
||||
try {
|
||||
const { open } = await import('@tauri-apps/plugin-shell');
|
||||
await open(cacheDir.path);
|
||||
} catch {
|
||||
toast({ title: 'Failed to open model folder', variant: 'destructive' });
|
||||
}
|
||||
}}
|
||||
>
|
||||
<FolderOpen className="h-3 w-3" />
|
||||
Open
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-xs text-muted-foreground h-7 px-2"
|
||||
onClick={async () => {
|
||||
try {
|
||||
const { open: openDialog } = await import('@tauri-apps/plugin-dialog');
|
||||
const selected = await openDialog({
|
||||
directory: true,
|
||||
title: 'Choose model storage folder',
|
||||
});
|
||||
if (!selected) return;
|
||||
const newDir =
|
||||
typeof selected === 'string' ? selected : (selected as { path: string }).path;
|
||||
if (!newDir) return;
|
||||
setPendingMigrateDir(newDir);
|
||||
} catch {
|
||||
toast({ title: 'Failed to open folder picker', variant: 'destructive' });
|
||||
}
|
||||
}}
|
||||
disabled={migrating}
|
||||
>
|
||||
{migrating ? (
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
) : (
|
||||
<FolderOpen className="h-3 w-3" />
|
||||
)}
|
||||
{migrating ? 'Migrating...' : 'Change'}
|
||||
</Button>
|
||||
{customModelsDir && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="text-xs text-muted-foreground h-7 px-2"
|
||||
disabled={migrating}
|
||||
onClick={async () => {
|
||||
setCustomModelsDir(null);
|
||||
toast({ title: 'Reset to default location. Restarting server...' });
|
||||
await platform.lifecycle.restartServer('');
|
||||
queryClient.invalidateQueries();
|
||||
}}
|
||||
>
|
||||
<RotateCcw className="h-3 w-3" />
|
||||
Reset
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Model list */}
|
||||
{isLoading ? (
|
||||
<div className="flex items-center justify-center py-16">
|
||||
@@ -434,9 +582,7 @@ export function ModelManagement() {
|
||||
{formatSize(model.size_mb)}
|
||||
</span>
|
||||
)}
|
||||
{!model.downloaded && !isDownloading && !hasError && (
|
||||
<span className="text-xs text-muted-foreground/60">Not downloaded</span>
|
||||
)}
|
||||
|
||||
<ChevronRight className="h-4 w-4 text-muted-foreground/40 group-hover:text-muted-foreground transition-colors" />
|
||||
</div>
|
||||
</button>
|
||||
@@ -542,25 +688,12 @@ export function ModelManagement() {
|
||||
Loaded
|
||||
</Badge>
|
||||
)}
|
||||
{freshSelectedModel.downloaded && !freshSelectedModel.loaded && (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
<CircleCheck className="h-3 w-3 mr-1" />
|
||||
Downloaded
|
||||
</Badge>
|
||||
)}
|
||||
{selectedState?.hasError && (
|
||||
<Badge variant="destructive" className="text-xs">
|
||||
<CircleX className="h-3 w-3 mr-1" />
|
||||
Error
|
||||
</Badge>
|
||||
)}
|
||||
{!freshSelectedModel.downloaded &&
|
||||
!selectedState?.isDownloading &&
|
||||
!selectedState?.hasError && (
|
||||
<Badge variant="outline" className="text-xs text-muted-foreground">
|
||||
Not downloaded
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* HuggingFace model card info */}
|
||||
@@ -571,26 +704,15 @@ export function ModelManagement() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Description */}
|
||||
{MODEL_DESCRIPTIONS[freshSelectedModel.model_name] && (
|
||||
<p className="text-xs text-muted-foreground leading-relaxed">
|
||||
{MODEL_DESCRIPTIONS[freshSelectedModel.model_name]}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{hfModelInfo && (
|
||||
<div className="space-y-3">
|
||||
{/* Stats row */}
|
||||
<div className="flex items-center gap-4 text-xs text-muted-foreground">
|
||||
<span className="flex items-center gap-1" title="Downloads">
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.downloads)}
|
||||
</span>
|
||||
<span className="flex items-center gap-1" title="Likes">
|
||||
<Heart className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.likes)}
|
||||
</span>
|
||||
{license && (
|
||||
<span className="flex items-center gap-1" title="License">
|
||||
<Scale className="h-3.5 w-3.5" />
|
||||
{formatLicense(license)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Pipeline tag + author */}
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{hfModelInfo.pipeline_tag && (
|
||||
@@ -610,6 +732,24 @@ export function ModelManagement() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Stats row */}
|
||||
<div className="flex items-center gap-4 text-xs text-muted-foreground">
|
||||
<span className="flex items-center gap-1" title="Downloads">
|
||||
<Download className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.downloads)}
|
||||
</span>
|
||||
<span className="flex items-center gap-1" title="Likes">
|
||||
<Heart className="h-3.5 w-3.5" />
|
||||
{formatDownloads(hfModelInfo.likes)}
|
||||
</span>
|
||||
{license && (
|
||||
<span className="flex items-center gap-1" title="License">
|
||||
<Scale className="h-3.5 w-3.5" />
|
||||
{formatLicense(license)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Languages */}
|
||||
{hfModelInfo.cardData?.language && hfModelInfo.cardData.language.length > 0 && (
|
||||
<div>
|
||||
@@ -625,8 +765,8 @@ export function ModelManagement() {
|
||||
|
||||
{/* Disk size */}
|
||||
{freshSelectedModel.downloaded && freshSelectedModel.size_mb && (
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<HardDrive className="h-4 w-4" />
|
||||
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<HardDrive className="h-3.5 w-3.5" />
|
||||
<span>{formatSize(freshSelectedModel.size_mb)} on disk</span>
|
||||
</div>
|
||||
)}
|
||||
@@ -639,7 +779,7 @@ export function ModelManagement() {
|
||||
)}
|
||||
|
||||
{/* Actions */}
|
||||
<div className="flex items-center gap-2 pt-2 border-t">
|
||||
<div className="flex items-center gap-2 pt-2">
|
||||
{selectedState?.hasError ? (
|
||||
<>
|
||||
<Button
|
||||
@@ -697,26 +837,46 @@ export function ModelManagement() {
|
||||
</Button>
|
||||
</>
|
||||
) : freshSelectedModel.downloaded ? (
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setModelToDelete({
|
||||
name: freshSelectedModel.model_name,
|
||||
displayName: freshSelectedModel.display_name,
|
||||
sizeMb: freshSelectedModel.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
variant="outline"
|
||||
disabled={freshSelectedModel.loaded}
|
||||
title={
|
||||
freshSelectedModel.loaded ? 'Unload model before deleting' : 'Delete model'
|
||||
}
|
||||
className="flex-1"
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
{freshSelectedModel.loaded ? 'Unload to Delete' : 'Delete Model'}
|
||||
</Button>
|
||||
<div className="flex gap-2 flex-1">
|
||||
{freshSelectedModel.loaded && (
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => unloadMutation.mutate(freshSelectedModel.model_name)}
|
||||
variant="outline"
|
||||
disabled={unloadMutation.isPending}
|
||||
className="flex-1"
|
||||
>
|
||||
{unloadMutation.isPending ? (
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
) : (
|
||||
<Unplug className="h-4 w-4 mr-2" />
|
||||
)}
|
||||
{unloadMutation.isPending ? 'Unloading...' : 'Unload'}
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
setModelToDelete({
|
||||
name: freshSelectedModel.model_name,
|
||||
displayName: freshSelectedModel.display_name,
|
||||
sizeMb: freshSelectedModel.size_mb,
|
||||
});
|
||||
setDeleteDialogOpen(true);
|
||||
}}
|
||||
variant="outline"
|
||||
disabled={freshSelectedModel.loaded}
|
||||
title={
|
||||
freshSelectedModel.loaded
|
||||
? 'Unload model before deleting'
|
||||
: 'Delete model'
|
||||
}
|
||||
className="flex-1"
|
||||
>
|
||||
<Trash2 className="h-4 w-4 mr-2" />
|
||||
Delete Model
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<Button
|
||||
size="sm"
|
||||
@@ -773,6 +933,229 @@ export function ModelManagement() {
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
{/* Migration confirmation dialog */}
|
||||
<AlertDialog
|
||||
open={!!pendingMigrateDir}
|
||||
onOpenChange={(open) => !open && setPendingMigrateDir(null)}
|
||||
>
|
||||
<AlertDialogContent>
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>Move models to new location?</AlertDialogTitle>
|
||||
<AlertDialogDescription>
|
||||
The server will shut down while models are being moved to the new folder. It will
|
||||
restart automatically once the migration is complete.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<div
|
||||
className="text-xs font-mono text-muted-foreground bg-muted/50 rounded px-3 py-2 truncate"
|
||||
title={pendingMigrateDir ?? ''}
|
||||
>
|
||||
{pendingMigrateDir}
|
||||
</div>
|
||||
<AlertDialogFooter>
|
||||
<AlertDialogCancel>Cancel</AlertDialogCancel>
|
||||
<AlertDialogAction
|
||||
onClick={async () => {
|
||||
if (!pendingMigrateDir) return;
|
||||
const newDir = pendingMigrateDir;
|
||||
setPendingMigrateDir(null);
|
||||
setMigrating(true);
|
||||
setMigrationProgress({
|
||||
current: 0,
|
||||
total: 0,
|
||||
progress: 0,
|
||||
status: 'downloading',
|
||||
filename: 'Preparing...',
|
||||
});
|
||||
try {
|
||||
// Start the migration (background task)
|
||||
await apiClient.migrateModels(newDir);
|
||||
|
||||
// Connect to SSE for progress
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const es = new EventSource(apiClient.getMigrationProgressUrl());
|
||||
es.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
setMigrationProgress(data);
|
||||
if (data.status === 'complete') {
|
||||
es.close();
|
||||
resolve();
|
||||
} else if (data.status === 'error') {
|
||||
es.close();
|
||||
reject(new Error(data.error || 'Migration failed'));
|
||||
}
|
||||
} catch {
|
||||
/* ignore parse errors */
|
||||
}
|
||||
};
|
||||
es.onerror = () => {
|
||||
es.close();
|
||||
reject(new Error('Lost connection during migration'));
|
||||
};
|
||||
});
|
||||
|
||||
setCustomModelsDir(newDir);
|
||||
setMigrationProgress({
|
||||
current: 1,
|
||||
total: 1,
|
||||
progress: 100,
|
||||
status: 'complete',
|
||||
filename: 'Restarting server...',
|
||||
});
|
||||
await platform.lifecycle.restartServer(newDir);
|
||||
queryClient.invalidateQueries();
|
||||
toast({ title: 'Models moved successfully' });
|
||||
} catch (e) {
|
||||
toast({
|
||||
title: 'Migration failed',
|
||||
description: e instanceof Error ? e.message : 'Failed to migrate models',
|
||||
variant: 'destructive',
|
||||
});
|
||||
} finally {
|
||||
setMigrating(false);
|
||||
setMigrationProgress(null);
|
||||
}
|
||||
}}
|
||||
>
|
||||
Move Models
|
||||
</AlertDialogAction>
|
||||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
|
||||
{/* Migration progress overlay */}
|
||||
{migrating && migrationProgress && (
|
||||
<div className="fixed inset-0 z-50 bg-background/95 backdrop-blur-sm flex items-center justify-center">
|
||||
<div className="w-full max-w-md px-8 space-y-6 text-center">
|
||||
<div className="space-y-2">
|
||||
<Loader2 className="h-8 w-8 animate-spin mx-auto text-muted-foreground" />
|
||||
<h2 className="text-lg font-semibold">Moving models</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{migrationProgress.status === 'complete'
|
||||
? 'Restarting server...'
|
||||
: 'The server is offline while models are being moved.'}
|
||||
</p>
|
||||
</div>
|
||||
{migrationProgress.total > 0 && (
|
||||
<div className="space-y-2">
|
||||
<Progress value={migrationProgress.progress} className="h-2" />
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span className="truncate max-w-[60%]">{migrationProgress.filename}</span>
|
||||
<span>
|
||||
{formatBytes(migrationProgress.current)} /{' '}
|
||||
{formatBytes(migrationProgress.total)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface ModelItemProps {
|
||||
model: {
|
||||
model_name: string;
|
||||
display_name: string;
|
||||
downloaded: boolean;
|
||||
downloading?: boolean; // From server - true if download in progress
|
||||
size_mb?: number;
|
||||
loaded: boolean;
|
||||
};
|
||||
onDownload: () => void;
|
||||
onDelete: () => void;
|
||||
isDownloading: boolean; // Local state - true if user just clicked download
|
||||
formatSize: (sizeMb?: number) => string;
|
||||
}
|
||||
|
||||
function ModelItem({ model, onDownload, onDelete, isDownloading, formatSize }: ModelItemProps) {
|
||||
// Use server's downloading state OR local state (for immediate feedback before server updates)
|
||||
const showDownloading = model.downloading || isDownloading;
|
||||
|
||||
const statusText = model.loaded
|
||||
? 'Loaded'
|
||||
: showDownloading
|
||||
? 'Downloading'
|
||||
: model.downloaded
|
||||
? 'Downloaded'
|
||||
: 'Not downloaded';
|
||||
const sizeText =
|
||||
model.downloaded && model.size_mb && !showDownloading ? `, ${formatSize(model.size_mb)}` : '';
|
||||
const rowLabel = `${model.display_name}, ${statusText}${sizeText}. Use Tab to reach Download or Delete.`;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex items-center justify-between p-3 border rounded-lg"
|
||||
role="group"
|
||||
tabIndex={0}
|
||||
aria-label={rowLabel}
|
||||
>
|
||||
<div className="flex-1">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-medium text-sm">{model.display_name}</span>
|
||||
{model.loaded && (
|
||||
<Badge variant="default" className="text-xs">
|
||||
Loaded
|
||||
</Badge>
|
||||
)}
|
||||
{/* Only show Downloaded if actually downloaded AND not downloading */}
|
||||
{model.downloaded && !model.loaded && !showDownloading && (
|
||||
<Badge variant="secondary" className="text-xs">
|
||||
Downloaded
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
{model.downloaded && model.size_mb && !showDownloading && (
|
||||
<div className="text-xs text-muted-foreground mt-1">
|
||||
Size: {formatSize(model.size_mb)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{model.downloaded && !showDownloading ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center gap-1 text-sm text-muted-foreground">
|
||||
<span>Ready</span>
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onDelete}
|
||||
variant="outline"
|
||||
disabled={model.loaded}
|
||||
title={model.loaded ? 'Unload model before deleting' : 'Delete model'}
|
||||
aria-label={
|
||||
model.loaded ? 'Unload model before deleting' : `Delete ${model.display_name}`
|
||||
}
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
) : showDownloading ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
disabled
|
||||
aria-label={`${model.display_name} downloading`}
|
||||
>
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
Downloading...
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onDownload}
|
||||
variant="outline"
|
||||
aria-label={`Download ${model.display_name}`}
|
||||
>
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
Download
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3,14 +3,13 @@ import { Badge } from '@/components/ui/badge';
|
||||
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { useServerHealth } from '@/lib/hooks/useServer';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
import { ModelProgress } from './ModelProgress';
|
||||
|
||||
export function ServerStatus() {
|
||||
const { data: health, isLoading, error } = useServerHealth();
|
||||
const serverUrl = useServerStore((state) => state.serverUrl);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Card role="region" aria-label="Server Status" tabIndex={0}>
|
||||
<CardHeader>
|
||||
<CardTitle>Server Status</CardTitle>
|
||||
</CardHeader>
|
||||
@@ -20,16 +19,6 @@ export function ServerStatus() {
|
||||
<div className="font-mono text-sm">{serverUrl}</div>
|
||||
</div>
|
||||
|
||||
{/* Model download progress */}
|
||||
<div className="space-y-2">
|
||||
<ModelProgress modelName="qwen-tts-1.7B" displayName="Qwen TTS 1.7B" />
|
||||
<ModelProgress modelName="qwen-tts-0.6B" displayName="Qwen TTS 0.6B" />
|
||||
<ModelProgress modelName="whisper-base" displayName="Whisper Base" />
|
||||
<ModelProgress modelName="whisper-small" displayName="Whisper Small" />
|
||||
<ModelProgress modelName="whisper-medium" displayName="Whisper Medium" />
|
||||
<ModelProgress modelName="whisper-large" displayName="Whisper Large" />
|
||||
</div>
|
||||
|
||||
{isLoading ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
|
||||
@@ -20,7 +20,11 @@ export function UpdateStatus() {
|
||||
}, [platform]);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Card
|
||||
role="region"
|
||||
aria-label="App Updates"
|
||||
tabIndex={0}
|
||||
>
|
||||
<CardHeader>
|
||||
<CardTitle>App Updates</CardTitle>
|
||||
</CardHeader>
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
import { ConnectionForm } from '@/components/ServerSettings/ConnectionForm';
|
||||
import { GenerationSettings } from '@/components/ServerSettings/GenerationSettings';
|
||||
import { GpuAcceleration } from '@/components/ServerSettings/GpuAcceleration';
|
||||
import { ServerStatus } from '@/components/ServerSettings/ServerStatus';
|
||||
import { UpdateStatus } from '@/components/ServerSettings/UpdateStatus';
|
||||
import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
|
||||
export function ServerTab() {
|
||||
const platform = usePlatform();
|
||||
const isPlayerVisible = !!usePlayerStore((state) => state.audioUrl);
|
||||
return (
|
||||
<div className="space-y-4 overflow-y-auto flex flex-col">
|
||||
<div
|
||||
className={cn('overflow-y-auto flex flex-col', isPlayerVisible && BOTTOM_SAFE_AREA_PADDING)}
|
||||
>
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<ConnectionForm />
|
||||
<ServerStatus />
|
||||
<GenerationSettings />
|
||||
{platform.metadata.isTauri && <GpuAcceleration />}
|
||||
{platform.metadata.isTauri && <UpdateStatus />}
|
||||
</div>
|
||||
{platform.metadata.isTauri && <GpuAcceleration />}
|
||||
{platform.metadata.isTauri && <UpdateStatus />}
|
||||
<div className="py-8 text-center text-sm text-muted-foreground">
|
||||
Created by{' '}
|
||||
<a
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { Link, useMatchRoute } from '@tanstack/react-router';
|
||||
import { Box, BookOpen, Loader2, Mic, Server, Speaker, Volume2 } from 'lucide-react';
|
||||
import { BookOpen, Box, Mic, Server, Speaker, Volume2 } from 'lucide-react';
|
||||
import voiceboxLogo from '@/assets/voicebox-logo.png';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { version } from '../../package.json';
|
||||
|
||||
interface SidebarProps {
|
||||
isMacOS?: boolean;
|
||||
@@ -19,10 +19,8 @@ const tabs = [
|
||||
];
|
||||
|
||||
export function Sidebar({ isMacOS }: SidebarProps) {
|
||||
const isGenerating = useGenerationStore((state) => state.isGenerating);
|
||||
const audioUrl = usePlayerStore((state) => state.audioUrl);
|
||||
const isPlayerVisible = !!audioUrl;
|
||||
const matchRoute = useMatchRoute();
|
||||
const isPlayerOpen = !!usePlayerStore((s) => s.audioUrl);
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -42,9 +40,7 @@ export function Sidebar({ isMacOS }: SidebarProps) {
|
||||
const Icon = tab.icon;
|
||||
// For index route, use exact match; for others, use default matching
|
||||
const isActive =
|
||||
tab.path === '/'
|
||||
? matchRoute({ to: '/', exact: true })
|
||||
: matchRoute({ to: tab.path });
|
||||
tab.path === '/' ? matchRoute({ to: '/', exact: true }) : matchRoute({ to: tab.path });
|
||||
|
||||
return (
|
||||
<Link
|
||||
@@ -64,20 +60,13 @@ export function Sidebar({ isMacOS }: SidebarProps) {
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Spacer to push loader to bottom */}
|
||||
<div className="flex-1" />
|
||||
|
||||
{/* Generation Loader */}
|
||||
{isGenerating && (
|
||||
<div
|
||||
className={cn(
|
||||
'w-full flex items-center justify-center transition-all duration-200',
|
||||
isPlayerVisible ? 'mb-[120px]' : 'mb-0',
|
||||
)}
|
||||
>
|
||||
<Loader2 className="h-6 w-6 text-accent animate-spin" />
|
||||
</div>
|
||||
)}
|
||||
{/* Version */}
|
||||
<div
|
||||
className="mt-auto text-[10px] text-muted-foreground/50 transition-all duration-300"
|
||||
style={{ paddingBottom: isPlayerOpen ? '7rem' : undefined }}
|
||||
>
|
||||
v{version}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,8 +13,11 @@ import {
|
||||
sortableKeyboardCoordinates,
|
||||
verticalListSortingStrategy,
|
||||
} from '@dnd-kit/sortable';
|
||||
import { Link } from '@tanstack/react-router';
|
||||
import { AnimatePresence, motion } from 'framer-motion';
|
||||
import { Download, Plus } from 'lucide-react';
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import Loader from 'react-loaders';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover';
|
||||
@@ -28,6 +31,7 @@ import {
|
||||
useStory,
|
||||
} from '@/lib/hooks/useStories';
|
||||
import { useStoryPlayback } from '@/lib/hooks/useStoryPlayback';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { useStoryStore } from '@/stores/storyStore';
|
||||
import { SortableStoryChatItem } from './StoryChatItem';
|
||||
|
||||
@@ -40,6 +44,7 @@ export function StoryContent() {
|
||||
const addStoryItem = useAddStoryItem();
|
||||
const { toast } = useToast();
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
const pendingCount = useGenerationStore((s) => s.pendingGenerationIds.size);
|
||||
|
||||
// Add generation popover state
|
||||
const [searchQuery, setSearchQuery] = useState('');
|
||||
@@ -53,9 +58,9 @@ export function StoryContent() {
|
||||
const query = searchQuery.toLowerCase();
|
||||
return historyData.items.filter(
|
||||
(gen) =>
|
||||
gen.status === 'completed' &&
|
||||
!storyGenerationIds.has(gen.id) &&
|
||||
(gen.text.toLowerCase().includes(query) ||
|
||||
gen.profile_name.toLowerCase().includes(query)),
|
||||
(gen.text.toLowerCase().includes(query) || gen.profile_name.toLowerCase().includes(query)),
|
||||
);
|
||||
}, [historyData, story, searchQuery]);
|
||||
|
||||
@@ -267,7 +272,31 @@ export function StoryContent() {
|
||||
<p className="text-sm text-muted-foreground mt-1">{story.description}</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<div className="flex gap-2 items-center">
|
||||
<AnimatePresence>
|
||||
{pendingCount > 0 && (
|
||||
<motion.div
|
||||
initial={{ opacity: 0, scale: 0.9, width: 0 }}
|
||||
animate={{ opacity: 1, scale: 1, width: 'auto' }}
|
||||
exit={{ opacity: 0, scale: 0.9, width: 0 }}
|
||||
transition={{ duration: 0.2 }}
|
||||
>
|
||||
<Link
|
||||
to="/"
|
||||
className="flex items-center gap-2 h-8 pl-1.5 pr-3 rounded-full bg-card border border-border hover:bg-muted/50 transition-all duration-200 cursor-pointer"
|
||||
>
|
||||
<div className="shrink-0 w-10 h-5 overflow-hidden flex items-center justify-center">
|
||||
<div className="scale-[0.45]">
|
||||
<Loader type="line-scale" active />
|
||||
</div>
|
||||
</div>
|
||||
<span className="text-xs text-muted-foreground whitespace-nowrap">
|
||||
Generating {pendingCount} {pendingCount === 1 ? 'audio' : 'audios'}
|
||||
</span>
|
||||
</Link>
|
||||
</motion.div>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
<Popover open={isAddOpen} onOpenChange={setIsAddOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button variant="outline" size="sm">
|
||||
@@ -287,9 +316,7 @@ export function StoryContent() {
|
||||
<div className="max-h-60 overflow-y-auto">
|
||||
{availableGenerations.length === 0 ? (
|
||||
<div className="p-4 text-center text-sm text-muted-foreground">
|
||||
{searchQuery
|
||||
? 'No matching generations found'
|
||||
: 'No available generations'}
|
||||
{searchQuery ? 'No matching generations found' : 'No available generations'}
|
||||
</div>
|
||||
) : (
|
||||
availableGenerations.map((gen) => (
|
||||
|
||||
@@ -194,17 +194,29 @@ export function StoryList() {
|
||||
storyList.map((story) => (
|
||||
<div
|
||||
key={story.id}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
className={cn(
|
||||
'h-24 p-4 border rounded-2xl transition-colors group flex items-center',
|
||||
'h-24 p-4 border rounded-2xl transition-colors group flex items-center cursor-pointer',
|
||||
selectedStoryId === story.id && 'bg-muted border-primary',
|
||||
)}
|
||||
aria-label={
|
||||
selectedStoryId === story.id
|
||||
? `Story ${story.name}, ${story.item_count} ${story.item_count === 1 ? 'item' : 'items'}, ${formatDate(story.updated_at)}. Selected. Press Enter to select.`
|
||||
: `Story ${story.name}, ${story.item_count} ${story.item_count === 1 ? 'item' : 'items'}, ${formatDate(story.updated_at)}. Press Enter to select.`
|
||||
}
|
||||
aria-pressed={selectedStoryId === story.id}
|
||||
onClick={() => setSelectedStoryId(story.id)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.target !== e.currentTarget) return;
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
setSelectedStoryId(story.id);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-2 w-full min-w-0">
|
||||
<button
|
||||
type="button"
|
||||
className="flex-1 min-w-0 text-left cursor-pointer overflow-hidden"
|
||||
onClick={() => setSelectedStoryId(story.id)}
|
||||
>
|
||||
<div className="flex-1 min-w-0 text-left overflow-hidden">
|
||||
<h3 className="font-medium truncate">{story.name}</h3>
|
||||
{story.description && (
|
||||
<p className="text-sm text-muted-foreground mt-1 truncate">
|
||||
@@ -218,7 +230,7 @@ export function StoryList() {
|
||||
<span>•</span>
|
||||
<span>{formatDate(story.updated_at)}</span>
|
||||
</div>
|
||||
</button>
|
||||
</div>
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
@@ -226,6 +238,7 @@ export function StoryList() {
|
||||
size="icon"
|
||||
className="h-8 w-8 opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
aria-label={`Actions for ${story.name}`}
|
||||
>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
@@ -736,6 +736,7 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
className="h-7 w-7"
|
||||
onClick={handlePlayPause}
|
||||
title="Play/Pause (Space)"
|
||||
aria-label={isCurrentlyPlaying ? 'Pause' : 'Play'}
|
||||
>
|
||||
{isCurrentlyPlaying ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}
|
||||
</Button>
|
||||
@@ -745,6 +746,7 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
className="h-7 w-7"
|
||||
onClick={handleStop}
|
||||
disabled={!isCurrentlyPlaying}
|
||||
aria-label="Stop"
|
||||
>
|
||||
<Square className="h-3 w-3" />
|
||||
</Button>
|
||||
@@ -762,6 +764,7 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
className="h-7 w-7"
|
||||
onClick={handleSplit}
|
||||
title="Split at playhead (S)"
|
||||
aria-label="Split at playhead"
|
||||
>
|
||||
<Scissors className="h-4 w-4" />
|
||||
</Button>
|
||||
@@ -771,6 +774,7 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
className="h-7 w-7"
|
||||
onClick={handleDuplicate}
|
||||
title="Duplicate (Cmd/Ctrl+D)"
|
||||
aria-label="Duplicate clip"
|
||||
>
|
||||
<Copy className="h-4 w-4" />
|
||||
</Button>
|
||||
@@ -780,6 +784,7 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
className="h-7 w-7"
|
||||
onClick={handleDelete}
|
||||
title="Delete (Delete/Backspace)"
|
||||
aria-label="Delete clip"
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
@@ -789,10 +794,22 @@ export function StoryTrackEditor({ storyId, items }: StoryTrackEditorProps) {
|
||||
{/* Zoom controls - right side */}
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs text-muted-foreground">Zoom:</span>
|
||||
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomOut}>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-6 w-6"
|
||||
onClick={handleZoomOut}
|
||||
aria-label="Zoom out"
|
||||
>
|
||||
<Minus className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button variant="ghost" size="icon" className="h-6 w-6" onClick={handleZoomIn}>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-6 w-6"
|
||||
onClick={handleZoomIn}
|
||||
aria-label="Zoom in"
|
||||
>
|
||||
<Plus className="h-3 w-3" />
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
@@ -140,7 +140,13 @@ export function AudioSampleRecording({
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground text-center">File: {file.name}</p>
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" size="icon" variant="outline" onClick={onPlayPause}>
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
variant="outline"
|
||||
onClick={onPlayPause}
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
>
|
||||
{isPlaying ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}
|
||||
</Button>
|
||||
<Button
|
||||
|
||||
@@ -77,7 +77,13 @@ export function AudioSampleSystem({
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground text-center">File: {file.name}</p>
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" size="icon" variant="outline" onClick={onPlayPause}>
|
||||
<Button
|
||||
type="button"
|
||||
size="icon"
|
||||
variant="outline"
|
||||
onClick={onPlayPause}
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
>
|
||||
{isPlaying ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}
|
||||
</Button>
|
||||
<Button
|
||||
|
||||
@@ -110,6 +110,7 @@ export function AudioSampleUpload({
|
||||
variant="outline"
|
||||
onClick={onPlayPause}
|
||||
disabled={isValidating}
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
>
|
||||
{isPlaying ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}
|
||||
</Button>
|
||||
|
||||
@@ -61,14 +61,32 @@ export function ProfileCard({ profile }: ProfileCardProps) {
|
||||
exportProfile.mutate(profile.id);
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent) => {
|
||||
const target = e.target as HTMLElement;
|
||||
if (target.closest('button')) return;
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
handleSelect();
|
||||
}
|
||||
};
|
||||
|
||||
const selectLabel = isSelected
|
||||
? `${profile.name}, ${profile.language}. Selected as voice for generation.`
|
||||
: `${profile.name}, ${profile.language}. Select as voice for generation.`;
|
||||
|
||||
return (
|
||||
<>
|
||||
<Card
|
||||
className={cn(
|
||||
'cursor-pointer hover:shadow-md transition-all flex flex-col',
|
||||
'cursor-pointer hover:shadow-md transition-all flex flex-col h-[162px]',
|
||||
isSelected && 'ring-2 ring-primary shadow-md',
|
||||
)}
|
||||
onClick={handleSelect}
|
||||
tabIndex={0}
|
||||
role="button"
|
||||
aria-label={selectLabel}
|
||||
aria-pressed={isSelected}
|
||||
onKeyDown={handleKeyDown}
|
||||
>
|
||||
<CardHeader className="p-3 pb-2">
|
||||
<CardTitle className="flex items-center gap-1.5 text-base font-medium">
|
||||
|
||||
@@ -41,9 +41,11 @@ export function ProfileList() {
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : (
|
||||
<div className="grid gap-4 grid-cols-3 auto-rows-auto p-1 pb-[150px]">
|
||||
<div className="flex gap-4 overflow-x-auto p-1 pb-1 lg:grid lg:grid-cols-3 lg:auto-rows-auto lg:overflow-x-visible lg:pb-[150px]">
|
||||
{allProfiles.map((profile) => (
|
||||
<ProfileCard key={profile.id} profile={profile} />
|
||||
<div key={profile.id} className="shrink-0 w-[200px] lg:w-auto lg:shrink">
|
||||
<ProfileCard profile={profile} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -102,6 +102,7 @@ function MiniSamplePlayer({ audioUrl }: MiniSamplePlayerProps) {
|
||||
className="h-7 w-7 shrink-0"
|
||||
onClick={handlePlayPause}
|
||||
disabled={isLoading}
|
||||
aria-label={isPlaying ? 'Pause sample' : 'Play sample'}
|
||||
>
|
||||
{isPlaying ? <Pause className="h-3.5 w-3.5" /> : <Play className="h-3.5 w-3.5 ml-0.5" />}
|
||||
</Button>
|
||||
@@ -113,6 +114,8 @@ function MiniSamplePlayer({ audioUrl }: MiniSamplePlayerProps) {
|
||||
max={100}
|
||||
step={0.1}
|
||||
className="flex-1"
|
||||
aria-label="Sample playback position"
|
||||
aria-valuetext={`${formatAudioDuration(currentTime)} of ${formatAudioDuration(duration)}`}
|
||||
/>
|
||||
<div className="flex items-center gap-1 text-xs text-muted-foreground shrink-0 min-w-[70px]">
|
||||
<span className="font-mono">{formatAudioDuration(currentTime)}</span>
|
||||
@@ -128,6 +131,7 @@ function MiniSamplePlayer({ audioUrl }: MiniSamplePlayerProps) {
|
||||
className="h-7 w-7 shrink-0"
|
||||
onClick={handleStop}
|
||||
title="Stop"
|
||||
aria-label="Stop playback"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
|
||||
@@ -179,25 +179,36 @@ function VoiceRow({
|
||||
onDelete,
|
||||
}: VoiceRowProps) {
|
||||
const { data: samples } = useProfileSamples(profile.id);
|
||||
const sampleCount = samples?.length || 0;
|
||||
|
||||
const rowLabel = `${profile.name}, ${profile.language}, ${generationCount} generations, ${sampleCount} samples. Press Enter to edit.`;
|
||||
|
||||
return (
|
||||
<TableRow className="cursor-pointer" onClick={onEdit}>
|
||||
<TableCell>
|
||||
<div className="flex items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full min-w-0 items-center gap-2 text-left focus:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 rounded"
|
||||
aria-label={rowLabel}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onEdit();
|
||||
}}
|
||||
>
|
||||
<div className="h-8 w-8 rounded-lg bg-muted flex items-center justify-center shrink-0">
|
||||
<Mic className="h-4 w-4 text-muted-foreground" />
|
||||
</div>
|
||||
<div>
|
||||
<div className="font-medium">{profile.name}</div>
|
||||
<div className="min-w-0">
|
||||
<div className="font-medium truncate">{profile.name}</div>
|
||||
{profile.description && (
|
||||
<div className="text-sm text-muted-foreground">{profile.description}</div>
|
||||
<div className="text-sm text-muted-foreground truncate">{profile.description}</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
</TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>{profile.language}</TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>{generationCount}</TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>{samples?.length || 0}</TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>{sampleCount}</TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>
|
||||
<MultiSelect
|
||||
options={channels.map((ch) => ({
|
||||
@@ -213,7 +224,7 @@ function VoiceRow({
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon">
|
||||
<Button variant="ghost" size="icon" aria-label={`Actions for ${profile.name}`}>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import * as React from 'react';
|
||||
import { Check } from 'lucide-react';
|
||||
import * as React from 'react';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
|
||||
export interface CheckboxProps {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
@import "tailwindcss" source(".");
|
||||
@import "loaders.css/loaders.min.css";
|
||||
|
||||
@theme {
|
||||
--radius-sm: calc(var(--radius) - 4px);
|
||||
@@ -155,3 +156,18 @@
|
||||
animation: fadeIn 0.5s ease-out 0.15s forwards;
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
/* react-loaders */
|
||||
.line-scale-pulse-out-rapid > div,
|
||||
.line-scale > div {
|
||||
background-color: hsl(var(--accent)) !important;
|
||||
}
|
||||
|
||||
.loader-hidden {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.loader-hidden > div > div {
|
||||
animation-play-state: paused !important;
|
||||
background-color: hsl(var(--muted-foreground)) !important;
|
||||
}
|
||||
|
||||
@@ -200,6 +200,12 @@ class ApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
async retryGeneration(generationId: string): Promise<GenerationResponse> {
|
||||
return this.request<GenerationResponse>(`/generate/${generationId}/retry`, {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
|
||||
// History
|
||||
async listHistory(query?: HistoryQuery): Promise<HistoryListResponse> {
|
||||
const params = new URLSearchParams();
|
||||
@@ -278,6 +284,11 @@ class ApiClient {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// Generation status SSE
|
||||
getGenerationStatusUrl(generationId: string): string {
|
||||
return `${this.getBaseUrl()}/generate/${generationId}/status`;
|
||||
}
|
||||
|
||||
// Audio
|
||||
getAudioUrl(audioId: string): string {
|
||||
return `${this.getBaseUrl()}/audio/${audioId}`;
|
||||
@@ -316,6 +327,21 @@ class ApiClient {
|
||||
return this.request<ModelStatusListResponse>('/models/status');
|
||||
}
|
||||
|
||||
async getModelsCacheDir(): Promise<{ path: string }> {
|
||||
return this.request<{ path: string }>('/models/cache-dir');
|
||||
}
|
||||
|
||||
async migrateModels(destination: string): Promise<{ source: string; destination: string }> {
|
||||
return this.request('/models/migrate', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ destination }),
|
||||
});
|
||||
}
|
||||
|
||||
getMigrationProgressUrl(): string {
|
||||
return `${this.getBaseUrl()}/models/migrate/progress`;
|
||||
}
|
||||
|
||||
async triggerModelDownload(modelName: string): Promise<{ message: string }> {
|
||||
console.log(
|
||||
'[API] triggerModelDownload called for:',
|
||||
@@ -337,6 +363,12 @@ class ApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
async unloadModel(modelName: string): Promise<{ message: string }> {
|
||||
return this.request<{ message: string }>(`/models/${modelName}/unload`, {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
|
||||
async cancelDownload(modelName: string): Promise<{ message: string }> {
|
||||
return this.request<{ message: string }>('/models/download/cancel', {
|
||||
method: 'POST',
|
||||
|
||||
@@ -34,8 +34,11 @@ export interface GenerationRequest {
|
||||
language: LanguageCode;
|
||||
seed?: number;
|
||||
model_size?: '1.7B' | '0.6B';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo';
|
||||
instruct?: string;
|
||||
max_chunk_chars?: number;
|
||||
crossfade_ms?: number;
|
||||
normalize?: boolean;
|
||||
}
|
||||
|
||||
export interface GenerationResponse {
|
||||
@@ -43,9 +46,14 @@ export interface GenerationResponse {
|
||||
profile_id: string;
|
||||
text: string;
|
||||
language: string;
|
||||
audio_path: string;
|
||||
duration: number;
|
||||
audio_path?: string;
|
||||
duration?: number;
|
||||
seed?: number;
|
||||
instruct?: string;
|
||||
engine?: string;
|
||||
model_size?: string;
|
||||
status: 'generating' | 'completed' | 'failed';
|
||||
error?: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,27 +1,86 @@
|
||||
/**
|
||||
* Supported languages for voice generation.
|
||||
* Most languages use Qwen3-TTS; Hebrew uses Chatterbox TTS.
|
||||
* Supported languages for voice generation, per engine.
|
||||
*
|
||||
* Qwen3-TTS supports 10 languages.
|
||||
* LuxTTS is English-only.
|
||||
* Chatterbox Multilingual supports 23 languages.
|
||||
* Chatterbox Turbo is English-only.
|
||||
*/
|
||||
|
||||
export const SUPPORTED_LANGUAGES = {
|
||||
zh: 'Chinese',
|
||||
/** All languages that any engine supports. */
|
||||
export const ALL_LANGUAGES = {
|
||||
ar: 'Arabic',
|
||||
da: 'Danish',
|
||||
de: 'German',
|
||||
el: 'Greek',
|
||||
en: 'English',
|
||||
es: 'Spanish',
|
||||
fi: 'Finnish',
|
||||
fr: 'French',
|
||||
he: 'Hebrew',
|
||||
hi: 'Hindi',
|
||||
it: 'Italian',
|
||||
ja: 'Japanese',
|
||||
ko: 'Korean',
|
||||
de: 'German',
|
||||
fr: 'French',
|
||||
ru: 'Russian',
|
||||
ms: 'Malay',
|
||||
nl: 'Dutch',
|
||||
no: 'Norwegian',
|
||||
pl: 'Polish',
|
||||
pt: 'Portuguese',
|
||||
es: 'Spanish',
|
||||
it: 'Italian',
|
||||
he: 'Hebrew',
|
||||
ru: 'Russian',
|
||||
sv: 'Swedish',
|
||||
sw: 'Swahili',
|
||||
tr: 'Turkish',
|
||||
zh: 'Chinese',
|
||||
} as const;
|
||||
|
||||
export type LanguageCode = keyof typeof SUPPORTED_LANGUAGES;
|
||||
export type LanguageCode = keyof typeof ALL_LANGUAGES;
|
||||
|
||||
export const LANGUAGE_CODES = Object.keys(SUPPORTED_LANGUAGES) as LanguageCode[];
|
||||
/** Per-engine supported language codes. */
|
||||
export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
|
||||
qwen: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'],
|
||||
luxtts: ['en'],
|
||||
chatterbox: [
|
||||
'ar',
|
||||
'da',
|
||||
'de',
|
||||
'el',
|
||||
'en',
|
||||
'es',
|
||||
'fi',
|
||||
'fr',
|
||||
'he',
|
||||
'hi',
|
||||
'it',
|
||||
'ja',
|
||||
'ko',
|
||||
'ms',
|
||||
'nl',
|
||||
'no',
|
||||
'pl',
|
||||
'pt',
|
||||
'ru',
|
||||
'sv',
|
||||
'sw',
|
||||
'tr',
|
||||
'zh',
|
||||
],
|
||||
chatterbox_turbo: ['en'],
|
||||
} as const;
|
||||
|
||||
/** Helper: get language options for a given engine. */
|
||||
export function getLanguageOptionsForEngine(engine: string) {
|
||||
const codes = ENGINE_LANGUAGES[engine] ?? ENGINE_LANGUAGES.qwen;
|
||||
return codes.map((code) => ({
|
||||
value: code,
|
||||
label: ALL_LANGUAGES[code],
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Backwards-compatible exports used elsewhere ──────────────────────
|
||||
export const SUPPORTED_LANGUAGES = ALL_LANGUAGES;
|
||||
export const LANGUAGE_CODES = Object.keys(ALL_LANGUAGES) as LanguageCode[];
|
||||
export const LANGUAGE_OPTIONS = LANGUAGE_CODES.map((code) => ({
|
||||
value: code,
|
||||
label: SUPPORTED_LANGUAGES[code],
|
||||
label: ALL_LANGUAGES[code],
|
||||
}));
|
||||
|
||||
@@ -8,15 +8,15 @@ import { LANGUAGE_CODES, type LanguageCode } from '@/lib/constants/languages';
|
||||
import { useGeneration } from '@/lib/hooks/useGeneration';
|
||||
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
const generationSchema = z.object({
|
||||
text: z.string().min(1, 'Text is required').max(5000),
|
||||
text: z.string().min(1, 'Text is required').max(50000),
|
||||
language: z.enum(LANGUAGE_CODES as [LanguageCode, ...LanguageCode[]]),
|
||||
seed: z.number().int().optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B']).optional(),
|
||||
instruct: z.string().max(500).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox']).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo']).optional(),
|
||||
});
|
||||
|
||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||
@@ -29,8 +29,10 @@ interface UseGenerationFormOptions {
|
||||
export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
const { toast } = useToast();
|
||||
const generation = useGeneration();
|
||||
const setAudioWithAutoPlay = usePlayerStore((state) => state.setAudioWithAutoPlay);
|
||||
const setIsGenerating = useGenerationStore((state) => state.setIsGenerating);
|
||||
const addPendingGeneration = useGenerationStore((state) => state.addPendingGeneration);
|
||||
const maxChunkChars = useServerStore((state) => state.maxChunkChars);
|
||||
const crossfadeMs = useServerStore((state) => state.crossfadeMs);
|
||||
const normalizeAudio = useServerStore((state) => state.normalizeAudio);
|
||||
const [downloadingModelName, setDownloadingModelName] = useState<string | null>(null);
|
||||
const [downloadingDisplayName, setDownloadingDisplayName] = useState<string | null>(null);
|
||||
|
||||
@@ -67,24 +69,27 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
}
|
||||
|
||||
try {
|
||||
setIsGenerating(true);
|
||||
|
||||
const engine = data.engine || 'qwen';
|
||||
const modelName =
|
||||
engine === 'luxtts'
|
||||
? 'luxtts'
|
||||
: engine === 'chatterbox'
|
||||
? 'chatterbox-tts'
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'chatterbox-turbo'
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
const displayName =
|
||||
engine === 'luxtts'
|
||||
? 'LuxTTS'
|
||||
: engine === 'chatterbox'
|
||||
? 'Chatterbox TTS'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
: engine === 'chatterbox_turbo'
|
||||
? 'Chatterbox Turbo'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
|
||||
// Check if model needs downloading
|
||||
try {
|
||||
const modelStatus = await apiClient.getModelStatus();
|
||||
const model = modelStatus.models.find((m) => m.model_name === modelName);
|
||||
@@ -98,6 +103,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
}
|
||||
|
||||
const isQwen = engine === 'qwen';
|
||||
// This now returns immediately with status="generating"
|
||||
const result = await generation.mutateAsync({
|
||||
profile_id: selectedProfileId,
|
||||
text: data.text,
|
||||
@@ -106,16 +112,15 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
model_size: isQwen ? data.modelSize : undefined,
|
||||
engine,
|
||||
instruct: isQwen ? data.instruct || undefined : undefined,
|
||||
max_chunk_chars: maxChunkChars,
|
||||
crossfade_ms: crossfadeMs,
|
||||
normalize: normalizeAudio,
|
||||
});
|
||||
|
||||
toast({
|
||||
title: 'Generation complete!',
|
||||
description: `Audio generated (${result.duration.toFixed(2)}s)`,
|
||||
});
|
||||
|
||||
const audioUrl = apiClient.getAudioUrl(result.id);
|
||||
setAudioWithAutoPlay(audioUrl, result.id, selectedProfileId, data.text.substring(0, 50));
|
||||
// Track this generation for SSE status updates
|
||||
addPendingGeneration(result.id);
|
||||
|
||||
// Reset form immediately — user can start typing again
|
||||
form.reset({
|
||||
text: '',
|
||||
language: data.language,
|
||||
@@ -132,7 +137,6 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
variant: 'destructive',
|
||||
});
|
||||
} finally {
|
||||
setIsGenerating(false);
|
||||
setDownloadingModelName(null);
|
||||
setDownloadingDisplayName(null);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useToast } from '@/components/ui/use-toast';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
interface GenerationStatusEvent {
|
||||
id: string;
|
||||
status: 'generating' | 'completed' | 'failed' | 'not_found';
|
||||
duration?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribes to SSE for all pending generations. When a generation completes,
|
||||
* invalidates the history query, removes it from pending, and auto-plays
|
||||
* if the player is idle.
|
||||
*/
|
||||
export function useGenerationProgress() {
|
||||
const queryClient = useQueryClient();
|
||||
const { toast } = useToast();
|
||||
const pendingIds = useGenerationStore((s) => s.pendingGenerationIds);
|
||||
const removePendingGeneration = useGenerationStore((s) => s.removePendingGeneration);
|
||||
const removePendingStoryAdd = useGenerationStore((s) => s.removePendingStoryAdd);
|
||||
const isPlaying = usePlayerStore((s) => s.isPlaying);
|
||||
const setAudioWithAutoPlay = usePlayerStore((s) => s.setAudioWithAutoPlay);
|
||||
const autoplayOnGenerate = useServerStore((s) => s.autoplayOnGenerate);
|
||||
|
||||
// Keep refs to avoid stale closures in EventSource handlers
|
||||
const isPlayingRef = useRef(isPlaying);
|
||||
const autoplayRef = useRef(autoplayOnGenerate);
|
||||
isPlayingRef.current = isPlaying;
|
||||
autoplayRef.current = autoplayOnGenerate;
|
||||
|
||||
// Track active EventSource instances
|
||||
const eventSourcesRef = useRef<Map<string, EventSource>>(new Map());
|
||||
|
||||
// Unmount-only cleanup — close all SSE connections when the hook is torn down
|
||||
useEffect(() => {
|
||||
const sources = eventSourcesRef.current;
|
||||
return () => {
|
||||
for (const source of sources.values()) {
|
||||
source.close();
|
||||
}
|
||||
sources.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const currentSources = eventSourcesRef.current;
|
||||
|
||||
// Close SSE connections for IDs no longer pending
|
||||
for (const [id, source] of currentSources.entries()) {
|
||||
if (!pendingIds.has(id)) {
|
||||
source.close();
|
||||
currentSources.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Open SSE connections for new pending IDs
|
||||
for (const id of pendingIds) {
|
||||
if (currentSources.has(id)) continue;
|
||||
|
||||
const url = apiClient.getGenerationStatusUrl(id);
|
||||
const source = new EventSource(url);
|
||||
|
||||
source.onmessage = (event) => {
|
||||
try {
|
||||
const data: GenerationStatusEvent = JSON.parse(event.data);
|
||||
|
||||
if (data.status === 'completed') {
|
||||
source.close();
|
||||
currentSources.delete(id);
|
||||
removePendingGeneration(id);
|
||||
|
||||
// Refresh history to pick up the completed generation
|
||||
queryClient.invalidateQueries({ queryKey: ['history'] });
|
||||
|
||||
// If this generation was queued for a story, add it now
|
||||
const storyId = removePendingStoryAdd(id);
|
||||
if (storyId) {
|
||||
apiClient
|
||||
.addStoryItem(storyId, { generation_id: id })
|
||||
.then(() => {
|
||||
queryClient.invalidateQueries({ queryKey: ['stories'] });
|
||||
queryClient.invalidateQueries({ queryKey: ['stories', storyId] });
|
||||
toast({
|
||||
title: 'Added to story',
|
||||
description: data.duration
|
||||
? `Audio generated (${data.duration.toFixed(2)}s) and added to story`
|
||||
: 'Audio generated and added to story',
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
toast({
|
||||
title: 'Generation complete',
|
||||
description: 'Audio generated but failed to add to story',
|
||||
variant: 'destructive',
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// toast({
|
||||
// title: 'Generation complete!',
|
||||
// description: data.duration
|
||||
// ? `Audio generated (${data.duration.toFixed(2)}s)`
|
||||
// : 'Audio generated',
|
||||
// });
|
||||
}
|
||||
|
||||
// Auto-play if enabled and nothing is currently playing
|
||||
if (autoplayRef.current && !isPlayingRef.current) {
|
||||
const genAudioUrl = apiClient.getAudioUrl(id);
|
||||
setAudioWithAutoPlay(genAudioUrl, id, '', '');
|
||||
}
|
||||
} else if (data.status === 'failed' || data.status === 'not_found') {
|
||||
source.close();
|
||||
currentSources.delete(id);
|
||||
removePendingGeneration(id);
|
||||
removePendingStoryAdd(id);
|
||||
|
||||
queryClient.invalidateQueries({ queryKey: ['history'] });
|
||||
|
||||
toast({
|
||||
title: data.status === 'not_found' ? 'Generation not found' : 'Generation failed',
|
||||
description: data.error || 'An error occurred during generation',
|
||||
variant: 'destructive',
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Ignore parse errors from heartbeats etc
|
||||
}
|
||||
};
|
||||
|
||||
source.onerror = () => {
|
||||
// EventSource auto-reconnects, but if we get repeated errors
|
||||
// just clean up
|
||||
source.close();
|
||||
currentSources.delete(id);
|
||||
removePendingGeneration(id);
|
||||
};
|
||||
|
||||
currentSources.set(id, source);
|
||||
}
|
||||
}, [
|
||||
pendingIds,
|
||||
removePendingGeneration,
|
||||
removePendingStoryAdd,
|
||||
queryClient,
|
||||
toast,
|
||||
setAudioWithAutoPlay,
|
||||
]);
|
||||
}
|
||||
@@ -1,23 +1,23 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
import type { ActiveDownloadTask } from '@/lib/api/types';
|
||||
import { useGenerationStore } from '@/stores/generationStore';
|
||||
|
||||
// Polling interval in milliseconds
|
||||
const POLL_INTERVAL = 2000;
|
||||
const POLL_INTERVAL = 30000;
|
||||
|
||||
/**
|
||||
* Hook to monitor active tasks (downloads and generations).
|
||||
* Polls the server periodically to catch downloads triggered from anywhere
|
||||
* (transcription, generation, explicit download, etc.).
|
||||
*
|
||||
*
|
||||
* Returns the active downloads so components can render download toasts.
|
||||
*/
|
||||
export function useRestoreActiveTasks() {
|
||||
const [activeDownloads, setActiveDownloads] = useState<ActiveDownloadTask[]>([]);
|
||||
const setIsGenerating = useGenerationStore((state) => state.setIsGenerating);
|
||||
const setActiveGenerationId = useGenerationStore((state) => state.setActiveGenerationId);
|
||||
|
||||
const addPendingGeneration = useGenerationStore((state) => state.addPendingGeneration);
|
||||
|
||||
// Track which downloads we've seen to detect new ones
|
||||
const seenDownloadsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
@@ -25,15 +25,15 @@ export function useRestoreActiveTasks() {
|
||||
try {
|
||||
const tasks = await apiClient.getActiveTasks();
|
||||
|
||||
// Update generation state
|
||||
// Restore pending generations (e.g., after page refresh)
|
||||
if (tasks.generations.length > 0) {
|
||||
setIsGenerating(true);
|
||||
setActiveGenerationId(tasks.generations[0].task_id);
|
||||
for (const gen of tasks.generations) {
|
||||
addPendingGeneration(gen.task_id);
|
||||
}
|
||||
} else {
|
||||
// Only clear if we were tracking a generation
|
||||
const currentId = useGenerationStore.getState().activeGenerationId;
|
||||
if (currentId) {
|
||||
setIsGenerating(false);
|
||||
setActiveGenerationId(null);
|
||||
}
|
||||
}
|
||||
@@ -41,14 +41,14 @@ export function useRestoreActiveTasks() {
|
||||
// Update active downloads
|
||||
// Keep track of all active downloads (including new ones)
|
||||
const currentDownloadNames = new Set(tasks.downloads.map((d) => d.model_name));
|
||||
|
||||
|
||||
// Remove completed downloads from our seen set
|
||||
for (const name of seenDownloadsRef.current) {
|
||||
if (!currentDownloadNames.has(name)) {
|
||||
seenDownloadsRef.current.delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Add new downloads to seen set
|
||||
for (const download of tasks.downloads) {
|
||||
seenDownloadsRef.current.add(download.model_name);
|
||||
@@ -59,7 +59,7 @@ export function useRestoreActiveTasks() {
|
||||
// Silently fail - server might be temporarily unavailable
|
||||
console.debug('Failed to fetch active tasks:', error);
|
||||
}
|
||||
}, [setIsGenerating, setActiveGenerationId]);
|
||||
}, [setActiveGenerationId, addPendingGeneration]);
|
||||
|
||||
useEffect(() => {
|
||||
// Fetch immediately on mount
|
||||
|
||||
@@ -21,10 +21,25 @@ export function formatDate(date: string | Date): string {
|
||||
} else {
|
||||
dateObj = date;
|
||||
}
|
||||
|
||||
|
||||
return formatDistance(dateObj, new Date(), { addSuffix: true }).replace(/^about /i, '');
|
||||
}
|
||||
|
||||
const ENGINE_DISPLAY_NAMES: Record<string, string> = {
|
||||
qwen: 'Qwen',
|
||||
luxtts: 'LuxTTS',
|
||||
chatterbox: 'Chatterbox',
|
||||
chatterbox_turbo: 'Chatterbox Turbo',
|
||||
};
|
||||
|
||||
export function formatEngineName(engine?: string, modelSize?: string): string {
|
||||
const name = ENGINE_DISPLAY_NAMES[engine ?? 'qwen'] ?? engine ?? 'Qwen';
|
||||
if (engine === 'qwen' && modelSize) {
|
||||
return `${name} ${modelSize}`;
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
export function formatFileSize(bytes: number): string {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
|
||||
@@ -49,9 +49,9 @@ export interface PlatformAudio {
|
||||
}
|
||||
|
||||
export interface PlatformLifecycle {
|
||||
startServer(remote?: boolean): Promise<string>;
|
||||
startServer(remote?: boolean, modelsDir?: string | null): Promise<string>;
|
||||
stopServer(): Promise<void>;
|
||||
restartServer(): Promise<string>;
|
||||
restartServer(modelsDir?: string | null): Promise<string>;
|
||||
setKeepServerRunning(keep: boolean): Promise<void>;
|
||||
setupWindowCloseHandler(): Promise<void>;
|
||||
onServerReady?: () => void;
|
||||
|
||||
@@ -8,8 +8,10 @@ import { Sidebar } from '@/components/Sidebar';
|
||||
import { StoriesTab } from '@/components/StoriesTab/StoriesTab';
|
||||
import { Toaster } from '@/components/ui/toaster';
|
||||
import { VoicesTab } from '@/components/VoicesTab/VoicesTab';
|
||||
import { useGenerationProgress } from '@/lib/hooks/useGenerationProgress';
|
||||
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
|
||||
import { MODEL_DISPLAY_NAMES, useRestoreActiveTasks } from '@/lib/hooks/useRestoreActiveTasks';
|
||||
|
||||
// Simple platform check that works in both web and Tauri
|
||||
const isMacOS = () => navigator.platform.toLowerCase().includes('mac');
|
||||
|
||||
@@ -18,6 +20,9 @@ function RootLayout() {
|
||||
// Monitor active downloads/generations and show toasts for them
|
||||
const activeDownloads = useRestoreActiveTasks();
|
||||
|
||||
// Subscribe to SSE for pending generations — handles completion, auto-play, and history refresh
|
||||
useGenerationProgress();
|
||||
|
||||
return (
|
||||
<AppFrame>
|
||||
<div className="flex flex-1 min-h-0 overflow-hidden">
|
||||
|
||||
@@ -1,15 +1,58 @@
|
||||
import { create } from 'zustand';
|
||||
|
||||
interface GenerationState {
|
||||
/** IDs of generations currently in progress */
|
||||
pendingGenerationIds: Set<string>;
|
||||
/** Whether any generation is in progress (derived from pendingGenerationIds) */
|
||||
isGenerating: boolean;
|
||||
activeGenerationId: string | null;
|
||||
setIsGenerating: (generating: boolean) => void;
|
||||
/** Map of generationId → storyId for deferred story additions */
|
||||
pendingStoryAdds: Map<string, string>;
|
||||
addPendingGeneration: (id: string) => void;
|
||||
removePendingGeneration: (id: string) => void;
|
||||
addPendingStoryAdd: (generationId: string, storyId: string) => void;
|
||||
removePendingStoryAdd: (generationId: string) => string | undefined;
|
||||
setActiveGenerationId: (id: string | null) => void;
|
||||
activeGenerationId: string | null;
|
||||
}
|
||||
|
||||
export const useGenerationStore = create<GenerationState>((set) => ({
|
||||
export const useGenerationStore = create<GenerationState>((set, get) => ({
|
||||
pendingGenerationIds: new Set(),
|
||||
isGenerating: false,
|
||||
activeGenerationId: null,
|
||||
setIsGenerating: (generating) => set({ isGenerating: generating }),
|
||||
pendingStoryAdds: new Map(),
|
||||
|
||||
addPendingGeneration: (id) =>
|
||||
set((state) => {
|
||||
const next = new Set(state.pendingGenerationIds);
|
||||
next.add(id);
|
||||
return { pendingGenerationIds: next, isGenerating: true };
|
||||
}),
|
||||
|
||||
removePendingGeneration: (id) =>
|
||||
set((state) => {
|
||||
const next = new Set(state.pendingGenerationIds);
|
||||
next.delete(id);
|
||||
return { pendingGenerationIds: next, isGenerating: next.size > 0 };
|
||||
}),
|
||||
|
||||
addPendingStoryAdd: (generationId, storyId) =>
|
||||
set((state) => {
|
||||
const next = new Map(state.pendingStoryAdds);
|
||||
next.set(generationId, storyId);
|
||||
return { pendingStoryAdds: next };
|
||||
}),
|
||||
|
||||
removePendingStoryAdd: (generationId) => {
|
||||
const storyId = get().pendingStoryAdds.get(generationId);
|
||||
if (storyId) {
|
||||
set((state) => {
|
||||
const next = new Map(state.pendingStoryAdds);
|
||||
next.delete(generationId);
|
||||
return { pendingStoryAdds: next };
|
||||
});
|
||||
}
|
||||
return storyId;
|
||||
},
|
||||
|
||||
setActiveGenerationId: (id) => set({ activeGenerationId: id }),
|
||||
}));
|
||||
|
||||
@@ -13,6 +13,21 @@ interface ServerStore {
|
||||
|
||||
keepServerRunningOnClose: boolean;
|
||||
setKeepServerRunningOnClose: (keepRunning: boolean) => void;
|
||||
|
||||
maxChunkChars: number;
|
||||
setMaxChunkChars: (value: number) => void;
|
||||
|
||||
crossfadeMs: number;
|
||||
setCrossfadeMs: (value: number) => void;
|
||||
|
||||
normalizeAudio: boolean;
|
||||
setNormalizeAudio: (value: boolean) => void;
|
||||
|
||||
autoplayOnGenerate: boolean;
|
||||
setAutoplayOnGenerate: (value: boolean) => void;
|
||||
|
||||
customModelsDir: string | null;
|
||||
setCustomModelsDir: (dir: string | null) => void;
|
||||
}
|
||||
|
||||
export const useServerStore = create<ServerStore>()(
|
||||
@@ -29,6 +44,21 @@ export const useServerStore = create<ServerStore>()(
|
||||
|
||||
keepServerRunningOnClose: false,
|
||||
setKeepServerRunningOnClose: (keepRunning) => set({ keepServerRunningOnClose: keepRunning }),
|
||||
|
||||
maxChunkChars: 800,
|
||||
setMaxChunkChars: (value) => set({ maxChunkChars: value }),
|
||||
|
||||
crossfadeMs: 50,
|
||||
setCrossfadeMs: (value) => set({ crossfadeMs: value }),
|
||||
|
||||
normalizeAudio: true,
|
||||
setNormalizeAudio: (value) => set({ normalizeAudio: value }),
|
||||
|
||||
autoplayOnGenerate: true,
|
||||
setAutoplayOnGenerate: (value) => set({ autoplayOnGenerate: value }),
|
||||
|
||||
customModelsDir: null,
|
||||
setCustomModelsDir: (dir) => set({ customModelsDir: dir }),
|
||||
}),
|
||||
{
|
||||
name: 'voicebox-server',
|
||||
|
||||
@@ -122,6 +122,7 @@ TTS_ENGINES = {
|
||||
"qwen": "Qwen TTS",
|
||||
"luxtts": "LuxTTS",
|
||||
"chatterbox": "Chatterbox TTS",
|
||||
"chatterbox_turbo": "Chatterbox Turbo",
|
||||
}
|
||||
|
||||
|
||||
@@ -171,6 +172,9 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
elif engine == "chatterbox":
|
||||
from .chatterbox_backend import ChatterboxTTSBackend
|
||||
backend = ChatterboxTTSBackend()
|
||||
elif engine == "chatterbox_turbo":
|
||||
from .chatterbox_turbo_backend import ChatterboxTurboTTSBackend
|
||||
backend = ChatterboxTurboTTSBackend()
|
||||
else:
|
||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||
|
||||
|
||||
@@ -136,6 +136,10 @@ class ChatterboxTTSBackend:
|
||||
import torch
|
||||
from chatterbox.mtl_tts import ChatterboxMultilingualTTS
|
||||
|
||||
# Load into a local variable first, apply all patches, then
|
||||
# assign to self.model. This avoids leaving a half-initialised
|
||||
# model on self.model if any patch step raises an exception.
|
||||
#
|
||||
# Monkey-patch torch.load for CPU loading. The model's .pt files
|
||||
# were saved on CUDA; from_pretrained() doesn't pass map_location
|
||||
# so loading on CPU fails without this.
|
||||
@@ -150,13 +154,13 @@ class ChatterboxTTSBackend:
|
||||
with ChatterboxTTSBackend._load_lock:
|
||||
torch.load = _patched_load
|
||||
try:
|
||||
self.model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
device=device,
|
||||
)
|
||||
finally:
|
||||
torch.load = _orig_torch_load
|
||||
else:
|
||||
self.model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
model = ChatterboxMultilingualTTS.from_pretrained(
|
||||
device=device,
|
||||
)
|
||||
finally:
|
||||
@@ -165,7 +169,7 @@ class ChatterboxTTSBackend:
|
||||
# Fix: transformers >= 4.36 defaults LlamaModel to sdpa attention
|
||||
# which doesn't support output_attentions=True (needed by
|
||||
# Chatterbox's AlignmentStreamAnalyzer). Force eager attention.
|
||||
t3_tfmr = self.model.t3.tfmr
|
||||
t3_tfmr = model.t3.tfmr
|
||||
if hasattr(t3_tfmr, "config") and hasattr(
|
||||
t3_tfmr.config, "_attn_implementation"
|
||||
):
|
||||
@@ -178,6 +182,36 @@ class ChatterboxTTSBackend:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
# Patch float64 → float32 dtype mismatches in upstream chatterbox.
|
||||
# librosa.load returns float64 numpy; multiple upstream code paths
|
||||
# convert it to a torch tensor via torch.from_numpy() without
|
||||
# casting, then matmul it against float32 model weights.
|
||||
import types
|
||||
|
||||
# Patch S3Tokenizer (used by s3gen.tokenizer)
|
||||
_tokzr = model.s3gen.tokenizer
|
||||
_orig_log_mel = _tokzr.log_mel_spectrogram.__func__
|
||||
|
||||
def _f32_log_mel(self_tokzr, audio, padding=0):
|
||||
import torch as _torch
|
||||
if _torch.is_tensor(audio):
|
||||
audio = audio.float()
|
||||
return _orig_log_mel(self_tokzr, audio, padding)
|
||||
|
||||
_tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr)
|
||||
|
||||
# Patch VoiceEncoder
|
||||
_ve = model.ve
|
||||
_orig_ve_forward = _ve.forward.__func__
|
||||
|
||||
def _f32_ve_forward(self_ve, mels):
|
||||
return _orig_ve_forward(self_ve, mels.float())
|
||||
|
||||
_ve.forward = types.MethodType(_f32_ve_forward, _ve)
|
||||
|
||||
# All patches applied successfully — publish the model
|
||||
self.model = model
|
||||
|
||||
logger.info("Chatterbox Multilingual TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
"""
|
||||
Chatterbox Turbo TTS backend implementation.
|
||||
|
||||
Wraps ChatterboxTurboTTS from chatterbox-tts for fast, English-only
|
||||
voice cloning with paralinguistic tag support ([laugh], [cough], etc.).
|
||||
Forces CPU on macOS due to known MPS tensor issues.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import platform
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import TTSBackend
|
||||
from ..utils.audio import normalize_audio, load_audio
|
||||
from ..utils.progress import get_progress_manager
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHATTERBOX_TURBO_HF_REPO = "ResembleAI/chatterbox-turbo"
|
||||
|
||||
# Files that must be present for the turbo model
|
||||
_TURBO_WEIGHT_FILES = [
|
||||
"t3_turbo_v1.safetensors",
|
||||
"s3gen_meanflow.safetensors",
|
||||
"ve.safetensors",
|
||||
]
|
||||
|
||||
|
||||
class ChatterboxTurboTTSBackend:
|
||||
"""Chatterbox Turbo TTS backend — fast, English-only, with paralinguistic tags."""
|
||||
|
||||
# Class-level lock for torch.load monkey-patching
|
||||
_load_lock: ClassVar[threading.Lock] = threading.Lock()
|
||||
|
||||
def __init__(self):
|
||||
self.model = None
|
||||
self.model_size = "default"
|
||||
self._device = None
|
||||
self._model_load_lock = asyncio.Lock()
|
||||
|
||||
def _get_device(self) -> str:
|
||||
"""Get the best available device. Forces CPU on macOS (MPS issue)."""
|
||||
if platform.system() == "Darwin":
|
||||
return "cpu"
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
return "cuda"
|
||||
except ImportError:
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
return self.model is not None
|
||||
|
||||
def _get_model_path(self, model_size: str = "default") -> str:
|
||||
return CHATTERBOX_TURBO_HF_REPO
|
||||
|
||||
def _is_model_cached(self, model_size: str = "default") -> bool:
|
||||
"""Check if the Chatterbox Turbo model is cached locally."""
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
repo_cache = Path(hf_constants.HF_HUB_CACHE) / (
|
||||
"models--" + CHATTERBOX_TURBO_HF_REPO.replace("/", "--")
|
||||
)
|
||||
|
||||
if not repo_cache.exists():
|
||||
return False
|
||||
|
||||
blobs_dir = repo_cache / "blobs"
|
||||
if blobs_dir.exists() and any(blobs_dir.glob("*.incomplete")):
|
||||
return False
|
||||
|
||||
# Check for turbo weight files
|
||||
snapshots_dir = repo_cache / "snapshots"
|
||||
if snapshots_dir.exists():
|
||||
for fname in _TURBO_WEIGHT_FILES:
|
||||
if not any(snapshots_dir.rglob(fname)):
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking Chatterbox Turbo cache: {e}")
|
||||
return False
|
||||
|
||||
async def load_model(self, model_size: str = "default") -> None:
|
||||
"""Load the Chatterbox Turbo model."""
|
||||
if self.model is not None:
|
||||
return
|
||||
async with self._model_load_lock:
|
||||
if self.model is not None:
|
||||
return
|
||||
await asyncio.to_thread(self._load_model_sync)
|
||||
|
||||
def _load_model_sync(self):
|
||||
"""Synchronous model loading."""
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
task_manager = get_task_manager()
|
||||
model_name = "chatterbox-turbo"
|
||||
|
||||
is_cached = self._is_model_cached()
|
||||
|
||||
# Set up HF progress tracking (intercepts tqdm for file-level progress)
|
||||
progress_callback = create_hf_progress_callback(model_name, progress_manager)
|
||||
tracker = HFProgressTracker(progress_callback, filter_non_downloads=is_cached)
|
||||
tracker_context = tracker.patch_download()
|
||||
tracker_context.__enter__()
|
||||
|
||||
if not is_cached:
|
||||
task_manager.start_download(model_name)
|
||||
progress_manager.update_progress(
|
||||
model_name=model_name,
|
||||
current=0,
|
||||
total=0,
|
||||
filename="Connecting to HuggingFace...",
|
||||
status="downloading",
|
||||
)
|
||||
|
||||
try:
|
||||
device = self._get_device()
|
||||
self._device = device
|
||||
|
||||
logger.info(f"Loading Chatterbox Turbo TTS on {device}...")
|
||||
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from chatterbox.tts_turbo import ChatterboxTurboTTS
|
||||
|
||||
# Download model files ourselves so we can pass token=None
|
||||
# (upstream from_pretrained passes token=True which requires
|
||||
# a stored HF token even though the repo is public).
|
||||
try:
|
||||
local_path = snapshot_download(
|
||||
repo_id=CHATTERBOX_TURBO_HF_REPO,
|
||||
token=None,
|
||||
allow_patterns=[
|
||||
"*.safetensors", "*.json", "*.txt", "*.pt", "*.model",
|
||||
],
|
||||
)
|
||||
finally:
|
||||
tracker_context.__exit__(None, None, None)
|
||||
|
||||
# Monkey-patch torch.load for CPU loading. The model's .pt files
|
||||
# were saved on CUDA; from_local() doesn't pass map_location
|
||||
# so loading on CPU fails without this.
|
||||
# Load into a local var, apply patches, then publish to
|
||||
# self.model so a failed patch doesn't leave us half-initialised.
|
||||
if device == "cpu":
|
||||
_orig_torch_load = torch.load
|
||||
|
||||
def _patched_load(*args, **kwargs):
|
||||
kwargs.setdefault("map_location", "cpu")
|
||||
return _orig_torch_load(*args, **kwargs)
|
||||
|
||||
with ChatterboxTurboTTSBackend._load_lock:
|
||||
torch.load = _patched_load
|
||||
try:
|
||||
model = ChatterboxTurboTTS.from_local(
|
||||
local_path, device,
|
||||
)
|
||||
finally:
|
||||
torch.load = _orig_torch_load
|
||||
else:
|
||||
model = ChatterboxTurboTTS.from_local(
|
||||
local_path, device,
|
||||
)
|
||||
|
||||
if not is_cached:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
# Patch float64 → float32 dtype mismatches in upstream chatterbox.
|
||||
# librosa.load returns float64 numpy; multiple upstream code paths
|
||||
# convert it to a torch tensor via torch.from_numpy() without
|
||||
# casting, then matmul it against float32 model weights.
|
||||
# We patch the two known entry points:
|
||||
#
|
||||
# 1. S3Tokenizer.log_mel_spectrogram — the audio tensor from
|
||||
# librosa hits _mel_filters (float32) in a matmul.
|
||||
# 2. VoiceEncoder.forward — float64 mel spectrograms hit the
|
||||
# float32 LSTM weights.
|
||||
import types
|
||||
|
||||
# Patch S3Tokenizer (used by s3gen.tokenizer)
|
||||
_tokzr = model.s3gen.tokenizer
|
||||
_orig_log_mel = _tokzr.log_mel_spectrogram.__func__
|
||||
|
||||
def _f32_log_mel(self_tokzr, audio, padding=0):
|
||||
import torch as _torch
|
||||
if _torch.is_tensor(audio):
|
||||
audio = audio.float()
|
||||
return _orig_log_mel(self_tokzr, audio, padding)
|
||||
|
||||
_tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr)
|
||||
|
||||
# Patch VoiceEncoder
|
||||
_ve = model.ve
|
||||
_orig_ve_forward = _ve.forward.__func__
|
||||
|
||||
def _f32_ve_forward(self_ve, mels):
|
||||
return _orig_ve_forward(self_ve, mels.float())
|
||||
|
||||
_ve.forward = types.MethodType(_f32_ve_forward, _ve)
|
||||
|
||||
# Only publish after all patches succeed
|
||||
self.model = model
|
||||
|
||||
logger.info("Chatterbox Turbo TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"chatterbox-tts package not found. "
|
||||
"Install with: pip install chatterbox-tts"
|
||||
)
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load Chatterbox Turbo: {e}")
|
||||
if not is_cached:
|
||||
progress_manager.mark_error(model_name, str(e))
|
||||
task_manager.error_download(model_name, str(e))
|
||||
raise
|
||||
|
||||
def unload_model(self) -> None:
|
||||
"""Unload model to free memory."""
|
||||
if self.model is not None:
|
||||
device = self._device
|
||||
del self.model
|
||||
self.model = None
|
||||
self._device = None
|
||||
if device == "cuda":
|
||||
import torch
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
logger.info("Chatterbox Turbo unloaded")
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
reference_text: str,
|
||||
use_cache: bool = True,
|
||||
) -> Tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt from reference audio.
|
||||
|
||||
Chatterbox Turbo processes reference audio at generation time, so the
|
||||
prompt just stores the file path.
|
||||
"""
|
||||
voice_prompt = {
|
||||
"ref_audio": str(audio_path),
|
||||
"ref_text": reference_text,
|
||||
}
|
||||
return voice_prompt, False
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: List[str],
|
||||
reference_texts: List[str],
|
||||
) -> Tuple[np.ndarray, str]:
|
||||
"""Combine multiple reference samples."""
|
||||
combined_audio = []
|
||||
for path in audio_paths:
|
||||
audio, _sr = load_audio(path)
|
||||
audio = normalize_audio(audio)
|
||||
combined_audio.append(audio)
|
||||
|
||||
mixed = np.concatenate(combined_audio)
|
||||
mixed = normalize_audio(mixed)
|
||||
combined_text = " ".join(reference_texts)
|
||||
return mixed, combined_text
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: Optional[int] = None,
|
||||
instruct: Optional[str] = None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio using Chatterbox Turbo TTS.
|
||||
|
||||
Supports paralinguistic tags in text: [laugh], [cough], [chuckle], etc.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize (may include paralinguistic tags)
|
||||
voice_prompt: Dict with ref_audio path
|
||||
language: Ignored (Turbo is English-only)
|
||||
seed: Random seed for reproducibility
|
||||
instruct: Unused (protocol compatibility)
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
await self.load_model()
|
||||
|
||||
ref_audio = voice_prompt.get("ref_audio")
|
||||
if ref_audio and not Path(ref_audio).exists():
|
||||
logger.warning(f"Reference audio not found: {ref_audio}")
|
||||
ref_audio = None
|
||||
|
||||
def _generate_sync():
|
||||
import torch
|
||||
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
|
||||
logger.info("[Chatterbox Turbo] Generating (English)")
|
||||
|
||||
wav = self.model.generate(
|
||||
text,
|
||||
audio_prompt_path=ref_audio,
|
||||
temperature=0.8,
|
||||
top_k=1000,
|
||||
top_p=0.95,
|
||||
repetition_penalty=1.2,
|
||||
)
|
||||
|
||||
# Convert tensor -> numpy
|
||||
if isinstance(wav, torch.Tensor):
|
||||
audio = wav.squeeze().cpu().numpy().astype(np.float32)
|
||||
else:
|
||||
audio = np.asarray(wav, dtype=np.float32)
|
||||
|
||||
sample_rate = (
|
||||
getattr(self.model, "sr", None)
|
||||
or getattr(self.model, "sample_rate", 24000)
|
||||
)
|
||||
|
||||
return audio, sample_rate
|
||||
|
||||
return await asyncio.to_thread(_generate_sync)
|
||||
@@ -5,8 +5,15 @@ MLX backend implementation for TTS and STT using mlx-audio.
|
||||
from typing import Optional, List, Tuple
|
||||
import asyncio
|
||||
import numpy as np
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# PATCH: Import and apply offline patch BEFORE any huggingface_hub usage
|
||||
# This prevents mlx_audio from making network requests when models are cached
|
||||
from ..utils.hf_offline_patch import patch_huggingface_hub_offline, ensure_original_qwen_config_cached
|
||||
patch_huggingface_hub_offline()
|
||||
ensure_original_qwen_config_cached()
|
||||
|
||||
from . import TTSBackend, STTBackend
|
||||
from ..utils.cache import get_cache_key, get_cached_voice_prompt, cache_voice_prompt
|
||||
from ..utils.audio import normalize_audio, load_audio
|
||||
@@ -14,6 +21,12 @@ from ..utils.progress import get_progress_manager
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
LANGUAGE_CODE_TO_NAME = {
|
||||
"zh": "chinese", "en": "english", "ja": "japanese", "ko": "korean",
|
||||
"de": "german", "fr": "french", "ru": "russian", "pt": "portuguese",
|
||||
"es": "spanish", "it": "italian",
|
||||
}
|
||||
|
||||
|
||||
class MLXTTSBackend:
|
||||
"""MLX-based TTS backend using mlx-audio."""
|
||||
@@ -159,15 +172,35 @@ class MLXTTSBackend:
|
||||
tracker_context = tracker.patch_download()
|
||||
tracker_context.__enter__()
|
||||
|
||||
# PATCH: Force offline mode when model is already cached
|
||||
# This prevents crashes when HuggingFace is unreachable
|
||||
original_hf_hub_offline = os.environ.get("HF_HUB_OFFLINE")
|
||||
if is_cached:
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
print(f"[PATCH] Model {model_size} is cached, forcing HF_HUB_OFFLINE=1 to avoid network requests")
|
||||
|
||||
# Import mlx_audio AFTER patching tqdm
|
||||
from mlx_audio.tts import load
|
||||
|
||||
# Load MLX model (downloads automatically)
|
||||
try:
|
||||
self.model = load(model_path)
|
||||
except Exception as load_error:
|
||||
# If offline mode failed, try with network enabled as fallback
|
||||
if is_cached and "offline" in str(load_error).lower():
|
||||
print(f"[PATCH] Offline load failed, trying with network: {load_error}")
|
||||
os.environ.pop("HF_HUB_OFFLINE", None)
|
||||
self.model = load(model_path)
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
# Exit the patch context
|
||||
tracker_context.__exit__(None, None, None)
|
||||
# Restore original HF_HUB_OFFLINE setting
|
||||
if original_hf_hub_offline is not None:
|
||||
os.environ["HF_HUB_OFFLINE"] = original_hf_hub_offline
|
||||
else:
|
||||
os.environ.pop("HF_HUB_OFFLINE", None)
|
||||
|
||||
# Only mark download as complete if we were tracking it
|
||||
if not is_cached:
|
||||
@@ -316,7 +349,8 @@ class MLXTTSBackend:
|
||||
# MLX generate() returns a generator yielding GenerationResult objects
|
||||
audio_chunks = []
|
||||
sample_rate = 24000
|
||||
|
||||
lang = LANGUAGE_CODE_TO_NAME.get(language, "auto")
|
||||
|
||||
# Set seed if provided (MLX uses numpy random)
|
||||
if seed is not None:
|
||||
import mlx.core as mx
|
||||
@@ -344,23 +378,23 @@ class MLXTTSBackend:
|
||||
sig = inspect.signature(self.model.generate)
|
||||
if "ref_audio" in sig.parameters:
|
||||
# Generate with voice cloning
|
||||
for result in self.model.generate(text, ref_audio=ref_audio, ref_text=ref_text):
|
||||
for result in self.model.generate(text, ref_audio=ref_audio, ref_text=ref_text, lang_code=lang):
|
||||
audio_chunks.append(np.array(result.audio))
|
||||
sample_rate = result.sample_rate
|
||||
else:
|
||||
# Fallback: generate without voice cloning
|
||||
for result in self.model.generate(text):
|
||||
for result in self.model.generate(text, lang_code=lang):
|
||||
audio_chunks.append(np.array(result.audio))
|
||||
sample_rate = result.sample_rate
|
||||
else:
|
||||
# No voice prompt, generate normally
|
||||
for result in self.model.generate(text):
|
||||
for result in self.model.generate(text, lang_code=lang):
|
||||
audio_chunks.append(np.array(result.audio))
|
||||
sample_rate = result.sample_rate
|
||||
except Exception as e:
|
||||
# If voice cloning fails, try without it
|
||||
print(f"Warning: Voice cloning failed, generating without voice prompt: {e}")
|
||||
for result in self.model.generate(text):
|
||||
for result in self.model.generate(text, lang_code=lang):
|
||||
audio_chunks.append(np.array(result.audio))
|
||||
sample_rate = result.sample_rate
|
||||
|
||||
|
||||
@@ -15,6 +15,12 @@ from ..utils.progress import get_progress_manager
|
||||
from ..utils.hf_progress import HFProgressTracker, create_hf_progress_callback
|
||||
from ..utils.tasks import get_task_manager
|
||||
|
||||
LANGUAGE_CODE_TO_NAME = {
|
||||
"zh": "chinese", "en": "english", "ja": "japanese", "ko": "korean",
|
||||
"de": "german", "fr": "french", "ru": "russian", "pt": "portuguese",
|
||||
"es": "spanish", "it": "italian",
|
||||
}
|
||||
|
||||
|
||||
class PyTorchTTSBackend:
|
||||
"""PyTorch-based TTS backend using Qwen3-TTS."""
|
||||
@@ -359,6 +365,7 @@ class PyTorchTTSBackend:
|
||||
wavs, sample_rate = self.model.generate_voice_clone(
|
||||
text=text,
|
||||
voice_clone_prompt=voice_prompt,
|
||||
language=LANGUAGE_CODE_TO_NAME.get(language, "auto"),
|
||||
instruct=instruct,
|
||||
)
|
||||
return wavs[0], sample_rate
|
||||
@@ -374,6 +381,7 @@ WHISPER_HF_REPOS = {
|
||||
"small": "openai/whisper-small",
|
||||
"medium": "openai/whisper-medium",
|
||||
"large": "openai/whisper-large-v3",
|
||||
"turbo": "openai/whisper-large-v3-turbo",
|
||||
}
|
||||
|
||||
|
||||
@@ -591,21 +599,20 @@ class PyTorchSTTBackend:
|
||||
)
|
||||
inputs = inputs.to(self.device)
|
||||
|
||||
# Set language if provided
|
||||
forced_decoder_ids = None
|
||||
# Generate transcription
|
||||
# If language is provided, force it; otherwise let Whisper auto-detect
|
||||
generate_kwargs = {}
|
||||
if language:
|
||||
# Support all languages from frontend: en, zh, ja, ko, de, fr, ru, pt, es, it
|
||||
# Whisper supports these and many more
|
||||
forced_decoder_ids = self.processor.get_decoder_prompt_ids(
|
||||
language=language,
|
||||
task="transcribe",
|
||||
)
|
||||
generate_kwargs["forced_decoder_ids"] = forced_decoder_ids
|
||||
|
||||
# Generate transcription
|
||||
with torch.no_grad():
|
||||
predicted_ids = self.model.generate(
|
||||
inputs["input_features"],
|
||||
forced_decoder_ids=forced_decoder_ids,
|
||||
**generate_kwargs,
|
||||
)
|
||||
|
||||
# Decode
|
||||
|
||||
+36
-2
@@ -45,10 +45,14 @@ class Generation(Base):
|
||||
profile_id = Column(String, ForeignKey("profiles.id"), nullable=False)
|
||||
text = Column(Text, nullable=False)
|
||||
language = Column(String, default="en")
|
||||
audio_path = Column(String, nullable=False)
|
||||
duration = Column(Float, nullable=False)
|
||||
audio_path = Column(String, nullable=True)
|
||||
duration = Column(Float, nullable=True)
|
||||
seed = Column(Integer)
|
||||
instruct = Column(Text)
|
||||
engine = Column(String, default="qwen")
|
||||
model_size = Column(String, nullable=True)
|
||||
status = Column(String, default="completed") # generating, completed, failed
|
||||
error = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
@@ -288,6 +292,36 @@ def _run_migrations(engine):
|
||||
conn.commit()
|
||||
print("Added avatar_path column to profiles")
|
||||
|
||||
# Migration: Add status and error columns to generations table
|
||||
if 'generations' in inspector.get_table_names():
|
||||
columns = {col['name'] for col in inspector.get_columns('generations')}
|
||||
if 'status' not in columns:
|
||||
print("Migrating generations: adding status column")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("ALTER TABLE generations ADD COLUMN status VARCHAR DEFAULT 'completed'"))
|
||||
conn.commit()
|
||||
print("Added status column to generations")
|
||||
if 'error' not in columns:
|
||||
print("Migrating generations: adding error column")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("ALTER TABLE generations ADD COLUMN error TEXT"))
|
||||
conn.commit()
|
||||
print("Added error column to generations")
|
||||
if 'engine' not in columns:
|
||||
print("Migrating generations: adding engine column")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("ALTER TABLE generations ADD COLUMN engine VARCHAR DEFAULT 'qwen'"))
|
||||
conn.commit()
|
||||
print("Added engine column to generations")
|
||||
# Re-read columns after engine migration (variable name shadows outer `engine`)
|
||||
columns = {col['name'] for col in inspector.get_columns('generations')}
|
||||
if 'model_size' not in columns:
|
||||
print("Migrating generations: adding model_size column")
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("ALTER TABLE generations ADD COLUMN model_size VARCHAR"))
|
||||
conn.commit()
|
||||
print("Added model_size column to generations")
|
||||
|
||||
|
||||
def get_db():
|
||||
"""Get database session (generator for dependency injection)."""
|
||||
|
||||
+42
-1
@@ -29,6 +29,10 @@ async def create_generation(
|
||||
seed: Optional[int],
|
||||
db: Session,
|
||||
instruct: Optional[str] = None,
|
||||
generation_id: Optional[str] = None,
|
||||
status: str = "completed",
|
||||
engine: Optional[str] = "qwen",
|
||||
model_size: Optional[str] = None,
|
||||
) -> GenerationResponse:
|
||||
"""
|
||||
Create a new generation history entry.
|
||||
@@ -42,12 +46,16 @@ async def create_generation(
|
||||
seed: Random seed used (if any)
|
||||
db: Database session
|
||||
instruct: Natural language instruction used (if any)
|
||||
generation_id: Pre-assigned ID (for async generation flow)
|
||||
status: Generation status (generating, completed, failed)
|
||||
engine: TTS engine used (qwen, luxtts, chatterbox, chatterbox_turbo)
|
||||
model_size: Model size variant (1.7B, 0.6B) — only relevant for qwen
|
||||
|
||||
Returns:
|
||||
Created generation entry
|
||||
"""
|
||||
db_generation = DBGeneration(
|
||||
id=str(uuid.uuid4()),
|
||||
id=generation_id or str(uuid.uuid4()),
|
||||
profile_id=profile_id,
|
||||
text=text,
|
||||
language=language,
|
||||
@@ -55,6 +63,9 @@ async def create_generation(
|
||||
duration=duration,
|
||||
seed=seed,
|
||||
instruct=instruct,
|
||||
engine=engine,
|
||||
model_size=model_size,
|
||||
status=status,
|
||||
created_at=datetime.utcnow(),
|
||||
)
|
||||
|
||||
@@ -65,6 +76,32 @@ async def create_generation(
|
||||
return GenerationResponse.model_validate(db_generation)
|
||||
|
||||
|
||||
async def update_generation_status(
|
||||
generation_id: str,
|
||||
status: str,
|
||||
db: Session,
|
||||
audio_path: Optional[str] = None,
|
||||
duration: Optional[float] = None,
|
||||
error: Optional[str] = None,
|
||||
) -> Optional[GenerationResponse]:
|
||||
"""Update the status of a generation (used by async generation flow)."""
|
||||
generation = db.query(DBGeneration).filter_by(id=generation_id).first()
|
||||
if not generation:
|
||||
return None
|
||||
|
||||
generation.status = status
|
||||
if audio_path is not None:
|
||||
generation.audio_path = audio_path
|
||||
if duration is not None:
|
||||
generation.duration = duration
|
||||
if error is not None:
|
||||
generation.error = error
|
||||
|
||||
db.commit()
|
||||
db.refresh(generation)
|
||||
return GenerationResponse.model_validate(generation)
|
||||
|
||||
|
||||
async def get_generation(
|
||||
generation_id: str,
|
||||
db: Session,
|
||||
@@ -143,6 +180,10 @@ async def list_generations(
|
||||
duration=generation.duration,
|
||||
seed=generation.seed,
|
||||
instruct=generation.instruct,
|
||||
engine=generation.engine or "qwen",
|
||||
model_size=generation.model_size,
|
||||
status=generation.status or "completed",
|
||||
error=generation.error,
|
||||
created_at=generation.created_at,
|
||||
))
|
||||
|
||||
|
||||
+596
-181
@@ -14,7 +14,6 @@ from datetime import datetime
|
||||
import asyncio
|
||||
import uvicorn
|
||||
import argparse
|
||||
import torch
|
||||
import tempfile
|
||||
import io
|
||||
from pathlib import Path
|
||||
@@ -22,6 +21,18 @@ import uuid
|
||||
import asyncio
|
||||
import signal
|
||||
import os
|
||||
|
||||
# Set HSA_OVERRIDE_GFX_VERSION for AMD GPUs that aren't officially listed in ROCm
|
||||
# (e.g., RX 6600 is gfx1032 which maps to gfx1030 target)
|
||||
# This must be set BEFORE any torch.cuda calls
|
||||
if not os.environ.get("HSA_OVERRIDE_GFX_VERSION"):
|
||||
os.environ["HSA_OVERRIDE_GFX_VERSION"] = "10.3.0"
|
||||
|
||||
# Suppress noisy MIOpen workspace warnings on AMD GPUs
|
||||
if not os.environ.get("MIOPEN_LOG_LEVEL"):
|
||||
os.environ["MIOPEN_LOG_LEVEL"] = "4"
|
||||
|
||||
import torch
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
@@ -51,6 +62,9 @@ from .platform_detect import get_backend_type
|
||||
# Keep references to fire-and-forget background tasks to prevent GC
|
||||
_background_tasks: set = set()
|
||||
|
||||
# Generation queue — serializes TTS inference to avoid GPU contention
|
||||
_generation_queue: asyncio.Queue = None # type: ignore # initialized at startup
|
||||
|
||||
|
||||
def _create_background_task(coro) -> asyncio.Task:
|
||||
"""Create a background task and prevent it from being garbage collected."""
|
||||
@@ -60,16 +74,47 @@ def _create_background_task(coro) -> asyncio.Task:
|
||||
return task
|
||||
|
||||
|
||||
async def _generation_worker():
|
||||
"""Worker that processes generation tasks one at a time."""
|
||||
while True:
|
||||
coro = await _generation_queue.get()
|
||||
try:
|
||||
await coro
|
||||
except Exception:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
_generation_queue.task_done()
|
||||
|
||||
|
||||
def _enqueue_generation(coro):
|
||||
"""Add a generation coroutine to the serial queue."""
|
||||
_generation_queue.put_nowait(coro)
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="voicebox API",
|
||||
description="Production-quality Qwen3-TTS voice cloning API",
|
||||
version=__version__,
|
||||
)
|
||||
|
||||
# CORS middleware
|
||||
# CORS middleware - restrict to known local origins by default.
|
||||
# Set VOICEBOX_CORS_ORIGINS env var to a comma-separated list of origins
|
||||
# to allow additional origins (e.g. for remote server mode).
|
||||
_default_origins = [
|
||||
"http://localhost:5173", # Vite dev server
|
||||
"http://127.0.0.1:5173",
|
||||
"http://localhost:17493",
|
||||
"http://127.0.0.1:17493",
|
||||
"tauri://localhost", # Tauri webview (macOS)
|
||||
"https://tauri.localhost", # Tauri webview (Windows/Linux)
|
||||
]
|
||||
_env_origins = os.environ.get("VOICEBOX_CORS_ORIGINS", "")
|
||||
_cors_origins = _default_origins + [o.strip() for o in _env_origins.split(",") if o.strip()]
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # Configure appropriately for production
|
||||
allow_origins=_cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
@@ -671,180 +716,255 @@ async def generate_speech(
|
||||
data: models.GenerationRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Generate speech from text using a voice profile."""
|
||||
"""Generate speech from text using a voice profile.
|
||||
|
||||
Creates a history entry immediately with status='generating' and kicks off
|
||||
TTS in the background. The frontend can poll or use SSE to detect completion.
|
||||
"""
|
||||
task_manager = get_task_manager()
|
||||
generation_id = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
# Start tracking generation
|
||||
task_manager.start_generation(
|
||||
task_id=generation_id,
|
||||
profile_id=data.profile_id,
|
||||
text=data.text,
|
||||
)
|
||||
|
||||
# Get profile
|
||||
profile = await profiles.get_profile(data.profile_id, db)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
|
||||
# Generate audio
|
||||
from .backends import get_tts_backend_for_engine
|
||||
|
||||
engine = data.engine or "qwen"
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
# Validate profile exists before creating the record
|
||||
profile = await profiles.get_profile(data.profile_id, db)
|
||||
if not profile:
|
||||
raise HTTPException(status_code=404, detail="Profile not found")
|
||||
|
||||
# Resolve model size (only relevant for Qwen engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
from .backends import get_tts_backend_for_engine
|
||||
engine = data.engine or "qwen"
|
||||
tts_model = get_tts_backend_for_engine(engine)
|
||||
model_size = data.model_size or "1.7B"
|
||||
|
||||
# Check if model needs to be downloaded first
|
||||
if engine == "qwen":
|
||||
if not tts_model._is_model_cached(model_size):
|
||||
model_name = f"qwen-tts-{model_size}"
|
||||
# Create the history entry immediately with status="generating"
|
||||
generation = await history.create_generation(
|
||||
profile_id=data.profile_id,
|
||||
text=data.text,
|
||||
language=data.language,
|
||||
audio_path="",
|
||||
duration=0,
|
||||
seed=data.seed,
|
||||
db=db,
|
||||
instruct=data.instruct,
|
||||
generation_id=generation_id,
|
||||
status="generating",
|
||||
engine=engine,
|
||||
model_size=model_size if engine == "qwen" else None,
|
||||
)
|
||||
|
||||
async def download_model_background():
|
||||
try:
|
||||
await tts_model.load_model_async(model_size)
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
_create_background_task(download_model_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": f"Model {model_size} is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
# Load (or switch to) the requested model
|
||||
await tts_model.load_model_async(model_size)
|
||||
elif engine == "luxtts":
|
||||
if not tts_model._is_model_cached():
|
||||
model_name = "luxtts"
|
||||
|
||||
async def download_luxtts_background():
|
||||
try:
|
||||
await tts_model.load_model()
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
_create_background_task(download_luxtts_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": "LuxTTS model is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox":
|
||||
if not tts_model._is_model_cached():
|
||||
model_name = "chatterbox-tts"
|
||||
|
||||
async def download_chatterbox_background():
|
||||
try:
|
||||
await tts_model.load_model()
|
||||
except Exception as e:
|
||||
task_manager.error_download(model_name, str(e))
|
||||
|
||||
task_manager.start_download(model_name)
|
||||
asyncio.create_task(download_chatterbox_background())
|
||||
|
||||
raise HTTPException(
|
||||
status_code=202,
|
||||
detail={
|
||||
"message": "Chatterbox model is being downloaded. Please wait and try again.",
|
||||
"model_name": model_name,
|
||||
"downloading": True,
|
||||
},
|
||||
)
|
||||
|
||||
await tts_model.load_model()
|
||||
|
||||
# Create voice prompt from profile
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id,
|
||||
db,
|
||||
use_cache=True,
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
audio, sample_rate = await tts_model.generate(
|
||||
data.text,
|
||||
voice_prompt,
|
||||
data.language,
|
||||
data.seed,
|
||||
data.instruct,
|
||||
)
|
||||
|
||||
# Trim trailing silence/hallucination for Chatterbox output
|
||||
if engine == "chatterbox":
|
||||
from .utils.audio import trim_tts_output
|
||||
audio = trim_tts_output(audio, sample_rate)
|
||||
|
||||
# Calculate duration
|
||||
duration = len(audio) / sample_rate
|
||||
|
||||
# Save audio
|
||||
audio_path = config.get_generations_dir() / f"{generation_id}.wav"
|
||||
|
||||
from .utils.audio import save_audio
|
||||
import errno
|
||||
# Track in task manager
|
||||
task_manager.start_generation(
|
||||
task_id=generation_id,
|
||||
profile_id=data.profile_id,
|
||||
text=data.text,
|
||||
)
|
||||
|
||||
# Kick off TTS in background
|
||||
async def _run_generation():
|
||||
bg_db = next(get_db())
|
||||
try:
|
||||
save_audio(audio, str(audio_path), sample_rate)
|
||||
except BrokenPipeError:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Audio save failed: broken pipe (the output stream was closed unexpectedly)",
|
||||
)
|
||||
except OSError as save_err:
|
||||
err_no = getattr(save_err, "errno", None) or (
|
||||
getattr(save_err.__cause__, "errno", None)
|
||||
if save_err.__cause__
|
||||
else None
|
||||
)
|
||||
if err_no == errno.ENOENT:
|
||||
msg = f"Audio save failed: directory not found — {audio_path.parent}"
|
||||
elif err_no == errno.EACCES:
|
||||
msg = f"Audio save failed: permission denied — {audio_path.parent}"
|
||||
elif err_no == errno.ENOSPC:
|
||||
msg = "Audio save failed: no disk space remaining"
|
||||
# Load model
|
||||
if engine == "qwen":
|
||||
await tts_model.load_model_async(model_size)
|
||||
else:
|
||||
msg = f"Audio save failed: {save_err}"
|
||||
raise HTTPException(status_code=500, detail=msg)
|
||||
await tts_model.load_model()
|
||||
|
||||
# Create history entry
|
||||
generation = await history.create_generation(
|
||||
profile_id=data.profile_id,
|
||||
text=data.text,
|
||||
language=data.language,
|
||||
audio_path=str(audio_path),
|
||||
duration=duration,
|
||||
seed=data.seed,
|
||||
db=db,
|
||||
instruct=data.instruct,
|
||||
)
|
||||
|
||||
# Mark generation as complete
|
||||
task_manager.complete_generation(generation_id)
|
||||
|
||||
return generation
|
||||
|
||||
except ValueError as e:
|
||||
task_manager.complete_generation(generation_id)
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
task_manager.complete_generation(generation_id)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
# Create voice prompt
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id,
|
||||
bg_db,
|
||||
use_cache=True,
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
audio, sample_rate = await generate_chunked(
|
||||
tts_model,
|
||||
data.text,
|
||||
voice_prompt,
|
||||
language=data.language,
|
||||
seed=data.seed,
|
||||
instruct=data.instruct,
|
||||
max_chunk_chars=data.max_chunk_chars,
|
||||
crossfade_ms=data.crossfade_ms,
|
||||
trim_fn=trim_fn,
|
||||
)
|
||||
|
||||
if data.normalize:
|
||||
from .utils.audio import normalize_audio
|
||||
audio = normalize_audio(audio)
|
||||
|
||||
duration = len(audio) / sample_rate
|
||||
audio_path = config.get_generations_dir() / f"{generation_id}.wav"
|
||||
|
||||
from .utils.audio import save_audio
|
||||
save_audio(audio, str(audio_path), sample_rate)
|
||||
|
||||
# Update the record to completed
|
||||
await history.update_generation_status(
|
||||
generation_id=generation_id,
|
||||
status="completed",
|
||||
db=bg_db,
|
||||
audio_path=str(audio_path),
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await history.update_generation_status(
|
||||
generation_id=generation_id,
|
||||
status="failed",
|
||||
db=bg_db,
|
||||
error=str(e),
|
||||
)
|
||||
finally:
|
||||
task_manager.complete_generation(generation_id)
|
||||
bg_db.close()
|
||||
|
||||
_enqueue_generation(_run_generation())
|
||||
|
||||
return generation
|
||||
|
||||
|
||||
@app.post("/generate/{generation_id}/retry", response_model=models.GenerationResponse)
|
||||
async def retry_generation(generation_id: str, db: Session = Depends(get_db)):
|
||||
"""Retry a failed generation using the same parameters."""
|
||||
gen = db.query(DBGeneration).filter_by(id=generation_id).first()
|
||||
if not gen:
|
||||
raise HTTPException(status_code=404, detail="Generation not found")
|
||||
|
||||
if (gen.status or "completed") != "failed":
|
||||
raise HTTPException(status_code=400, detail="Only failed generations can be retried")
|
||||
|
||||
# Reset the record to generating
|
||||
gen.status = "generating"
|
||||
gen.error = None
|
||||
gen.audio_path = ""
|
||||
gen.duration = 0
|
||||
db.commit()
|
||||
db.refresh(gen)
|
||||
|
||||
task_manager = get_task_manager()
|
||||
task_manager.start_generation(
|
||||
task_id=generation_id,
|
||||
profile_id=gen.profile_id,
|
||||
text=gen.text,
|
||||
)
|
||||
|
||||
# Resolve engine/model from stored values
|
||||
retry_engine = gen.engine or "qwen"
|
||||
retry_model_size = gen.model_size or "1.7B"
|
||||
|
||||
from .backends import get_tts_backend_for_engine
|
||||
tts_model = get_tts_backend_for_engine(retry_engine)
|
||||
|
||||
async def _run_retry():
|
||||
bg_db = next(get_db())
|
||||
try:
|
||||
if retry_engine == "qwen":
|
||||
await tts_model.load_model_async(retry_model_size)
|
||||
else:
|
||||
await tts_model.load_model()
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
gen.profile_id,
|
||||
bg_db,
|
||||
use_cache=True,
|
||||
engine=retry_engine,
|
||||
)
|
||||
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if retry_engine in ("chatterbox", "chatterbox_turbo"):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
audio, sample_rate = await generate_chunked(
|
||||
tts_model,
|
||||
gen.text,
|
||||
voice_prompt,
|
||||
language=gen.language,
|
||||
seed=gen.seed,
|
||||
instruct=gen.instruct,
|
||||
trim_fn=trim_fn,
|
||||
)
|
||||
|
||||
duration = len(audio) / sample_rate
|
||||
audio_path = config.get_generations_dir() / f"{generation_id}.wav"
|
||||
|
||||
from .utils.audio import save_audio
|
||||
save_audio(audio, str(audio_path), sample_rate)
|
||||
|
||||
await history.update_generation_status(
|
||||
generation_id=generation_id,
|
||||
status="completed",
|
||||
db=bg_db,
|
||||
audio_path=str(audio_path),
|
||||
duration=duration,
|
||||
)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await history.update_generation_status(
|
||||
generation_id=generation_id,
|
||||
status="failed",
|
||||
db=bg_db,
|
||||
error=str(e),
|
||||
)
|
||||
finally:
|
||||
task_manager.complete_generation(generation_id)
|
||||
bg_db.close()
|
||||
|
||||
_enqueue_generation(_run_retry())
|
||||
|
||||
return models.GenerationResponse.model_validate(gen)
|
||||
|
||||
|
||||
@app.get("/generate/{generation_id}/status")
|
||||
async def get_generation_status(generation_id: str, db: Session = Depends(get_db)):
|
||||
"""SSE endpoint that streams generation status updates.
|
||||
|
||||
Polls the DB every second and yields the current status. Closes when
|
||||
the generation reaches 'completed' or 'failed'.
|
||||
"""
|
||||
import json
|
||||
|
||||
async def event_stream():
|
||||
while True:
|
||||
db.expire_all()
|
||||
gen = db.query(DBGeneration).filter_by(id=generation_id).first()
|
||||
if not gen:
|
||||
yield f"data: {json.dumps({'status': 'not_found', 'id': generation_id})}\n\n"
|
||||
return
|
||||
|
||||
payload = {
|
||||
"id": gen.id,
|
||||
"status": gen.status or "completed",
|
||||
"duration": gen.duration,
|
||||
"error": gen.error,
|
||||
}
|
||||
yield f"data: {json.dumps(payload)}\n\n"
|
||||
|
||||
if (gen.status or "completed") in ("completed", "failed"):
|
||||
return
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
return StreamingResponse(
|
||||
event_stream(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/generate/stream")
|
||||
@@ -890,23 +1010,40 @@ async def stream_speech(
|
||||
detail="Chatterbox model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
elif engine == "chatterbox_turbo":
|
||||
if not tts_model._is_model_cached():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Chatterbox Turbo model is not downloaded yet. Use /generate to trigger a download.",
|
||||
)
|
||||
await tts_model.load_model()
|
||||
|
||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||
data.profile_id, db, engine=engine,
|
||||
)
|
||||
|
||||
audio, sample_rate = await tts_model.generate(
|
||||
from .utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
if engine in ("chatterbox", "chatterbox_turbo"):
|
||||
from .utils.audio import trim_tts_output
|
||||
trim_fn = trim_tts_output
|
||||
|
||||
audio, sample_rate = await generate_chunked(
|
||||
tts_model,
|
||||
data.text,
|
||||
voice_prompt,
|
||||
data.language,
|
||||
data.seed,
|
||||
data.instruct,
|
||||
language=data.language,
|
||||
seed=data.seed,
|
||||
instruct=data.instruct,
|
||||
max_chunk_chars=data.max_chunk_chars,
|
||||
crossfade_ms=data.crossfade_ms,
|
||||
trim_fn=trim_fn,
|
||||
)
|
||||
|
||||
# Trim trailing silence/hallucination for Chatterbox output
|
||||
if engine == "chatterbox":
|
||||
from .utils.audio import trim_tts_output
|
||||
audio = trim_tts_output(audio, sample_rate)
|
||||
if data.normalize:
|
||||
from .utils.audio import normalize_audio
|
||||
audio = normalize_audio(audio)
|
||||
|
||||
wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
|
||||
|
||||
@@ -1114,11 +1251,12 @@ async def transcribe_audio(
|
||||
# Transcribe
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
|
||||
# Check if Whisper model is downloaded (uses default size "base")
|
||||
# Check if Whisper model is downloaded
|
||||
model_size = whisper_model.model_size
|
||||
# Map model sizes to HF repo IDs (whisper-large needs -v3 suffix)
|
||||
# Map model sizes to HF repo IDs (some need special suffixes)
|
||||
whisper_hf_repos = {
|
||||
"large": "openai/whisper-large-v3",
|
||||
"turbo": "openai/whisper-large-v3-turbo",
|
||||
}
|
||||
model_name = whisper_hf_repos.get(model_size, f"openai/whisper-{model_size}")
|
||||
|
||||
@@ -1424,7 +1562,7 @@ async def load_model(model_size: str = "1.7B"):
|
||||
|
||||
@app.post("/models/unload")
|
||||
async def unload_model():
|
||||
"""Unload TTS model to free memory."""
|
||||
"""Unload the default Qwen TTS model to free memory."""
|
||||
try:
|
||||
tts.unload_tts_model()
|
||||
return {"message": "Model unloaded successfully"}
|
||||
@@ -1432,6 +1570,71 @@ async def unload_model():
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.post("/models/{model_name}/unload")
|
||||
async def unload_model_by_name(model_name: str):
|
||||
"""Unload a specific model from memory without deleting it from disk."""
|
||||
# Map of model_name -> (model_type, model_size)
|
||||
model_types = {
|
||||
"qwen-tts-1.7B": ("tts", "1.7B"),
|
||||
"qwen-tts-0.6B": ("tts", "0.6B"),
|
||||
"luxtts": ("luxtts", "default"),
|
||||
"chatterbox-tts": ("chatterbox", "default"),
|
||||
"chatterbox-turbo": ("chatterbox_turbo", "default"),
|
||||
"whisper-base": ("whisper", "base"),
|
||||
"whisper-small": ("whisper", "small"),
|
||||
"whisper-medium": ("whisper", "medium"),
|
||||
"whisper-large": ("whisper", "large"),
|
||||
"whisper-turbo": ("whisper", "turbo"),
|
||||
}
|
||||
|
||||
if model_name not in model_types:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown model: {model_name}")
|
||||
|
||||
model_type, model_size = model_types[model_name]
|
||||
|
||||
try:
|
||||
if model_type == "tts":
|
||||
tts_model = tts.get_tts_model()
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
if tts_model.is_loaded() and loaded_size == model_size:
|
||||
tts.unload_tts_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "luxtts":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("luxtts")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "chatterbox":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "chatterbox_turbo":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
if backend.is_loaded():
|
||||
backend.unload_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
elif model_type == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == model_size:
|
||||
transcribe.unload_whisper_model()
|
||||
else:
|
||||
return {"message": f"Model {model_name} is not loaded"}
|
||||
|
||||
return {"message": f"Model {model_name} unloaded successfully"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
|
||||
|
||||
@app.get("/models/progress/{model_name}")
|
||||
async def get_model_progress(model_name: str):
|
||||
"""Get model download progress via Server-Sent Events."""
|
||||
@@ -1455,6 +1658,141 @@ async def get_model_progress(model_name: str):
|
||||
)
|
||||
|
||||
|
||||
@app.get("/models/cache-dir")
|
||||
async def get_models_cache_dir():
|
||||
"""Get the path to the HuggingFace model cache directory."""
|
||||
from huggingface_hub import constants as hf_constants
|
||||
return {"path": str(Path(hf_constants.HF_HUB_CACHE))}
|
||||
|
||||
|
||||
def _get_dir_size(path: Path) -> int:
|
||||
"""Get total size of a directory in bytes."""
|
||||
total = 0
|
||||
for f in path.rglob("*"):
|
||||
if f.is_file():
|
||||
total += f.stat().st_size
|
||||
return total
|
||||
|
||||
|
||||
def _copy_with_progress(src: Path, dst: Path, progress_manager, copied_so_far: int, total_bytes: int) -> int:
|
||||
"""Copy a directory tree with byte-level progress tracking."""
|
||||
import shutil
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.iterdir():
|
||||
dest_item = dst / item.name
|
||||
if item.is_dir():
|
||||
copied_so_far = _copy_with_progress(item, dest_item, progress_manager, copied_so_far, total_bytes)
|
||||
else:
|
||||
size = item.stat().st_size
|
||||
shutil.copy2(str(item), str(dest_item))
|
||||
copied_so_far += size
|
||||
progress_manager.update_progress(
|
||||
"migration", copied_so_far, total_bytes,
|
||||
filename=item.name, status="downloading",
|
||||
)
|
||||
return copied_so_far
|
||||
|
||||
|
||||
@app.post("/models/migrate")
|
||||
async def migrate_models(request: models.ModelMigrateRequest):
|
||||
"""Move all downloaded models to a new directory with byte-level progress via SSE."""
|
||||
import shutil
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
source = Path(hf_constants.HF_HUB_CACHE)
|
||||
destination = Path(request.destination)
|
||||
|
||||
if not source.exists():
|
||||
raise HTTPException(status_code=404, detail="Current model cache directory not found")
|
||||
|
||||
model_dirs = [d for d in source.iterdir() if d.name.startswith("models--") and d.is_dir()]
|
||||
if not model_dirs:
|
||||
return {"moved": 0, "errors": [], "source": str(source), "destination": str(destination)}
|
||||
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
|
||||
# Check if source and destination are on the same filesystem (rename is instant)
|
||||
same_fs = False
|
||||
try:
|
||||
same_fs = source.stat().st_dev == destination.stat().st_dev
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
async def migrate_background():
|
||||
moved = 0
|
||||
errors = []
|
||||
try:
|
||||
if same_fs:
|
||||
# Same filesystem: rename is instant, just track model count
|
||||
total = len(model_dirs)
|
||||
for i, item in enumerate(model_dirs):
|
||||
dest_item = destination / item.name
|
||||
try:
|
||||
if dest_item.exists():
|
||||
shutil.rmtree(dest_item)
|
||||
shutil.move(str(item), str(dest_item))
|
||||
moved += 1
|
||||
progress_manager.update_progress(
|
||||
"migration", i + 1, total,
|
||||
filename=item.name, status="downloading",
|
||||
)
|
||||
except Exception as e:
|
||||
errors.append(f"{item.name}: {str(e)}")
|
||||
else:
|
||||
# Cross-filesystem: copy with byte-level progress, then delete source
|
||||
total_bytes = sum(_get_dir_size(d) for d in model_dirs)
|
||||
progress_manager.update_progress("migration", 0, total_bytes, filename="Calculating...", status="downloading")
|
||||
|
||||
copied = 0
|
||||
for item in model_dirs:
|
||||
dest_item = destination / item.name
|
||||
try:
|
||||
if dest_item.exists():
|
||||
shutil.rmtree(dest_item)
|
||||
copied = await asyncio.to_thread(
|
||||
_copy_with_progress, item, dest_item, progress_manager, copied, total_bytes
|
||||
)
|
||||
# Remove source after successful copy
|
||||
await asyncio.to_thread(shutil.rmtree, str(item))
|
||||
moved += 1
|
||||
except Exception as e:
|
||||
errors.append(f"{item.name}: {str(e)}")
|
||||
|
||||
progress_manager.update_progress("migration", 1, 1, status="complete")
|
||||
progress_manager.mark_complete("migration")
|
||||
except Exception as e:
|
||||
progress_manager.update_progress("migration", 0, 0, status="error")
|
||||
progress_manager.mark_error("migration", str(e))
|
||||
|
||||
_create_background_task(migrate_background())
|
||||
|
||||
return {"source": str(source), "destination": str(destination)}
|
||||
|
||||
|
||||
@app.get("/models/migrate/progress")
|
||||
async def get_migration_progress():
|
||||
"""Get model migration progress via Server-Sent Events."""
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
progress_manager = get_progress_manager()
|
||||
|
||||
async def event_generator():
|
||||
async for event in progress_manager.subscribe("migration"):
|
||||
yield event
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/models/status", response_model=models.ModelStatusListResponse)
|
||||
async def get_model_status():
|
||||
"""Get status of all available models."""
|
||||
@@ -1478,7 +1816,10 @@ async def get_model_status():
|
||||
"""Check if TTS model is loaded with specific size."""
|
||||
try:
|
||||
tts_model = tts.get_tts_model()
|
||||
return tts_model.is_loaded() and getattr(tts_model, 'model_size', None) == model_size
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
return tts_model.is_loaded() and loaded_size == model_size
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -1525,6 +1866,15 @@ async def get_model_status():
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
# Check if Chatterbox Turbo backend is loaded
|
||||
def check_chatterbox_turbo_loaded():
|
||||
try:
|
||||
from .backends import get_tts_backend_for_engine
|
||||
backend = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
model_configs = [
|
||||
{
|
||||
"model_name": "qwen-tts-1.7B",
|
||||
@@ -1554,6 +1904,13 @@ async def get_model_status():
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "chatterbox-turbo",
|
||||
"display_name": "Chatterbox Turbo (English, Tags)",
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"check_loaded": check_chatterbox_turbo_loaded,
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-base",
|
||||
"display_name": "Whisper Base",
|
||||
@@ -1582,6 +1939,13 @@ async def get_model_status():
|
||||
"model_size": "large",
|
||||
"check_loaded": lambda: check_whisper_loaded("large"),
|
||||
},
|
||||
{
|
||||
"model_name": "whisper-turbo",
|
||||
"display_name": "Whisper Turbo",
|
||||
"hf_repo_id": "openai/whisper-large-v3-turbo",
|
||||
"model_size": "turbo",
|
||||
"check_loaded": lambda: check_whisper_loaded("turbo"),
|
||||
},
|
||||
]
|
||||
|
||||
# Build a mapping of model_name -> hf_repo_id so we can check if shared repos are downloading
|
||||
@@ -1760,6 +2124,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox").load_model(),
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"model_size": "default",
|
||||
"load_func": lambda: get_tts_backend_for_engine("chatterbox_turbo").load_model(),
|
||||
},
|
||||
"whisper-base": {
|
||||
"model_size": "base",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("base"),
|
||||
@@ -1776,6 +2144,10 @@ async def trigger_model_download(request: models.ModelDownloadRequest):
|
||||
"model_size": "large",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("large"),
|
||||
},
|
||||
"whisper-turbo": {
|
||||
"model_size": "turbo",
|
||||
"load_func": lambda: transcribe.get_whisper_model().load_model("turbo"),
|
||||
},
|
||||
}
|
||||
|
||||
if request.model_name not in model_configs:
|
||||
@@ -1882,6 +2254,11 @@ async def delete_model(model_name: str):
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox",
|
||||
},
|
||||
"chatterbox-turbo": {
|
||||
"hf_repo_id": "ResembleAI/chatterbox-turbo",
|
||||
"model_size": "default",
|
||||
"model_type": "chatterbox_turbo",
|
||||
},
|
||||
"whisper-base": {
|
||||
"hf_repo_id": "openai/whisper-base",
|
||||
"model_size": "base",
|
||||
@@ -1902,6 +2279,11 @@ async def delete_model(model_name: str):
|
||||
"model_size": "large",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
"whisper-turbo": {
|
||||
"hf_repo_id": "openai/whisper-large-v3-turbo",
|
||||
"model_size": "turbo",
|
||||
"model_type": "whisper",
|
||||
},
|
||||
}
|
||||
|
||||
if model_name not in model_configs:
|
||||
@@ -1914,7 +2296,10 @@ async def delete_model(model_name: str):
|
||||
# Check if model is loaded and unload it first
|
||||
if config["model_type"] == "tts":
|
||||
tts_model = tts.get_tts_model()
|
||||
if tts_model.is_loaded() and tts_model.model_size == config["model_size"]:
|
||||
loaded_size = getattr(
|
||||
tts_model, "_current_model_size", None
|
||||
) or getattr(tts_model, "model_size", None)
|
||||
if tts_model.is_loaded() and loaded_size == config["model_size"]:
|
||||
tts.unload_tts_model()
|
||||
elif config["model_type"] == "luxtts":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
@@ -1926,6 +2311,11 @@ async def delete_model(model_name: str):
|
||||
chatterbox = get_tts_backend_for_engine("chatterbox")
|
||||
if chatterbox.is_loaded():
|
||||
chatterbox.unload_model()
|
||||
elif config["model_type"] == "chatterbox_turbo":
|
||||
from .backends import get_tts_backend_for_engine
|
||||
turbo = get_tts_backend_for_engine("chatterbox_turbo")
|
||||
if turbo.is_loaded():
|
||||
turbo.unload_model()
|
||||
elif config["model_type"] == "whisper":
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
if whisper_model.is_loaded() and whisper_model.model_size == config["model_size"]:
|
||||
@@ -2136,7 +2526,12 @@ def _get_gpu_status() -> str:
|
||||
"""Get GPU availability status."""
|
||||
backend_type = get_backend_type()
|
||||
if torch.cuda.is_available():
|
||||
return f"CUDA ({torch.cuda.get_device_name(0)})"
|
||||
device_name = torch.cuda.get_device_name(0)
|
||||
# Check if this is ROCm (AMD) or CUDA (NVIDIA)
|
||||
is_rocm = hasattr(torch.version, 'hip') and torch.version.hip is not None
|
||||
if is_rocm:
|
||||
return f"ROCm ({device_name})"
|
||||
return f"CUDA ({device_name})"
|
||||
elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
|
||||
return "MPS (Apple Silicon)"
|
||||
elif backend_type == "mlx":
|
||||
@@ -2147,9 +2542,29 @@ def _get_gpu_status() -> str:
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Run on application startup."""
|
||||
global _generation_queue
|
||||
print("voicebox API starting up...")
|
||||
database.init_db()
|
||||
print(f"Database initialized at {database._db_path}")
|
||||
|
||||
# Start the serial generation worker
|
||||
_generation_queue = asyncio.Queue()
|
||||
_create_background_task(_generation_worker())
|
||||
|
||||
# Mark any stale "generating" records as failed — these are leftovers
|
||||
# from a previous process that was killed mid-generation
|
||||
try:
|
||||
from sqlalchemy import text as sa_text
|
||||
db = next(get_db())
|
||||
result = db.execute(
|
||||
sa_text("UPDATE generations SET status = 'failed', error = 'Server was shut down during generation' WHERE status = 'generating'")
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
print(f"Marked {result.rowcount} stale generation(s) as failed")
|
||||
db.commit()
|
||||
db.close()
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not clean up stale generations: {e}")
|
||||
backend_type = get_backend_type()
|
||||
print(f"Backend: {backend_type.upper()}")
|
||||
print(f"GPU available: {_get_gpu_status()}")
|
||||
|
||||
+27
-11
@@ -11,7 +11,7 @@ class VoiceProfileCreate(BaseModel):
|
||||
"""Request model for creating a voice profile."""
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
description: Optional[str] = Field(None, max_length=500)
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$")
|
||||
|
||||
|
||||
class VoiceProfileResponse(BaseModel):
|
||||
@@ -52,12 +52,15 @@ class ProfileSampleResponse(BaseModel):
|
||||
class GenerationRequest(BaseModel):
|
||||
"""Request model for voice generation."""
|
||||
profile_id: str
|
||||
text: str = Field(..., min_length=1, max_length=5000)
|
||||
text: str = Field(..., min_length=1, max_length=50000)
|
||||
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
|
||||
seed: Optional[int] = Field(None, ge=0)
|
||||
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
|
||||
instruct: Optional[str] = Field(None, max_length=500)
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox)$")
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo)$")
|
||||
max_chunk_chars: int = Field(default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting")
|
||||
crossfade_ms: int = Field(default=50, ge=0, le=500, description="Crossfade duration in ms between chunks (0 for hard cut)")
|
||||
normalize: bool = Field(default=True, description="Normalize output audio volume")
|
||||
|
||||
|
||||
class GenerationResponse(BaseModel):
|
||||
@@ -66,10 +69,14 @@ class GenerationResponse(BaseModel):
|
||||
profile_id: str
|
||||
text: str
|
||||
language: str
|
||||
audio_path: str
|
||||
duration: float
|
||||
seed: Optional[int]
|
||||
instruct: Optional[str]
|
||||
audio_path: Optional[str] = None
|
||||
duration: Optional[float] = None
|
||||
seed: Optional[int] = None
|
||||
instruct: Optional[str] = None
|
||||
engine: Optional[str] = "qwen"
|
||||
model_size: Optional[str] = None
|
||||
status: str = "completed"
|
||||
error: Optional[str] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
@@ -91,10 +98,14 @@ class HistoryResponse(BaseModel):
|
||||
profile_name: str
|
||||
text: str
|
||||
language: str
|
||||
audio_path: str
|
||||
duration: float
|
||||
seed: Optional[int]
|
||||
instruct: Optional[str]
|
||||
audio_path: Optional[str] = None
|
||||
duration: Optional[float] = None
|
||||
seed: Optional[int] = None
|
||||
instruct: Optional[str] = None
|
||||
engine: Optional[str] = "qwen"
|
||||
model_size: Optional[str] = None
|
||||
status: str = "completed"
|
||||
error: Optional[str] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
@@ -168,6 +179,11 @@ class ModelDownloadRequest(BaseModel):
|
||||
model_name: str
|
||||
|
||||
|
||||
class ModelMigrateRequest(BaseModel):
|
||||
"""Request model for migrating models to a new directory."""
|
||||
destination: str
|
||||
|
||||
|
||||
class ActiveDownloadTask(BaseModel):
|
||||
"""Response model for active download task."""
|
||||
model_name: str
|
||||
|
||||
+6
-6
@@ -270,11 +270,14 @@ async def add_item_to_story(
|
||||
generation_created_at=generation.created_at,
|
||||
)
|
||||
|
||||
# Get track from data or default to 0
|
||||
track = data.track if data.track is not None else 0
|
||||
|
||||
# Calculate start_time_ms if not provided
|
||||
if data.start_time_ms is not None:
|
||||
start_time_ms = data.start_time_ms
|
||||
else:
|
||||
# Find the maximum end time (start_time_ms + duration_ms) of existing items
|
||||
# Find the maximum end time on the target track only
|
||||
existing_items = db.query(
|
||||
DBStoryItem,
|
||||
DBGeneration
|
||||
@@ -282,11 +285,11 @@ async def add_item_to_story(
|
||||
DBGeneration,
|
||||
DBStoryItem.generation_id == DBGeneration.id
|
||||
).filter(
|
||||
DBStoryItem.story_id == story_id
|
||||
DBStoryItem.story_id == story_id,
|
||||
DBStoryItem.track == track,
|
||||
).all()
|
||||
|
||||
if not existing_items:
|
||||
# First item starts at 0
|
||||
start_time_ms = 0
|
||||
else:
|
||||
max_end_time_ms = 0
|
||||
@@ -297,9 +300,6 @@ async def add_item_to_story(
|
||||
# Add 200ms gap after the last item
|
||||
start_time_ms = max_end_time_ms + 200
|
||||
|
||||
# Get track from data or default to 0
|
||||
track = data.track if data.track is not None else 0
|
||||
|
||||
# Create item
|
||||
item = DBStoryItem(
|
||||
id=str(uuid.uuid4()),
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
Tests for CORS origin restrictions.
|
||||
|
||||
Validates that the CORS middleware only allows known local origins
|
||||
and respects the VOICEBOX_CORS_ORIGINS environment variable.
|
||||
|
||||
Uses a minimal FastAPI app that mirrors the exact CORS configuration
|
||||
from backend/main.py, so tests run without heavy ML dependencies.
|
||||
|
||||
Usage:
|
||||
pip install httpx pytest fastapi starlette
|
||||
python -m pytest backend/tests/test_cors.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
|
||||
def _build_app(env_origins: str = "") -> FastAPI:
|
||||
"""
|
||||
Build a minimal FastAPI app with the same CORS logic as backend/main.py.
|
||||
|
||||
This mirrors the exact code in main.py so the test validates the real
|
||||
configuration without needing torch/numpy/transformers installed.
|
||||
"""
|
||||
app = FastAPI()
|
||||
|
||||
_default_origins = [
|
||||
"http://localhost:5173",
|
||||
"http://127.0.0.1:5173",
|
||||
"http://localhost:17493",
|
||||
"http://127.0.0.1:17493",
|
||||
"tauri://localhost",
|
||||
"https://tauri.localhost",
|
||||
]
|
||||
_cors_origins = _default_origins + [o.strip() for o in env_origins.split(",") if o.strip()]
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=_cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client():
|
||||
return TestClient(_build_app())
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client_with_custom_origins():
|
||||
return TestClient(_build_app("https://custom.example.com,https://other.example.com"))
|
||||
|
||||
|
||||
def _get_with_origin(client: TestClient, origin: str) -> dict:
|
||||
"""Send a GET with Origin header, return response headers."""
|
||||
response = client.get("/health", headers={"Origin": origin})
|
||||
return dict(response.headers)
|
||||
|
||||
|
||||
def _preflight(client: TestClient, origin: str) -> dict:
|
||||
"""Send CORS preflight OPTIONS request, return response headers."""
|
||||
response = client.options(
|
||||
"/health",
|
||||
headers={
|
||||
"Origin": origin,
|
||||
"Access-Control-Request-Method": "GET",
|
||||
},
|
||||
)
|
||||
return dict(response.headers)
|
||||
|
||||
|
||||
class TestCORSDefaultOrigins:
|
||||
"""CORS should allow known local origins and block everything else."""
|
||||
|
||||
@pytest.mark.parametrize("origin", [
|
||||
"http://localhost:5173",
|
||||
"http://127.0.0.1:5173",
|
||||
"http://localhost:17493",
|
||||
"http://127.0.0.1:17493",
|
||||
"tauri://localhost",
|
||||
"https://tauri.localhost",
|
||||
])
|
||||
def test_allowed_origins(self, client, origin):
|
||||
headers = _get_with_origin(client, origin)
|
||||
assert headers.get("access-control-allow-origin") == origin
|
||||
|
||||
@pytest.mark.parametrize("origin", [
|
||||
"http://evil.com",
|
||||
"http://localhost:9999",
|
||||
"https://attacker.example.com",
|
||||
"null",
|
||||
])
|
||||
def test_blocked_origins(self, client, origin):
|
||||
headers = _get_with_origin(client, origin)
|
||||
assert "access-control-allow-origin" not in headers
|
||||
|
||||
def test_preflight_allowed(self, client):
|
||||
headers = _preflight(client, "http://localhost:5173")
|
||||
assert headers.get("access-control-allow-origin") == "http://localhost:5173"
|
||||
|
||||
def test_preflight_blocked(self, client):
|
||||
headers = _preflight(client, "http://evil.com")
|
||||
assert "access-control-allow-origin" not in headers
|
||||
|
||||
def test_credentials_header_present(self, client):
|
||||
headers = _get_with_origin(client, "http://localhost:5173")
|
||||
assert headers.get("access-control-allow-credentials") == "true"
|
||||
|
||||
|
||||
class TestCORSCustomOrigins:
|
||||
"""VOICEBOX_CORS_ORIGINS env var should extend the allowlist."""
|
||||
|
||||
def test_custom_origin_allowed(self, client_with_custom_origins):
|
||||
headers = _get_with_origin(client_with_custom_origins, "https://custom.example.com")
|
||||
assert headers.get("access-control-allow-origin") == "https://custom.example.com"
|
||||
|
||||
def test_other_custom_origin_allowed(self, client_with_custom_origins):
|
||||
headers = _get_with_origin(client_with_custom_origins, "https://other.example.com")
|
||||
assert headers.get("access-control-allow-origin") == "https://other.example.com"
|
||||
|
||||
def test_default_origins_still_work(self, client_with_custom_origins):
|
||||
headers = _get_with_origin(client_with_custom_origins, "http://localhost:5173")
|
||||
assert headers.get("access-control-allow-origin") == "http://localhost:5173"
|
||||
|
||||
def test_unlisted_origin_still_blocked(self, client_with_custom_origins):
|
||||
headers = _get_with_origin(client_with_custom_origins, "http://evil.com")
|
||||
assert "access-control-allow-origin" not in headers
|
||||
|
||||
|
||||
class TestCORSEnvVarParsing:
|
||||
"""Edge cases for VOICEBOX_CORS_ORIGINS parsing."""
|
||||
|
||||
def test_empty_env_var(self):
|
||||
app = _build_app("")
|
||||
client = TestClient(app)
|
||||
headers = _get_with_origin(client, "http://evil.com")
|
||||
assert "access-control-allow-origin" not in headers
|
||||
|
||||
def test_whitespace_trimmed(self):
|
||||
app = _build_app(" https://spaced.example.com ")
|
||||
client = TestClient(app)
|
||||
headers = _get_with_origin(client, "https://spaced.example.com")
|
||||
assert headers.get("access-control-allow-origin") == "https://spaced.example.com"
|
||||
|
||||
def test_trailing_comma_ignored(self):
|
||||
app = _build_app("https://one.example.com,")
|
||||
client = TestClient(app)
|
||||
headers = _get_with_origin(client, "https://one.example.com")
|
||||
assert headers.get("access-control-allow-origin") == "https://one.example.com"
|
||||
@@ -92,8 +92,9 @@ def save_audio(
|
||||
# Ensure parent directory exists
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write to temporary file first
|
||||
sf.write(temp_path, audio, sample_rate)
|
||||
# Write to temporary file first (explicit format since .tmp
|
||||
# extension is not recognised by soundfile)
|
||||
sf.write(temp_path, audio, sample_rate, format='WAV')
|
||||
|
||||
# Atomic rename to final path
|
||||
os.replace(temp_path, path)
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
"""
|
||||
Chunked TTS generation utilities.
|
||||
|
||||
Splits long text into sentence-boundary chunks, generates audio per-chunk
|
||||
via any TTSBackend, and concatenates with crossfade. All logic is
|
||||
engine-agnostic — it wraps the standard ``TTSBackend.generate()`` interface.
|
||||
|
||||
Short text (≤ max_chunk_chars) uses the single-shot fast path with zero
|
||||
overhead.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger("voicebox.chunked-tts")
|
||||
|
||||
# Default chunk size in characters. Can be overridden per-request via
|
||||
# the ``max_chunk_chars`` field on GenerationRequest.
|
||||
DEFAULT_MAX_CHUNK_CHARS = 800
|
||||
|
||||
# Common abbreviations that should NOT be treated as sentence endings.
|
||||
# Lowercase for case-insensitive matching.
|
||||
_ABBREVIATIONS = frozenset(
|
||||
{
|
||||
"mr",
|
||||
"mrs",
|
||||
"ms",
|
||||
"dr",
|
||||
"prof",
|
||||
"sr",
|
||||
"jr",
|
||||
"st",
|
||||
"ave",
|
||||
"blvd",
|
||||
"inc",
|
||||
"ltd",
|
||||
"corp",
|
||||
"dept",
|
||||
"est",
|
||||
"approx",
|
||||
"vs",
|
||||
"etc",
|
||||
"e.g",
|
||||
"i.e",
|
||||
"a.m",
|
||||
"p.m",
|
||||
"u.s",
|
||||
"u.s.a",
|
||||
"u.k",
|
||||
}
|
||||
)
|
||||
|
||||
# Paralinguistic tags used by Chatterbox Turbo. The splitter must never
|
||||
# cut inside one of these.
|
||||
_PARA_TAG_RE = re.compile(r"\[[^\]]*\]")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Text splitting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def split_text_into_chunks(text: str, max_chars: int = DEFAULT_MAX_CHUNK_CHARS) -> List[str]:
|
||||
"""Split *text* at natural boundaries into chunks of at most *max_chars*.
|
||||
|
||||
Priority: sentence-end (``.!?`` not preceded by an abbreviation and not
|
||||
inside brackets) → clause boundary (``;:,—``) → whitespace → hard cut.
|
||||
|
||||
Paralinguistic tags like ``[laugh]`` are treated as atomic and will not
|
||||
be split across chunks.
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return []
|
||||
if len(text) <= max_chars:
|
||||
return [text]
|
||||
|
||||
chunks: List[str] = []
|
||||
remaining = text
|
||||
|
||||
while remaining:
|
||||
remaining = remaining.lstrip()
|
||||
if not remaining:
|
||||
break
|
||||
if len(remaining) <= max_chars:
|
||||
chunks.append(remaining)
|
||||
break
|
||||
|
||||
segment = remaining[:max_chars]
|
||||
|
||||
# Try to split at the last real sentence ending
|
||||
split_pos = _find_last_sentence_end(segment)
|
||||
if split_pos == -1:
|
||||
split_pos = _find_last_clause_boundary(segment)
|
||||
if split_pos == -1:
|
||||
split_pos = segment.rfind(" ")
|
||||
if split_pos == -1:
|
||||
# Absolute fallback: hard cut but avoid splitting inside a tag
|
||||
split_pos = _safe_hard_cut(segment, max_chars)
|
||||
|
||||
chunk = remaining[: split_pos + 1].strip()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
remaining = remaining[split_pos + 1 :]
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def _find_last_sentence_end(text: str) -> int:
|
||||
"""Return the index of the last sentence-ending punctuation in *text*.
|
||||
|
||||
Skips periods that follow common abbreviations (``Dr.``, ``Mr.``, etc.)
|
||||
and periods inside bracket tags (``[laugh]``). Also handles CJK
|
||||
sentence-ending punctuation (``。!?``).
|
||||
"""
|
||||
best = -1
|
||||
# ASCII sentence ends
|
||||
for m in re.finditer(r"[.!?](?:\s|$)", text):
|
||||
pos = m.start()
|
||||
char = text[pos]
|
||||
# Skip periods after abbreviations
|
||||
if char == ".":
|
||||
# Walk backwards to find the preceding word
|
||||
word_start = pos - 1
|
||||
while word_start >= 0 and text[word_start].isalpha():
|
||||
word_start -= 1
|
||||
word = text[word_start + 1 : pos].lower()
|
||||
if word in _ABBREVIATIONS:
|
||||
continue
|
||||
# Skip decimal numbers (digit immediately before the period)
|
||||
if word_start >= 0 and text[word_start].isdigit():
|
||||
continue
|
||||
# Skip if we're inside a bracket tag
|
||||
if _inside_bracket_tag(text, pos):
|
||||
continue
|
||||
best = pos
|
||||
# CJK sentence-ending punctuation
|
||||
for m in re.finditer(r"[\u3002\uff01\uff1f]", text):
|
||||
if m.start() > best:
|
||||
best = m.start()
|
||||
return best
|
||||
|
||||
|
||||
def _find_last_clause_boundary(text: str) -> int:
|
||||
"""Return the index of the last clause-boundary punctuation."""
|
||||
best = -1
|
||||
for m in re.finditer(r"[;:,\u2014](?:\s|$)", text):
|
||||
pos = m.start()
|
||||
# Skip if inside a bracket tag
|
||||
if _inside_bracket_tag(text, pos):
|
||||
continue
|
||||
best = pos
|
||||
return best
|
||||
|
||||
|
||||
def _inside_bracket_tag(text: str, pos: int) -> bool:
|
||||
"""Return True if *pos* falls inside a ``[...]`` tag."""
|
||||
for m in _PARA_TAG_RE.finditer(text):
|
||||
if m.start() < pos < m.end():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _safe_hard_cut(segment: str, max_chars: int) -> int:
|
||||
"""Find a hard-cut position that doesn't split a ``[tag]``."""
|
||||
cut = max_chars - 1
|
||||
# Check if the cut falls inside a bracket tag; if so, move before it
|
||||
for m in _PARA_TAG_RE.finditer(segment):
|
||||
if m.start() < cut < m.end():
|
||||
return m.start() - 1 if m.start() > 0 else cut
|
||||
return cut
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio concatenation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def concatenate_audio_chunks(
|
||||
chunks: List[np.ndarray],
|
||||
sample_rate: int,
|
||||
crossfade_ms: int = 50,
|
||||
) -> np.ndarray:
|
||||
"""Concatenate audio arrays with a short crossfade to eliminate clicks.
|
||||
|
||||
Each chunk is expected to be a 1-D float32 ndarray at *sample_rate* Hz.
|
||||
"""
|
||||
if not chunks:
|
||||
return np.array([], dtype=np.float32)
|
||||
if len(chunks) == 1:
|
||||
return chunks[0]
|
||||
|
||||
crossfade_samples = int(sample_rate * crossfade_ms / 1000)
|
||||
result = np.array(chunks[0], dtype=np.float32, copy=True)
|
||||
|
||||
for chunk in chunks[1:]:
|
||||
if len(chunk) == 0:
|
||||
continue
|
||||
overlap = min(crossfade_samples, len(result), len(chunk))
|
||||
if overlap > 0:
|
||||
fade_out = np.linspace(1.0, 0.0, overlap, dtype=np.float32)
|
||||
fade_in = np.linspace(0.0, 1.0, overlap, dtype=np.float32)
|
||||
result[-overlap:] = result[-overlap:] * fade_out + chunk[:overlap] * fade_in
|
||||
result = np.concatenate([result, chunk[overlap:]])
|
||||
else:
|
||||
result = np.concatenate([result, chunk])
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Engine-agnostic chunked generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def generate_chunked(
|
||||
backend,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: int | None = None,
|
||||
instruct: str | None = None,
|
||||
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
|
||||
crossfade_ms: int = 50,
|
||||
trim_fn=None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""Generate audio with automatic chunking for long text.
|
||||
|
||||
For text shorter than *max_chunk_chars* this is a thin wrapper around
|
||||
``backend.generate()`` with zero overhead.
|
||||
|
||||
For longer text the input is split at natural sentence boundaries,
|
||||
each chunk is generated independently, optionally trimmed (useful for
|
||||
Chatterbox engines that hallucinate trailing noise), and the results
|
||||
are concatenated with a crossfade (or hard cut if *crossfade_ms* is 0).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
backend : TTSBackend
|
||||
Any backend implementing the ``generate()`` protocol.
|
||||
text : str
|
||||
Input text (may be arbitrarily long).
|
||||
voice_prompt, language, seed, instruct
|
||||
Forwarded to ``backend.generate()`` verbatim.
|
||||
max_chunk_chars : int
|
||||
Maximum characters per chunk (default 800).
|
||||
crossfade_ms : int
|
||||
Crossfade duration in milliseconds between chunks. 0 for a hard
|
||||
cut with no overlap (default 50).
|
||||
trim_fn : callable | None
|
||||
Optional ``(audio, sample_rate) -> audio`` post-processing
|
||||
function applied to each chunk before concatenation (e.g.
|
||||
``trim_tts_output`` for Chatterbox engines).
|
||||
|
||||
Returns
|
||||
-------
|
||||
(audio, sample_rate) : Tuple[np.ndarray, int]
|
||||
"""
|
||||
chunks = split_text_into_chunks(text, max_chunk_chars)
|
||||
|
||||
if len(chunks) <= 1:
|
||||
# Short text — single-shot fast path
|
||||
audio, sample_rate = await backend.generate(
|
||||
text, voice_prompt, language, seed, instruct,
|
||||
)
|
||||
if trim_fn is not None:
|
||||
audio = trim_fn(audio, sample_rate)
|
||||
return audio, sample_rate
|
||||
|
||||
# Long text — chunked generation
|
||||
logger.info(
|
||||
"Splitting %d chars into %d chunks (max %d chars each)",
|
||||
len(text), len(chunks), max_chunk_chars,
|
||||
)
|
||||
audio_chunks: List[np.ndarray] = []
|
||||
sample_rate: int | None = None
|
||||
|
||||
for i, chunk_text in enumerate(chunks):
|
||||
logger.info(
|
||||
"Generating chunk %d/%d (%d chars)",
|
||||
i + 1, len(chunks), len(chunk_text),
|
||||
)
|
||||
# Vary the seed per chunk to avoid correlated RNG artefacts,
|
||||
# but keep it deterministic so the same (text, seed) pair
|
||||
# always produces the same output.
|
||||
chunk_seed = (seed + i) if seed is not None else None
|
||||
|
||||
chunk_audio, chunk_sr = await backend.generate(
|
||||
chunk_text, voice_prompt, language, chunk_seed, instruct,
|
||||
)
|
||||
if trim_fn is not None:
|
||||
chunk_audio = trim_fn(chunk_audio, chunk_sr)
|
||||
|
||||
audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
|
||||
if sample_rate is None:
|
||||
sample_rate = chunk_sr
|
||||
|
||||
audio = concatenate_audio_chunks(audio_chunks, sample_rate, crossfade_ms=crossfade_ms)
|
||||
return audio, sample_rate
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
Monkey patch for huggingface_hub to force offline mode with cached models.
|
||||
This prevents mlx_audio from making network requests when models are already downloaded.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
|
||||
def patch_huggingface_hub_offline():
|
||||
"""
|
||||
Monkey-patch huggingface_hub to force offline mode.
|
||||
This must be called BEFORE importing mlx_audio.
|
||||
"""
|
||||
try:
|
||||
import huggingface_hub
|
||||
from huggingface_hub import constants as hf_constants
|
||||
from huggingface_hub.file_download import _try_to_load_from_cache
|
||||
|
||||
# Store original function
|
||||
original_try_load = _try_to_load_from_cache
|
||||
|
||||
def _patched_try_to_load_from_cache(
|
||||
repo_id: str,
|
||||
filename: str,
|
||||
cache_dir: Union[str, Path, None] = None,
|
||||
revision: Optional[str] = None,
|
||||
repo_type: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Patched version that forces offline mode.
|
||||
Returns None if not cached (instead of making network request).
|
||||
"""
|
||||
# Always use the original function, but we're already in HF_HUB_OFFLINE mode
|
||||
result = original_try_load(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
cache_dir=cache_dir,
|
||||
revision=revision,
|
||||
repo_type=repo_type,
|
||||
)
|
||||
|
||||
if result is None:
|
||||
# File not in cache - log this for debugging
|
||||
cache_path = Path(hf_constants.HF_HUB_CACHE) / f"models--{repo_id.replace('/', '--')}"
|
||||
print(f"[HF_PATCH] File not cached: {repo_id}/{filename}")
|
||||
print(f"[HF_PATCH] Expected at: {cache_path}")
|
||||
else:
|
||||
print(f"[HF_PATCH] Cache hit: {repo_id}/{filename}")
|
||||
|
||||
return result
|
||||
|
||||
# Replace the function
|
||||
import huggingface_hub.file_download as fd
|
||||
fd._try_to_load_from_cache = _patched_try_to_load_from_cache
|
||||
|
||||
print("[HF_PATCH] huggingface_hub patched for offline mode")
|
||||
|
||||
except ImportError:
|
||||
print("[HF_PATCH] huggingface_hub not found, skipping patch")
|
||||
except Exception as e:
|
||||
print(f"[HF_PATCH] Error patching huggingface_hub: {e}")
|
||||
|
||||
|
||||
def ensure_original_qwen_config_cached():
|
||||
"""
|
||||
The MLX community model is based on the original Qwen model.
|
||||
mlx_audio may try to fetch config from the original repo.
|
||||
We need to ensure that config is available in the cache.
|
||||
"""
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
# Original Qwen model that mlx_audio might reference
|
||||
original_repo = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
mlx_repo = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
||||
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
|
||||
original_path = cache_dir / f"models--{original_repo.replace('/', '--')}"
|
||||
mlx_path = cache_dir / f"models--{mlx_repo.replace('/', '--')}"
|
||||
|
||||
# If original repo cache doesn't exist but MLX does, create a symlink or copy config
|
||||
if not original_path.exists() and mlx_path.exists():
|
||||
print(f"[HF_PATCH] Original repo not cached, but MLX version is")
|
||||
print(f"[HF_PATCH] Creating symlink from {original_repo} -> {mlx_repo}")
|
||||
|
||||
try:
|
||||
# Create a symlink so the cache lookup succeeds
|
||||
original_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
original_path.symlink_to(mlx_path, target_is_directory=True)
|
||||
print(f"[HF_PATCH] Symlink created successfully")
|
||||
except Exception as e:
|
||||
print(f"[HF_PATCH] Could not create symlink: {e}")
|
||||
|
||||
|
||||
# Auto-apply patch when module is imported
|
||||
if os.environ.get("VOICEBOX_OFFLINE_PATCH", "1") != "0":
|
||||
patch_huggingface_hub_offline()
|
||||
ensure_original_qwen_config_cached()
|
||||
@@ -4,6 +4,10 @@
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "voicebox",
|
||||
"dependencies": {
|
||||
"loaders.css": "^0.1.2",
|
||||
"react-loaders": "^3.0.1",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.12",
|
||||
"@types/node": "^20.0.0",
|
||||
@@ -13,7 +17,7 @@
|
||||
},
|
||||
"app": {
|
||||
"name": "@voicebox/app",
|
||||
"version": "0.1.11",
|
||||
"version": "0.1.13",
|
||||
"dependencies": {
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
@@ -68,7 +72,7 @@
|
||||
},
|
||||
"landing": {
|
||||
"name": "@voicebox/landing",
|
||||
"version": "0.1.11",
|
||||
"version": "0.1.13",
|
||||
"dependencies": {
|
||||
"@radix-ui/react-separator": "^1.1.8",
|
||||
"@radix-ui/react-slot": "^1.2.4",
|
||||
@@ -93,7 +97,7 @@
|
||||
},
|
||||
"tauri": {
|
||||
"name": "@voicebox/tauri",
|
||||
"version": "0.1.11",
|
||||
"version": "0.1.13",
|
||||
"dependencies": {
|
||||
"@tauri-apps/api": "^2.0.0",
|
||||
"@tauri-apps/plugin-dialog": "^2.0.0",
|
||||
@@ -116,7 +120,7 @@
|
||||
},
|
||||
"web": {
|
||||
"name": "@voicebox/web",
|
||||
"version": "0.1.11",
|
||||
"version": "0.1.13",
|
||||
"dependencies": {
|
||||
"@tanstack/react-query": "^5.0.0",
|
||||
"react": "^18.3.0",
|
||||
@@ -125,6 +129,7 @@
|
||||
"zustand": "^4.5.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@typescript-eslint/eslint-plugin": "^7.0.0",
|
||||
@@ -677,6 +682,8 @@
|
||||
|
||||
"class-variance-authority": ["[email protected]", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="],
|
||||
|
||||
"classnames": ["[email protected]", "", {}, "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="],
|
||||
|
||||
"client-only": ["[email protected]", "", {}, "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="],
|
||||
|
||||
"clsx": ["[email protected]", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
|
||||
@@ -873,6 +880,8 @@
|
||||
|
||||
"lines-and-columns": ["[email protected]", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="],
|
||||
|
||||
"loaders.css": ["[email protected]", "", {}, "sha512-Rhowlq24ey1VOeor+3wYOt9+MjaxBOJm1u4KlQgNC3+0xJ0LS4wq4iG57D/BPzvuD/7HHDGQOWJ+81oR2EI9bQ=="],
|
||||
|
||||
"locate-path": ["[email protected]", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="],
|
||||
|
||||
"lodash.merge": ["[email protected]", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="],
|
||||
@@ -959,6 +968,8 @@
|
||||
|
||||
"prelude-ls": ["[email protected]", "", {}, "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g=="],
|
||||
|
||||
"prop-types": ["[email protected]", "", { "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", "react-is": "^16.13.1" } }, "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg=="],
|
||||
|
||||
"punycode": ["[email protected]", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
|
||||
|
||||
"queue-microtask": ["[email protected]", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="],
|
||||
@@ -969,6 +980,10 @@
|
||||
|
||||
"react-hook-form": ["[email protected]", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-9SUJKCGKo8HUSsCO+y0CtqkqI5nNuaDqTxyqPsZPqIwudpj4rCrAz/jZV+jn57bx5gtZKOh3neQu94DXMc+w5w=="],
|
||||
|
||||
"react-is": ["[email protected]", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="],
|
||||
|
||||
"react-loaders": ["[email protected]", "", { "dependencies": { "classnames": "^2.2.3" }, "peerDependencies": { "prop-types": ">=15.6.0", "react": ">=15" } }, "sha512-4igMNqs9Fb3d4Z+0UHIGQNJsw/37gX0nUO8QxupnEKRn1dtyYC1LGwk5GuaoDciMQCQc/MmPwb4Fn6ZfdoX1FQ=="],
|
||||
|
||||
"react-refresh": ["[email protected]", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
||||
|
||||
"react-remove-scroll": ["[email protected]", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="],
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
services:
|
||||
voicebox:
|
||||
build: .
|
||||
container_name: voicebox
|
||||
restart: unless-stopped
|
||||
|
||||
ports:
|
||||
# Bind to localhost only for security
|
||||
- "127.0.0.1:17493:17493"
|
||||
|
||||
volumes:
|
||||
# Bind-mount for generated audio (customize the host path as needed)
|
||||
# Host side: ./output/
|
||||
# Container side: /app/data/generations/
|
||||
- ./output:/app/data/generations
|
||||
|
||||
# Named volume for profiles, DB, cache (persists across container restarts)
|
||||
- voicebox-data:/app/data
|
||||
|
||||
# HuggingFace model cache (so models aren't re-downloaded on rebuild)
|
||||
- huggingface-cache:/home/voicebox/.cache/huggingface
|
||||
|
||||
environment:
|
||||
- LOG_LEVEL=info
|
||||
|
||||
networks:
|
||||
- voicebox-net
|
||||
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '4'
|
||||
memory: 8G
|
||||
|
||||
networks:
|
||||
voicebox-net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
voicebox-data:
|
||||
huggingface-cache:
|
||||
@@ -0,0 +1,70 @@
|
||||
# Accessibility: screen reader and keyboard improvements
|
||||
|
||||
## Summary
|
||||
|
||||
Improvements to support screen reader and keyboard users across the main app surfaces: audio player, generation UI, voice selection, history, voices tab, model management, server tab, and stories.
|
||||
|
||||
**Tested with NVDA and Narrator on Windows.**
|
||||
|
||||
---
|
||||
|
||||
## What changed
|
||||
|
||||
### Audio player (after generating audio)
|
||||
|
||||
- **Play/Pause, Loop, Mute, Close** – `aria-label` added so each control is announced (e.g. "Play", "Pause", "Loop", "Mute", "Close player").
|
||||
- **Playback position slider** – `aria-label="Playback position"` and `aria-valuetext` with current/total time (e.g. "0:30 of 2:15").
|
||||
- **Volume** – Wrapped in a labelled group; volume slider has an associated screen-reader-only label and `aria-valuetext` for the level (e.g. "Volume level, 75%").
|
||||
|
||||
### Generation UI (text box and voice choice)
|
||||
|
||||
- **Generate speech** (submit) and **Fine-tune instructions** (sliders) – Icon buttons now have `aria-label` (and state for fine-tune, e.g. "Fine-tune instructions, on").
|
||||
|
||||
### Voice selection (cards on Generate screen)
|
||||
|
||||
- Each **voice card** is focusable (`tabIndex={0}`), has `role="button"`, and an `aria-label` (e.g. "Prashant, en. Select as voice for generation.") with `aria-pressed` when selected.
|
||||
- **Enter/Space** on the card selects that voice; tab order is card → Export/Edit/Delete.
|
||||
|
||||
### History list (generated samples)
|
||||
|
||||
- Each **sample row** is focusable with `role="button"` and an `aria-label` (e.g. "Sample from [profile], [duration], [date]. Press Enter to play."); **Enter/Space** plays or restarts.
|
||||
- **Transcript textarea** has `aria-label` (e.g. "Transcript for sample from [profile], [duration]") so when you focus on the text area, the sample is announced in context.
|
||||
|
||||
### Voices tab (table)
|
||||
|
||||
- Each **voice row** is focusable with `role="button"` and an `aria-label` (e.g. "[Name], [language], [N] generations, [N] samples. Press Enter to edit."); **Enter/Space** opens edit (except when focus is in a control).
|
||||
- **Actions** dropdown trigger has `aria-label="Actions for [profile name]"`.
|
||||
|
||||
### Model management
|
||||
|
||||
- Each **model row** is a focusable region (`tabIndex={0}`, `role="group"`) with an `aria-label` (e.g. "[Model name], [status], [size]. Use Tab to reach Download or Delete.").
|
||||
- **Download** and **Delete** (and Downloading) buttons have `aria-label` (e.g. "Download [name]", "Delete [name]").
|
||||
|
||||
### Server tab (panels)
|
||||
|
||||
- **Server Connection**, **Server Status**, and **App Updates** cards are landmarks: `role="region"`, `aria-label`, and `tabIndex={0}` so each panel is focusable and announced (e.g. "Server Connection", "Server Status", "App Updates").
|
||||
|
||||
### Stories list
|
||||
|
||||
- Each **story row** is a focusable control (`role="button"`, `tabIndex={0}`) with `aria-label` (e.g. "Story [name], [N] items, [date]. Press Enter to select."); **Enter/Space** selects the story. Actions button has `aria-label="Actions for [story name]"`.
|
||||
|
||||
### Other controls
|
||||
|
||||
- **Story list** – Actions (⋮) button: `aria-label="Actions for [story name]"`.
|
||||
- **Story track editor** – Play/Pause, Stop, Split, Duplicate, Delete, Zoom in/out: `aria-label` on all icon buttons.
|
||||
- **Voice profile samples** (SampleList, AudioSampleUpload, AudioSampleRecording, AudioSampleSystem) – Play/Pause and Stop: `aria-label` (e.g. "Play sample", "Pause", "Stop playback").
|
||||
- **SampleList** mini sample player – Seek slider has `aria-label="Sample playback position"` and `aria-valuetext` for time.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- **Screen readers:** Tested with **NVDA** and **Narrator** on Windows.
|
||||
- **Keyboard:** Tab order and Enter/Space activation verified for focusable rows and buttons.
|
||||
|
||||
---
|
||||
|
||||
## Tech note
|
||||
|
||||
- React + TypeScript; Radix UI primitives; labels added via `aria-label`, `aria-labelledby`, `aria-valuetext`, and `role`/`tabIndex` where needed.
|
||||
- No new dependencies.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Voicebox Issue Pain Points (Snapshot)
|
||||
|
||||
## Scope
|
||||
|
||||
- Dataset: **128 total issues** (**107 open**, **21 closed**)
|
||||
- Source: GitHub issues in `jamiepine/voicebox`
|
||||
- Classification: keyword/theme clustering
|
||||
- Note: counts below are **non-exclusive** (one issue can belong to multiple pain points)
|
||||
|
||||
## Most Common Pain Points (Open Issues)
|
||||
|
||||
| Rank | Pain Point | Open Issues | What users are reporting |
|
||||
|---|---|---:|---|
|
||||
| 1 | Model download & offline reliability | **32** | Downloads failing/stalling, cache/offline behavior inconsistent, wrong model size selected, Errno issues |
|
||||
| 2 | GPU/backend compatibility | **22** | GPU not detected, backend fallback surprises, platform-specific runtime failures (Windows/Mac) |
|
||||
| 3 | Export/save/file persistence | **15** | Export fails, "failed to fetch/download audio", samples/profiles not saving |
|
||||
| 4 | Language/accent quality & coverage | **14** | Missing language support, accent mismatch, robotic outputs |
|
||||
| 5 | Update/restart safety + long-op controls | **4** | Auto-restart without warning, update confusion, lack of cancel/pause controls |
|
||||
|
||||
## Representative Issues by Pain Point
|
||||
|
||||
### 1) Model download & offline reliability (32)
|
||||
|
||||
- [#159](https://github.com/jamiepine/voicebox/issues/159) - Qwen download fails with Errno 22
|
||||
- [#151](https://github.com/jamiepine/voicebox/issues/151) - Model loading hangs / server crashes
|
||||
- [#150](https://github.com/jamiepine/voicebox/issues/150) - Internet required despite downloaded models
|
||||
- [#149](https://github.com/jamiepine/voicebox/issues/149) - Cancel/pause controls for large downloads
|
||||
- [#96](https://github.com/jamiepine/voicebox/issues/96) - 0.6B selection still uses/downloads 1.7B
|
||||
|
||||
### 2) GPU/backend compatibility (22)
|
||||
|
||||
- [#164](https://github.com/jamiepine/voicebox/issues/164) - Windows: no GPU usage + multiple breakages
|
||||
- [#141](https://github.com/jamiepine/voicebox/issues/141) - Using CPU only, GPU not used
|
||||
- [#131](https://github.com/jamiepine/voicebox/issues/131) - Numpy ABI mismatch in bundled app
|
||||
- [#130](https://github.com/jamiepine/voicebox/issues/130) - Intel Mac tensor/padding generation error
|
||||
- [#127](https://github.com/jamiepine/voicebox/issues/127) - GPU not found
|
||||
|
||||
### 3) Export/save/file persistence (15)
|
||||
|
||||
- [#148](https://github.com/jamiepine/voicebox/issues/148) - Japanese export fails on 0.1.12
|
||||
- [#143](https://github.com/jamiepine/voicebox/issues/143) - Samples not saving
|
||||
- [#134](https://github.com/jamiepine/voicebox/issues/134) - Can't save profile
|
||||
- [#105](https://github.com/jamiepine/voicebox/issues/105) - Export audio fails (failed to fetch)
|
||||
- [#49](https://github.com/jamiepine/voicebox/issues/49) - Export filename/location ignored on Windows
|
||||
|
||||
### 4) Language/accent quality & coverage (14)
|
||||
|
||||
- [#162](https://github.com/jamiepine/voicebox/issues/162) - Persian audio request/problem
|
||||
- [#117](https://github.com/jamiepine/voicebox/issues/117) - Arabic language support
|
||||
- [#113](https://github.com/jamiepine/voicebox/issues/113) - Polish language support
|
||||
- [#109](https://github.com/jamiepine/voicebox/issues/109) - Ukrainian support
|
||||
- [#100](https://github.com/jamiepine/voicebox/issues/100) - Non-US accent quality issues
|
||||
|
||||
### 5) Update/restart safety + controls (4)
|
||||
|
||||
- [#164](https://github.com/jamiepine/voicebox/issues/164) - Update behavior + usability failures
|
||||
- [#136](https://github.com/jamiepine/voicebox/issues/136) - Auto-restart without warning
|
||||
- [#86](https://github.com/jamiepine/voicebox/issues/86) - Unexpected restart with no confirmation
|
||||
- [#149](https://github.com/jamiepine/voicebox/issues/149) - Need pause/cancel and pre-download confirmation
|
||||
|
||||
## Additional Signal
|
||||
|
||||
- There is also a large **feature-request/misc** bucket (**36 open**) that is competing with stability triage (audiobook, Linux build, additional ASR/TTS models, integrations).
|
||||
|
||||
## Takeaway
|
||||
|
||||
Most user pain is concentrated in four stability areas: **download/offline path**, **GPU/backend detection**, **save/export reliability**, and **language/accent correctness**. Addressing those first should reduce the majority of current support friction.
|
||||
+222
-194
@@ -1,6 +1,6 @@
|
||||
# Voicebox Project Status & Roadmap
|
||||
|
||||
> Last updated: 2026-03-12 | Current version: **v0.1.13** | 13.1k stars | 176 open issues | 28 open PRs
|
||||
> Last updated: 2026-03-13 | Current version: **v0.1.13** | 13.1k stars | ~176 open issues | 25 open PRs
|
||||
|
||||
---
|
||||
|
||||
@@ -30,14 +30,18 @@
|
||||
│ │ HTTP :17493 │
|
||||
│ ┌──────────────────────▼────────────────────────┐ │
|
||||
│ │ FastAPI Backend (backend/) │ │
|
||||
│ │ ┌─────────────┐ ┌───────────┐ ┌─────────┐ │ │
|
||||
│ │ │ TTSBackend │ │ STTBackend│ │ Profiles│ │ │
|
||||
│ │ │ (Protocol) │ │ (Whisper) │ │ History │ │ │
|
||||
│ │ │ ┌────────┐ │ └───────────┘ │ Stories │ │ │
|
||||
│ │ │ │PyTorch │ │ └─────────┘ │ │
|
||||
│ │ │ │or MLX │ │ │ │
|
||||
│ │ │ └────────┘ │ │ │
|
||||
│ │ └─────────────┘ │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ TTSBackend Protocol │ │ │
|
||||
│ │ │ ┌──────────┐ ┌───────┐ ┌───────────┐ │ │ │
|
||||
│ │ │ │ Qwen3-TTS│ │LuxTTS │ │Chatterbox │ │ │ │
|
||||
│ │ │ │(Py/MLX) │ │ │ │(MTL+Turbo)│ │ │ │
|
||||
│ │ │ └──────────┘ └───────┘ └───────────┘ │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ ┌───────────┐ ┌─────────┐ │ │
|
||||
│ │ │ STTBackend│ │ Profiles│ │ │
|
||||
│ │ │ (Whisper) │ │ History │ │ │
|
||||
│ │ └───────────┘ │ Stories │ │ │
|
||||
│ │ └─────────┘ │ │
|
||||
│ └───────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
@@ -46,131 +50,180 @@
|
||||
|
||||
| Layer | File | Purpose |
|
||||
|-------|------|---------|
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~1700 lines) |
|
||||
| Backend entry | `backend/main.py` | FastAPI app, all API routes (~2100 lines) |
|
||||
| TTS protocol | `backend/backends/__init__.py:14-81` | `TTSBackend` Protocol definition |
|
||||
| TTS factory | `backend/backends/__init__.py:118-137` | Singleton backend selection (MLX vs PyTorch) |
|
||||
| TTS factory | `backend/backends/__init__.py:138-178` | Thread-safe engine registry (double-checked locking) |
|
||||
| PyTorch TTS | `backend/backends/pytorch_backend.py` | Qwen3-TTS via `qwen_tts` package |
|
||||
| MLX TTS | `backend/backends/mlx_backend.py` | Qwen3-TTS via `mlx_audio.tts` |
|
||||
| LuxTTS | `backend/backends/luxtts_backend.py` | LuxTTS — fast, CPU-friendly |
|
||||
| Chatterbox MTL | `backend/backends/chatterbox_backend.py` | Chatterbox Multilingual — 23 languages |
|
||||
| Chatterbox Turbo | `backend/backends/chatterbox_turbo_backend.py` | Chatterbox Turbo — English, paralinguistic tags |
|
||||
| Platform detect | `backend/platform_detect.py` | Apple Silicon → MLX, else → PyTorch |
|
||||
| API types | `backend/models.py` | Pydantic request/response models |
|
||||
| HF progress | `backend/utils/hf_progress.py` | HFProgressTracker (tqdm patching for download progress) |
|
||||
| Audio utils | `backend/utils/audio.py` | `trim_tts_output()`, normalize, load/save audio |
|
||||
| Frontend API | `app/src/lib/api/client.ts` | Hand-written fetch wrapper |
|
||||
| Frontend types | `app/src/lib/api/types.ts` | TypeScript API types |
|
||||
| Generation form | `app/src/components/Generation/GenerationForm.tsx` | TTS generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status UI |
|
||||
| Floating gen box | `app/src/components/Generation/FloatingGenerateBox.tsx` | Compact generation UI |
|
||||
| Model manager | `app/src/components/ServerSettings/ModelManagement.tsx` | Model download/status/progress UI |
|
||||
| GPU acceleration | `app/src/components/ServerSettings/GpuAcceleration.tsx` | CUDA backend swap UI |
|
||||
| Gen form hook | `app/src/lib/hooks/useGenerationForm.ts` | Form validation + submission |
|
||||
| Language constants | `app/src/lib/constants/languages.ts` | Per-engine language maps |
|
||||
|
||||
### How TTS Generation Works (Current Flow)
|
||||
|
||||
```
|
||||
POST /generate
|
||||
1. Look up voice profile from DB
|
||||
2. Check model cache → if missing, trigger background download, return HTTP 202
|
||||
3. Load model (lazy): tts_backend.load_model(model_size)
|
||||
4. Create voice prompt: profiles.create_voice_prompt_for_profile()
|
||||
2. Resolve engine from request (qwen | luxtts | chatterbox | chatterbox_turbo)
|
||||
3. Get backend: get_tts_backend_for_engine(engine) # thread-safe singleton per engine
|
||||
4. Check model cache → if missing, trigger background download, return HTTP 202
|
||||
5. Load model (lazy): tts_backend.load_model(model_size)
|
||||
6. Create voice prompt: profiles.create_voice_prompt_for_profile(engine=engine)
|
||||
→ tts_backend.create_voice_prompt(audio_path, reference_text)
|
||||
5. Generate: tts_backend.generate(text, voice_prompt, language, seed, instruct)
|
||||
6. Save WAV → data/generations/{id}.wav
|
||||
7. Insert history record in SQLite
|
||||
8. Return GenerationResponse
|
||||
7. Generate: tts_backend.generate(text, voice_prompt, language, seed, instruct)
|
||||
8. Post-process: trim_tts_output() for Chatterbox engines
|
||||
9. Save WAV → data/generations/{id}.wav
|
||||
10. Insert history record in SQLite
|
||||
11. Return GenerationResponse
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Current State
|
||||
|
||||
### What's Shipped (v0.1.13)
|
||||
### What's Shipped (v0.1.13 + recent merges)
|
||||
|
||||
**Core TTS:**
|
||||
- Qwen3-TTS voice cloning (1.7B and 0.6B models)
|
||||
- MLX backend for Apple Silicon, PyTorch for everything else
|
||||
- Multi-engine TTS architecture with thread-safe backend registry (PR #254)
|
||||
- LuxTTS integration — fast, CPU-friendly English TTS (PR #254)
|
||||
- Chatterbox Multilingual TTS — 23 languages including Hebrew (PR #257)
|
||||
- Delivery instructions (instruct parameter, Qwen only)
|
||||
- Single flat model dropdown (Qwen 1.7B, Qwen 0.6B, LuxTTS, Chatterbox, Chatterbox Turbo)
|
||||
|
||||
**Infrastructure:**
|
||||
- CUDA backend swap via binary download and restart (PR #252)
|
||||
- GPU acceleration settings UI
|
||||
- Voice profiles with multi-sample support
|
||||
- Stories editor (multi-track DAW timeline)
|
||||
- Whisper transcription (base, small, medium, large variants)
|
||||
- Model management UI with download progress (SSE)
|
||||
- Model management UI with inline download progress bars (HFProgressTracker)
|
||||
- Download cancel/clear UI with error panel (PR #238)
|
||||
- Generation history with caching
|
||||
- Streaming generation endpoint (MLX only)
|
||||
- Delivery instructions (instruct parameter)
|
||||
- Duplicate profile name validation (PR #175)
|
||||
- Linux NVIDIA GBM buffer + WebKitGTK microphone fix (PR #210)
|
||||
|
||||
### What's NOT Shipped But Has Code
|
||||
### What's In-Flight
|
||||
|
||||
| Feature | Branch | Status |
|
||||
|---------|--------|--------|
|
||||
| External provider binaries (CUDA split) | `external-provider-binaries` | PR #33, significant work done, stale since Feb |
|
||||
| Dual server binaries | `feat/dual-server-binaries` | Branch exists, no PR |
|
||||
| Multi-sample fix | `fix-multi-sample` | Branch exists, no PR |
|
||||
| Model download notification fix | `fix-dl-notification-...` | Branch exists, no PR |
|
||||
| Feature | Branch/PR | Status |
|
||||
|---------|-----------|--------|
|
||||
| Chatterbox Turbo + per-engine language lists | `feat/chatterbox-turbo` / PR #258 | Open, ready for review |
|
||||
|
||||
### Hardcoded Qwen3-TTS Assumptions
|
||||
### TTS Engine Comparison
|
||||
|
||||
These are the specific coupling points that block multi-model support:
|
||||
| Engine | Model Name | Languages | Size | Key Features |
|
||||
|--------|-----------|-----------|------|-------------|
|
||||
| Qwen3-TTS 1.7B | `qwen-tts-1.7B` | 10 (zh, en, ja, ko, de, fr, ru, pt, es, it) | ~3.5 GB | Instruct mode, highest quality |
|
||||
| Qwen3-TTS 0.6B | `qwen-tts-0.6B` | 10 | ~1.2 GB | Lighter, faster |
|
||||
| LuxTTS | `luxtts` | English | ~300 MB | CPU-friendly, 48 kHz, fast |
|
||||
| Chatterbox | `chatterbox-tts` | 23 (incl. Hebrew, Arabic, Hindi, etc.) | ~3.2 GB | Zero-shot cloning, multilingual |
|
||||
| Chatterbox Turbo | `chatterbox-turbo` | English | ~1.5 GB | Paralinguistic tags ([laugh], [cough]), 350M params, low latency |
|
||||
|
||||
| Location | What's Hardcoded |
|
||||
|----------|-----------------|
|
||||
| `backend/models.py:58` | `model_size` regex: `^(1\.7B\|0\.6B)$` |
|
||||
| `backend/main.py:611` | Default: `model_size or "1.7B"` |
|
||||
| `backend/main.py:1322-1365` | Model status list (2 Qwen + 4 Whisper) |
|
||||
| `backend/main.py:1523-1548` | Download trigger map |
|
||||
| `backend/main.py:1597-1628` | Delete map |
|
||||
| `backend/backends/pytorch_backend.py:65-68` | HF repo ID map |
|
||||
| `backend/backends/mlx_backend.py:41-44` | MLX repo ID map |
|
||||
| `backend/backends/__init__.py:118-137` | Single global TTS backend |
|
||||
| `app/src/lib/hooks/useGenerationForm.ts:17` | `modelSize: z.enum(['1.7B', '0.6B'])` |
|
||||
| `app/src/lib/hooks/useGenerationForm.ts:70-71` | `modelName = "qwen-tts-${data.modelSize}"` |
|
||||
| `app/src/components/Generation/GenerationForm.tsx:140-141` | Hardcoded "Qwen TTS" labels |
|
||||
| `app/src/components/ServerSettings/ModelManagement.tsx:166-213` | Filters by `qwen-tts` and `whisper` prefix |
|
||||
| `backend/utils/cache.py` | Voice prompt cache uses `torch.save()` |
|
||||
### Multi-Engine Architecture (Shipped)
|
||||
|
||||
The singleton TTS backend blocker described in the previous version of this doc has been **resolved**. The architecture now supports:
|
||||
|
||||
- **Thread-safe backend registry** (`_tts_backends` dict + `_tts_backends_lock`) with double-checked locking
|
||||
- **Per-engine backend instances** — each engine gets its own singleton, loaded lazily
|
||||
- **Engine field on GenerationRequest** — frontend sends `engine: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo'`
|
||||
- **Per-engine language filtering** — `ENGINE_LANGUAGES` map in frontend, backend regex accepts all languages
|
||||
- **Per-engine voice prompts** — `create_voice_prompt_for_profile()` dispatches to the correct backend
|
||||
- **Trim post-processing** — `trim_tts_output()` for Chatterbox engines (cuts trailing silence/hallucination)
|
||||
|
||||
### Known Limitations
|
||||
|
||||
- **HF XET progress**: Large files downloaded via `hf-xet` (HuggingFace's new transfer backend) report `n=0` in tqdm updates. Progress bars may appear stuck for large `.safetensors` files even though the download is proceeding. This is a known upstream limitation.
|
||||
- **Chatterbox Turbo upstream token bug**: `from_pretrained()` passes `token=os.getenv("HF_TOKEN") or True` which fails without a stored HF token. Our backend works around this by calling `snapshot_download(token=None)` + `from_local()`.
|
||||
- **chatterbox-tts must install with `--no-deps`**: It pins `numpy<1.26`, `torch==2.6.0`, `transformers==4.46.3` — all incompatible with our stack (Python 3.12, torch 2.10, transformers 4.57.3). Sub-deps listed explicitly in `requirements.txt`.
|
||||
- **Streaming generation** only works for Qwen on MLX. Other engines use the non-streaming `/generate` endpoint.
|
||||
- **dicta-onnx** (Hebrew diacritization) not included — upstream Chatterbox bug requires `model_path` arg but calls `Dicta()` with none. Hebrew works fine without it.
|
||||
|
||||
---
|
||||
|
||||
## Open PRs — Triage & Analysis
|
||||
|
||||
### Recently Merged (Since Last Update)
|
||||
|
||||
| PR | Title | Merged |
|
||||
|----|-------|--------|
|
||||
| **#257** | feat: Chatterbox TTS engine with multilingual voice cloning | 2026-03-13 |
|
||||
| **#254** | feat: LuxTTS integration — multi-engine TTS support | 2026-03-13 |
|
||||
| **#252** | feat: CUDA backend swap via binary download and restart | 2026-03-13 |
|
||||
| **#238** | Download cancel/clear UI, fixed model downloading | 2026-03-13 |
|
||||
| **#250** | docs: align local API port examples | 2026-03-13 |
|
||||
| **#210** | fix: Linux NVIDIA GBM buffer crash | 2026-03-13 |
|
||||
| **#175** | Fix #134: duplicate profile name validation | 2026-03-13 |
|
||||
|
||||
### In-Flight (Our Work)
|
||||
|
||||
| PR | Title | Status | Notes |
|
||||
|----|-------|--------|-------|
|
||||
| **#258** | feat: Chatterbox Turbo engine + per-engine language lists | Open | Ready for review. Adds Turbo engine + dynamic language dropdown. |
|
||||
|
||||
### Merge-Ready / Near-Ready (Bug Fixes & Small Features)
|
||||
|
||||
| PR | Title | Risk | Notes |
|
||||
|----|-------|------|-------|
|
||||
| **#250** | docs: align local API port examples | None | Docs-only |
|
||||
| **#230** | docs: fix README grammar | None | Docs-only |
|
||||
| **#243** | a11y: screen reader and keyboard improvements | Low | Accessibility, no backend changes |
|
||||
| **#175** | Fix #134: duplicate profile name validation | Low | Simple validation |
|
||||
| **#178** | Fix #168 #140: generation error handling | Low | Error handling improvements |
|
||||
| **#152** | Fix: prevent crashes when HuggingFace unreachable | Medium | Monkey-patches HF hub; solves real offline bug (#150, #151) |
|
||||
| **#218** | fix: unify qwen tts cache dir on Windows | Low | Windows-specific path fix |
|
||||
| **#214** | fix: panic on launch from tokio::spawn | Low | Rust-side Tauri fix |
|
||||
| **#210** | fix: Linux NVIDIA GBM buffer crash | Low | Linux-specific, narrowly scoped |
|
||||
| **#88** | security: restrict CORS to known local origins | Low | Security hardening |
|
||||
| **#133** | feat: network access toggle | Low | Wires up existing plumbing |
|
||||
|
||||
### Significant Feature PRs
|
||||
|
||||
| PR | Title | Complexity | Dependencies | Notes |
|
||||
|----|-------|-----------|--------------|-------|
|
||||
| **#97** | fix: pass language parameter to TTS models | Medium | None | **Critical bug** — language param was silently dropped. Adds `LANGUAGE_CODE_TO_NAME` mapping to both backends. Should be high priority. |
|
||||
| **#133** | feat: network access toggle | Low | None | Wires up existing plumbing (`--host 0.0.0.0`). Clean, small. |
|
||||
| **#238** | download cancel/clear UI + error panel | Medium | None | Adds cancel buttons, VS Code-style Problems panel, fixes whisper-large repo. Quality-of-life win. |
|
||||
| **#99** | feat: chunked TTS with quality selector | Medium | None | Solves the 500-char/2048-token limit. Sentence-aware splitting, crossfade concat, 44.1kHz upsampling. Addresses #191, #203, #69, #111. |
|
||||
| **#154** | feat: Audiobook tab | Medium | Depends on #99 concepts | Full audiobook workflow — chunked gen, preview, auto-save to Stories. New route + tab. |
|
||||
| **#91** | fix: CoreAudio device enumeration | Medium | None | macOS audio device handling. |
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#253** | Enhance speech tokenizer with 48kHz version | Medium | Qwen tokenizer upgrade |
|
||||
| **#97** | fix: pass language parameter to TTS models | Medium | May be partially obsoleted by multi-engine work — needs review |
|
||||
| **#99** | feat: chunked TTS with quality selector | Medium | Solves 500-char limit. Addresses #191, #203, #69, #111. |
|
||||
| **#154** | feat: Audiobook tab | Medium | Full audiobook workflow. Depends on #99 concepts. |
|
||||
| **#91** | fix: CoreAudio device enumeration | Medium | macOS audio device handling |
|
||||
|
||||
### Architectural PRs (Need Careful Review)
|
||||
|
||||
| PR | Title | Complexity | Notes |
|
||||
|----|-------|-----------|-------|
|
||||
| **#33** | CUDA GPU Support — External Provider Binaries | **Very High** | The big one. Splits monolithic backend into main app + downloadable provider executables (PyTorch CPU, CUDA). New provider management system, CI/CD for R2 uploads, provider settings UI. Created Feb 1, significant codebase. **This is the foundation for multi-model support** but is currently Qwen-only. |
|
||||
| **#225** | feat: custom HuggingFace model support | High | Adds `custom_models.py`, `custom:<slug>` model IDs, frontend model grouping (Built-in vs Custom). **Takes a different approach than #33** — keeps single backend but allows arbitrary HF repos. These two PRs may conflict architecturally. |
|
||||
| **#194** | feat: Hebrew + Chatterbox TTS | High | **First non-Qwen TTS model.** Adds `ChatterboxTTSBackend` alongside existing backends. Routes by language (`he` → Chatterbox, else → Qwen). Adds Hebrew Whisper models. Includes a lot of cleanup. Important precedent for multi-model. |
|
||||
| **#195** | feat: per-profile LoRA fine-tuning | **Very High** | Depends on #194. Training pipeline, adapter management, SSE progress, 15 new API endpoints. New DB tables. Forces PyTorch even on MLX systems for adapter inference. |
|
||||
| **#161** | feat: Docker + web deployment | High | 3-stage Dockerfile, SPA serving from FastAPI, docker-compose. Implements the Docker deployment plan. |
|
||||
| **#124** | Add Dockerfiles + docker-compose + docs | Medium | Earlier, simpler Docker attempt. Overlaps with #161. |
|
||||
| **#123** | added docker | Low | Minimal Docker PR. Overlaps with #161 and #124. |
|
||||
| **#227** | fix: harden input validation & file safety | Medium | Follow-up to #225. Atomic writes, threading locks, input validation. Good hardening but coupled to the custom models feature. |
|
||||
| **#225** | feat: custom HuggingFace model support | High | Arbitrary HF repo loading. May need rework given multi-engine arch is now shipped. |
|
||||
| **#194** | feat: Hebrew + Chatterbox TTS | High | **Superseded** by PR #257 which shipped Chatterbox multilingual (23 langs incl. Hebrew). May be closeable. |
|
||||
| **#195** | feat: per-profile LoRA fine-tuning | Very High | Training pipeline, adapter management, 15 new endpoints. Depends on #194 (now superseded). |
|
||||
| **#161** | feat: Docker + web deployment | High | 3-stage Dockerfile, SPA serving. Independent of TTS engine work. |
|
||||
| **#124** / **#123** | Docker (simpler attempts) | Low-Medium | Overlap with #161 |
|
||||
| **#227** | fix: harden input validation & file safety | Medium | Coupled to #225 (custom models) |
|
||||
|
||||
### PRs That Need Author Action / Are Stale
|
||||
|
||||
| PR | Title | Notes |
|
||||
|----|-------|-------|
|
||||
| **#237** | fix: bundle qwen_tts source files in PyInstaller | Solves #212 but needs review for build system impact |
|
||||
| **#237** | fix: bundle qwen_tts source files in PyInstaller | Build system, needs review |
|
||||
| **#215** | Update prerequisites with Tauri deps | Branch is `main` — will have conflicts |
|
||||
| **#89** | Linux Support | Branch is `main` — will have conflicts. Broad scope. |
|
||||
| **#83** | Update download links for v0.1.12 | Outdated (we're on v0.1.13) |
|
||||
|
||||
### PRs Likely Superseded
|
||||
|
||||
| PR | Superseded By | Notes |
|
||||
|----|--------------|-------|
|
||||
| **#194** (Hebrew + Chatterbox) | PR #257 (merged) | #257 ships Chatterbox multilingual with 23 languages including Hebrew. #194 took a different approach (route by language). Can likely be closed. |
|
||||
| **#33** (External provider binaries) | PR #252 (merged) | #252 shipped CUDA backend swap. #33's broader provider architecture may still have value but needs reassessment. |
|
||||
|
||||
---
|
||||
|
||||
## Open Issues — Categorized
|
||||
@@ -186,15 +239,15 @@ The single most reported category. Users on Windows with NVIDIA GPUs frequently
|
||||
|
||||
**Key issues:** #239, #222, #220, #217, #208, #198, #192, #167, #164, #141, #130, #127
|
||||
|
||||
**Fix path:** PR #33 (external provider binaries) is designed to solve this. Ship a small main app, let users download the CUDA provider separately.
|
||||
**Fix path:** PR #252 (CUDA backend swap) is now merged. Users can download the CUDA binary separately from the GPU acceleration settings. Many of these issues may now be resolvable — needs triage to confirm.
|
||||
|
||||
### Model Downloads (20 issues)
|
||||
|
||||
Second most reported. Users get stuck downloads, can't resume, no cancel button, no offline fallback.
|
||||
Second most reported. Users get stuck downloads, can't resume, no offline fallback.
|
||||
|
||||
**Key issues:** #249, #240, #221, #216, #212, #181, #180, #159, #150, #149, #145, #143, #135, #134
|
||||
|
||||
**Fix path:** PR #238 (cancel/clear UI), PR #152 (offline crash fix). Resume support not yet addressed.
|
||||
**Fix path:** PR #238 (cancel/clear UI) is now merged. PR #152 (offline crash fix) still open. Inline progress bars now show for all engines. Resume support not yet addressed.
|
||||
|
||||
### Language Requests (18 issues)
|
||||
|
||||
@@ -202,7 +255,7 @@ Strong demand for: Hindi (#245), Indonesian (#247), Dutch (#236), Hebrew (#199),
|
||||
|
||||
**Key issues:** #247, #245, #236, #211, #205, #199, #189, #188, #187, #183, #179, #162
|
||||
|
||||
**Fix path:** PR #97 (pass language param — currently silently dropped!) is the prerequisite. Qwen3-TTS already supports many languages; the bug is that the language code isn't forwarded. Multi-model (#194 Chatterbox for Hebrew) expands coverage further.
|
||||
**Fix path:** Chatterbox Multilingual (merged via #257) now supports 23 languages including many of the requested ones: Arabic, Danish, German, Greek, Finnish, Hebrew, Hindi, Dutch, Norwegian, Polish, Swedish, Swahili, Turkish. Per-engine language filtering (PR #258) ensures the UI shows correct options. Several of these issues may be closeable.
|
||||
|
||||
### New Model Requests (5 explicit issues)
|
||||
|
||||
@@ -214,7 +267,7 @@ Strong demand for: Hindi (#245), Indonesian (#247), Dutch (#236), Hebrew (#199),
|
||||
| #132 | LavaSR (transcription) |
|
||||
| #76 | (General model expansion) |
|
||||
|
||||
Community is also vocally requesting: LuxTTS, Chatterbox, XTTS-v2, Fish Speech, CosyVoice, Kokoro on social media and in issue comments.
|
||||
Community also requests: XTTS-v2, Fish Speech, CosyVoice, Kokoro. The multi-engine architecture is now in place, making new model integration significantly easier.
|
||||
|
||||
### Long-Form / Chunking (5 issues)
|
||||
|
||||
@@ -255,153 +308,128 @@ Notable requests:
|
||||
|
||||
| Document | Target Version | Status | Relevance |
|
||||
|----------|---------------|--------|-----------|
|
||||
| `TTS_PROVIDER_ARCHITECTURE.md` | v0.1.13 | **Partially implemented** in PR #33 | Core architecture for multi-model + CUDA distribution |
|
||||
| `EXTERNAL_PROVIDERS.md` | v0.2.0 | **Not started** | Remote server support. API path inconsistency with provider arch doc (`/v1/` vs `/tts/`) |
|
||||
| `MLX_AUDIO.md` | — | **Shipped** (the only one) | MLX backend is live. 0.6B MLX model still missing. |
|
||||
| `DOCKER_DEPLOYMENT.md` | v0.2.0 | **PR exists** (#161) | Waiting on review. No official images published. |
|
||||
| `OPENAI_SUPPORT.md` | v0.2.0 | **Not started** | OpenAI-compatible API layer. Linked to issue #10. Low complexity. |
|
||||
|
||||
### Cross-Document Conflicts
|
||||
|
||||
1. **API path inconsistency:** Provider arch uses `/tts/generate`, External providers uses `/v1/generate`, OpenAI compat uses `/v1/audio/speech`. Need to reconcile.
|
||||
2. **Docker vs. Provider split:** Docker doc assumes monolithic backend. Provider arch splits into separate binaries. Need to decide: does Docker run the monolith or individual providers?
|
||||
3. **Version targeting:** Provider arch targets v0.1.13 (current!) but isn't merged. Everything else targets v0.2.0.
|
||||
| `TTS_PROVIDER_ARCHITECTURE.md` | v0.1.13 | **Partially superseded** by multi-engine arch + CUDA swap | Core concepts implemented differently than planned |
|
||||
| `CUDA_BACKEND_SWAP.md` | — | **Shipped** (PR #252) | CUDA binary download + backend restart |
|
||||
| `CUDA_BACKEND_SWAP_FINAL.md` | — | **Shipped** (PR #252) | Final implementation plan |
|
||||
| `EXTERNAL_PROVIDERS.md` | v0.2.0 | **Not started** | Remote server support |
|
||||
| `MLX_AUDIO.md` | — | **Shipped** | MLX backend is live |
|
||||
| `DOCKER_DEPLOYMENT.md` | v0.2.0 | **PR exists** (#161) | Waiting on review |
|
||||
| `OPENAI_SUPPORT.md` | v0.2.0 | **Not started** | OpenAI-compatible API layer |
|
||||
| `PR33_CUDA_PROVIDER_REVIEW.md` | — | **Reference** | Analysis of the original provider approach |
|
||||
|
||||
---
|
||||
|
||||
## New Model Integration — Landscape
|
||||
|
||||
### Models Worth Supporting (2026 SOTA)
|
||||
### Models Worth Supporting (2026 SOTA — updated March 13)
|
||||
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Repo |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|------|
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English-first | <1 GB | Easy | `ysharma3501/LuxTTS` |
|
||||
| **Chatterbox** | 5s zero-shot | Sub-200ms streaming | 24-48 kHz | 23+ | Low | Medium | `resemble-ai/chatterbox` |
|
||||
| **XTTS-v2** | 6s zero-shot | Fast mid-GPU | 24 kHz | 17+ | Medium | Medium | `coqui/XTTS-v2` |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Medium | `fishaudio/fish-speech` |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Easy | Alibaba HF org |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny | Medium | Kokoro repo |
|
||||
| Model | Cloning | Speed | Sample Rate | Languages | VRAM | Integration Ease | Status |
|
||||
|-------|---------|-------|-------------|-----------|------|-----------------|--------|
|
||||
| **Qwen3-TTS** | 10s zero-shot | Medium | 24 kHz | 10 | Medium | **Shipped** | v0.1.13 |
|
||||
| **LuxTTS** | 3s zero-shot | 150x RT, CPU ok | 48 kHz | English | <1 GB | **Shipped** | PR #254 |
|
||||
| **Chatterbox MTL** | 5s zero-shot | Medium | 24 kHz | 23 | Medium | **Shipped** | PR #257 |
|
||||
| **Chatterbox Turbo** | 5s zero-shot | Fast | 24 kHz | English | Low | **PR #258** | In review |
|
||||
| **HumeAI TADA 1B/3B** | Zero-shot | 5× faster than LLM-TTS | — | EN (1B), Multilingual (3B) | Medium | Needs vetting | MIT, 700s+ coherent, synced transcript output |
|
||||
| **MOSS-TTS Family** | Zero-shot | — | — | Multilingual | Medium | Needs vetting | Apache 2.0, multi-speaker dialogue, text-to-voice design (no ref audio) |
|
||||
| **VoxCPM 1.5** | Zero-shot (seconds) | ~0.15 RTF streaming | — | Bilingual (EN/ZH) | Medium | Needs vetting | Apache 2.0, tokenizer-free continuous diffusion, LoRA-friendly |
|
||||
| **Pocket TTS** | Zero-shot + streaming | >1× RT on CPU | — | English | ~100M params, CPU-first | Needs vetting | MIT, Kyutai Labs, no GPU required |
|
||||
| **Kokoro-82M** | 3s instant | CPU realtime | 24 kHz | English | Tiny (82M) | Ready | Apache 2.0, multi-engine arch in place |
|
||||
| **XTTS-v2** | 6s zero-shot | Mid-GPU | 24 kHz | 17+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **Fish Speech** | 10-30s few-shot | Real-time | 24-44 kHz | 50+ | Medium | Ready | Multi-engine arch in place |
|
||||
| **CosyVoice2-0.5B** | 3-10s zero-shot | Very fast | 24 kHz | Multilingual | Low | Ready | Multi-engine arch in place |
|
||||
|
||||
### What's Needed Architecturally for Multi-Model
|
||||
#### Notes on New Candidates (March 2026)
|
||||
|
||||
The current codebase assumes one TTS model family (Qwen3-TTS). Adding any new model requires:
|
||||
- **HumeAI TADA** — Text-Audio Dual Alignment arch. Near-zero hallucinations/drift, free synced transcript. 700+ seconds coherent audio. Best candidate for Stories long-form reliability. [HF: HumeAI/tada-1b](https://huggingface.co/HumeAI/tada-1b) | [GitHub: HumeAI/tada](https://github.com/HumeAI/tada)
|
||||
- **MOSS-TTS** — Modular suite: flagship cloning, MOSS-TTSD (multi-speaker dialogue), MOSS-VoiceGenerator (create voices from text descriptions, no ref audio). Unique UX for Stories voice design. [GitHub: OpenMOSS/MOSS-TTS](https://github.com/OpenMOSS/MOSS-TTS)
|
||||
- **VoxCPM 1.5** — Tokenizer-free continuous diffusion + autoregressive. No discrete token artifacts. Context-aware prosody/emotion, real-time streaming, LoRA fine-tuning. Trained on 1.8M+ hours. [GitHub: OpenBMB/VoxCPM](https://github.com/OpenBMB/VoxCPM)
|
||||
- **Pocket TTS** — 100M param CPU-first model from Kyutai Labs (Moshi team). Runs >1× realtime without GPU. Broadens hardware support significantly. [GitHub: kyutai-labs/pocket-tts](https://github.com/kyutai-labs/pocket-tts)
|
||||
- **Watch list:** MioTTS-2.6B (fast LLM-based EN/JP, vLLM compatible), Oolel-Voices (Soynade Research, expressive modular control)
|
||||
- **Skipped:** Fish Audio S2 — restrictive research license (commercial use requires approval), despite strong features
|
||||
|
||||
1. **Model type concept** — A `model_type` field (e.g. `qwen`, `luxtts`, `chatterbox`) alongside `model_size`. The `GenerationRequest` schema, frontend form, and all model config dicts need updating.
|
||||
### Adding a New Engine (Now Straightforward)
|
||||
|
||||
2. **Multiple backend instances** — The singleton `get_tts_backend()` needs to become a registry. Different models have different voice prompt formats, different inference APIs, different sample rates.
|
||||
With the multi-engine architecture shipped, adding a new TTS engine requires:
|
||||
|
||||
3. **Voice prompt format abstraction** — Qwen uses `torch.save()`-serialized tensors. LuxTTS uses `encode_prompt()` returning its own format. Chatterbox uses audio-path-based cloning. The cache system (`backend/utils/cache.py`) needs to handle heterogeneous formats.
|
||||
1. **Create `backend/backends/<engine>_backend.py`** — implement `TTSBackend` protocol (~200-300 lines)
|
||||
2. **Register in `backend/backends/__init__.py`** — add to `TTS_ENGINES` dict + factory function
|
||||
3. **Update `backend/models.py`** — add engine name to regex
|
||||
4. **Update `backend/main.py`** — add engine cases in generate, stream, model-status, download, delete (5 dispatch points)
|
||||
5. **Update frontend** — add to engine union type, form schema, model dropdown, language map (5-6 files)
|
||||
|
||||
4. **Sample rate normalization** — Qwen outputs 24 kHz. LuxTTS outputs 48 kHz. The Stories editor and audio pipeline need to handle mixed rates.
|
||||
|
||||
5. **Per-model capabilities** — Not all models support `instruct` (delivery instructions), not all support streaming, not all support the same languages. The UI needs to adapt.
|
||||
|
||||
### PR #194 as Precedent
|
||||
|
||||
The Hebrew/Chatterbox PR (#194) is the first attempt at multi-model. It takes a pragmatic approach: route by language (`he` → Chatterbox, else → Qwen). This works for one extra model but doesn't scale — what happens when you want Chatterbox for English too?
|
||||
|
||||
### PR #225 as Alternative Approach
|
||||
|
||||
The custom HuggingFace models PR (#225) takes a different angle: let users register arbitrary HF repos and attempt to load them through the existing Qwen backend. This is flexible but fragile — it assumes all models have the same API as Qwen3-TTS.
|
||||
|
||||
### PR #33 as Foundation
|
||||
|
||||
The external provider binaries PR (#33) has the most robust architecture for multi-model, since each provider is a separate process with its own dependencies. But it's complex, currently Qwen-only, and has been stale since early February.
|
||||
Total effort: **~1 day** for a well-documented model with a PyPI package.
|
||||
|
||||
---
|
||||
|
||||
## Architectural Bottlenecks
|
||||
|
||||
### 1. Single Backend Singleton
|
||||
### ~~1. Single Backend Singleton~~ — RESOLVED
|
||||
|
||||
**File:** `backend/backends/__init__.py:118-137`
|
||||
The singleton TTS backend was replaced with a thread-safe per-engine registry in PR #254. Multiple engines can now be loaded simultaneously.
|
||||
|
||||
The entire TTS system runs through one global `_tts_backend` instance. You literally cannot have two models loaded. This is the #1 blocker for multi-model support.
|
||||
### 2. `main.py` is 2100+ Lines
|
||||
|
||||
### 2. `main.py` is 1700+ Lines
|
||||
All API routes, all model configs, all business logic in one file. Five separate dispatch points for each engine. Any new engine touches this file in 5 places. A model config registry pattern would reduce duplication.
|
||||
|
||||
All API routes, all model configs, all business logic in one file. Three separate hardcoded model config dicts that must stay in sync. Any multi-model change touches this file heavily.
|
||||
### 3. Model Config is Scattered (Improved)
|
||||
|
||||
### 3. Model Config is Scattered
|
||||
|
||||
Model identifiers, HF repo IDs, display names, and download logic are duplicated across:
|
||||
- `main.py` (3 separate dicts)
|
||||
- `pytorch_backend.py` (HF repo map)
|
||||
- `mlx_backend.py` (MLX repo map)
|
||||
- `GenerationForm.tsx` (UI labels)
|
||||
- `useGenerationForm.ts` (validation schema)
|
||||
- `ModelManagement.tsx` (prefix filters)
|
||||
|
||||
There is no single source of truth for "what models does Voicebox support."
|
||||
Model identifiers are still duplicated across `main.py` (3 dicts), backend files, frontend components, and the languages constant. However, the pattern is now consistent and well-understood. A centralized model registry would help but isn't blocking.
|
||||
|
||||
### 4. Voice Prompt Cache Assumes PyTorch Tensors
|
||||
|
||||
`backend/utils/cache.py` uses `torch.save()` / `torch.load()` for caching voice prompts. Models that don't use PyTorch tensors (LuxTTS, MLX-native models) can't use this cache.
|
||||
`backend/utils/cache.py` uses `torch.save()` / `torch.load()`. LuxTTS and Chatterbox backends work around this by storing reference audio paths instead of tensors in their voice prompt dicts. Not ideal but functional.
|
||||
|
||||
### 5. Frontend Assumes Qwen Model Sizes
|
||||
### 5. ~~Frontend Assumes Qwen Model Sizes~~ — RESOLVED
|
||||
|
||||
The generation form schema (`useGenerationForm.ts:17`) validates `model_size` as `'1.7B' | '0.6B'`. The model management UI filters by string prefix `qwen-tts`. Adding any model requires touching 3-4 frontend files.
|
||||
The generation form now uses a flat model dropdown with engine-based routing. Per-engine language filtering is in place. Model size is only sent for Qwen.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Priorities
|
||||
|
||||
### Tier 1 — Ship Now (Bug Fixes & Critical Improvements)
|
||||
### Tier 1 — Ship Now (Low Risk)
|
||||
|
||||
These PRs fix real user pain with low risk. Can be reviewed and merged quickly.
|
||||
| Priority | PR/Item | Impact | Effort |
|
||||
|----------|---------|--------|--------|
|
||||
| 1 | **#258** — Chatterbox Turbo + per-engine languages | Paralinguistic tags, proper language filtering | Review only |
|
||||
| 2 | **#152** — Offline mode crash fix | Fixes #150, #151 | Low |
|
||||
| 3 | **#99** — Chunked TTS + quality selector | Removes 500-char limit, addresses 5 issues | Medium |
|
||||
| 4 | **#218** — Windows HF cache dir fix | Windows-specific pain | Low |
|
||||
| 5 | **#178** — Generation error handling | Error UX | Low |
|
||||
| 6 | **#230** — Docs fixes | Zero risk | None |
|
||||
| 7 | **#133** — Network access toggle | Wires up existing code | Low |
|
||||
| 8 | **#88** — CORS restriction | Security improvement | Low |
|
||||
| 9 | **#214** — Tauri window close panic fix | Stability | Low |
|
||||
| 10 | Triage GPU issues | Many may be resolved by CUDA swap (#252) | Low |
|
||||
| 11 | Close superseded PRs | #194 (superseded by #257), #83 (outdated) | None |
|
||||
|
||||
| Priority | PR | Impact | Effort |
|
||||
|----------|-----|--------|--------|
|
||||
| 1 | **#97** — Pass language param to TTS | Fixes all non-English generation (18 language issues) | Low |
|
||||
| 2 | **#238** — Download cancel/clear UI | Addresses 20 download-related issues | Low |
|
||||
| 3 | **#152** — Offline mode crash fix | Fixes #150, #151 | Low |
|
||||
| 4 | **#99** — Chunked TTS + quality selector | Removes 500-char limit, addresses 5 issues | Medium |
|
||||
| 5 | **#218** — Windows HF cache dir fix | Windows-specific pain | Low |
|
||||
| 6 | **#175, #178** — Profile validation + error handling | Small fixes | Low |
|
||||
| 7 | **#250, #230** — Docs fixes | Zero risk | None |
|
||||
| 8 | **#133** — Network access toggle | Wires up existing code | Low |
|
||||
| 9 | **#88** — CORS restriction | Security improvement | Low |
|
||||
| 10 | **#214** — Tauri window close panic fix | Stability | Low |
|
||||
### Tier 2 — Next Release (v0.2.0)
|
||||
|
||||
### Tier 2 — Next Release (v0.2.0 Foundations)
|
||||
|
||||
These require more review but unlock major capabilities.
|
||||
|
||||
| Priority | Item | Impact | Effort | Dependencies |
|
||||
|----------|------|--------|--------|-------------|
|
||||
| 1 | **PR #33** — External provider binaries | Solves GPU distribution (19 issues), foundation for multi-model | Very High | Needs rebase, thorough review |
|
||||
| 2 | **Multi-model abstraction layer** | Required before adding LuxTTS/Chatterbox/etc. | High | Informed by #33, #194, #225 |
|
||||
| 3 | **PR #161** — Docker deployment | Server/headless users | Medium | Independent of #33 |
|
||||
| 4 | **PR #194** — Hebrew + Chatterbox | First non-Qwen model, language expansion | High | Should align with multi-model abstraction |
|
||||
| 5 | **PR #154** — Audiobook tab | Significant feature for long-form users | Medium | Benefits from #99 (chunking) |
|
||||
| Priority | Item | Impact | Effort |
|
||||
|----------|------|--------|--------|
|
||||
| 1 | **#253** — 48kHz speech tokenizer | Quality improvement | Medium |
|
||||
| 2 | **#161** — Docker deployment | Server/headless users | Medium |
|
||||
| 3 | **#154** — Audiobook tab | Long-form users | Medium |
|
||||
| 4 | **Model config registry** | Reduce 5-dispatch-point duplication in main.py | Medium |
|
||||
| 5 | **#225** — Custom HuggingFace models | User-supplied models | High (needs rework for multi-engine) |
|
||||
|
||||
### Tier 3 — Future (v0.3.0+)
|
||||
|
||||
| Item | Notes |
|
||||
|------|-------|
|
||||
| LuxTTS integration | 48 kHz, low VRAM, but needs multi-model arch first |
|
||||
| XTTS-v2 / Fish Speech | Multilingual powerhouses |
|
||||
| OpenAI-compatible API (plan doc exists) | Low effort once API is stable |
|
||||
| LoRA fine-tuning (PR #195) | Complex, depends on #194 |
|
||||
| External/remote providers (plan doc exists) | Depends on provider architecture |
|
||||
| GGUF support (#226) | Depends on model ecosystem maturity |
|
||||
| Queue system (#234) | Batch generation |
|
||||
| Real-time streaming synthesis | MLX-only currently, needs PyTorch path |
|
||||
|
||||
### Decision Point: Multi-Model Architecture
|
||||
|
||||
Before adding any new TTS model, a decision is needed on *how*:
|
||||
|
||||
**Option A — Provider Binary Split (PR #33 approach)**
|
||||
Each model family is a separate executable/process. Most isolated, most flexible, but most complex. Solves the CUDA distribution problem simultaneously.
|
||||
|
||||
**Option B — In-Process Model Registry**
|
||||
Keep everything in one process but replace the singleton with a registry that can instantiate multiple `TTSBackend` implementations. Simpler, but doesn't solve binary size / CUDA distribution.
|
||||
|
||||
**Option C — Hybrid (Recommended)**
|
||||
Use Option B for lightweight models (LuxTTS, Kokoro — small, CPU-friendly) that can coexist in-process. Use Option A for heavy models (CUDA Qwen3-TTS, Fish Speech) that need their own process/dependencies. The provider architecture from PR #33 becomes the escape hatch for heavy models, while light models are built-in.
|
||||
|
||||
This matches how PR #194 already works (Chatterbox loaded in-process alongside Qwen) while keeping the door open for PR #33's provider split.
|
||||
| Priority | Item | Notes |
|
||||
|----------|------|-------|
|
||||
| 1 | **HumeAI TADA** | Long-form reliability for Stories, synced transcripts. Addresses #234, #203, #191, #111, #69. Needs API vetting. |
|
||||
| 2 | **Pocket TTS** (Kyutai) | CPU-first 100M model, broadens hardware support. Kyutai ships clean code. Needs API vetting. |
|
||||
| 3 | **MOSS-TTS** | Text-to-voice design (no ref audio) is unique. Multi-speaker dialogue for Stories. Needs thorough API vetting. |
|
||||
| 4 | **Kokoro-82M** | 82M params, CPU realtime, Apache 2.0. Easy win. |
|
||||
| 5 | **Model config registry refactor** | Reduce 5-dispatch-point duplication in main.py — do before adding 3+ more engines |
|
||||
| 6 | XTTS-v2 / Fish Speech / CosyVoice | Multi-engine arch is ready; just needs backend implementation |
|
||||
| 7 | **VoxCPM 1.5** | Tokenizer-free streaming, interesting but uncertain integration surface |
|
||||
| 8 | OpenAI-compatible API (plan doc exists) | Low effort once API is stable |
|
||||
| 9 | LoRA fine-tuning (PR #195) | Complex, needs rework for multi-engine |
|
||||
| 10 | External/remote providers | Depends on use case demand |
|
||||
| 11 | GGUF support (#226) | Depends on model ecosystem maturity |
|
||||
| 12 | Queue system (#234) | Batch generation |
|
||||
| 13 | Streaming for non-MLX engines | Currently MLX-only |
|
||||
|
||||
---
|
||||
|
||||
@@ -409,24 +437,20 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
|
||||
| Branch | PR | Status | Notes |
|
||||
|--------|-----|--------|-------|
|
||||
| `external-provider-binaries` | #33 | Open, stale | Major architecture work |
|
||||
| `feat/dual-server-binaries` | — | No PR | Related to provider split? |
|
||||
| `feat/chatterbox-turbo` | #258 | Open | Chatterbox Turbo + per-engine languages |
|
||||
| `feat/chatterbox` | #257 | **Merged** | Chatterbox Multilingual |
|
||||
| `feat/luxtts` | #254 | **Merged** | LuxTTS + multi-engine arch |
|
||||
| `external-provider-binaries` | #33 | Superseded by #252 | Original CUDA provider approach |
|
||||
| `feat/dual-server-binaries` | — | No PR | Related to provider split |
|
||||
| `fix-multi-sample` | — | No PR | Voice profile multi-sample fix |
|
||||
| `fix-dl-notification-...` | — | No PR | Model download UX |
|
||||
| `improvements` | — | No PR | Unknown scope |
|
||||
| `stories` | — | No PR | Stories editor work? |
|
||||
| `windows-server-shutdown` | — | No PR | Windows lifecycle |
|
||||
| `model-dl-fix` | — | No PR | Model download fix |
|
||||
| `channels` | — | No PR | Audio channels |
|
||||
| `audio-export-entitlement-fix` | — | No PR | macOS entitlements |
|
||||
| `better-docs` | — | No PR | Documentation |
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference: API Endpoints
|
||||
|
||||
<details>
|
||||
<summary>All current endpoints (v0.1.13)</summary>
|
||||
<summary>All current endpoints</summary>
|
||||
|
||||
| Endpoint | Method | Purpose |
|
||||
|----------|--------|---------|
|
||||
@@ -437,20 +461,21 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
| `/profiles/{id}/avatar` | POST, GET, DELETE | Avatar management |
|
||||
| `/profiles/{id}/export` | GET | Export profile as ZIP |
|
||||
| `/profiles/import` | POST | Import profile from ZIP |
|
||||
| `/generate` | POST | Generate speech |
|
||||
| `/generate/stream` | POST | Stream speech (SSE) |
|
||||
| `/generate` | POST | Generate speech (engine param selects TTS backend) |
|
||||
| `/generate/stream` | POST | Stream speech (MLX only) |
|
||||
| `/history` | GET | List generation history |
|
||||
| `/history/{id}` | GET, DELETE | Get/delete generation |
|
||||
| `/history/{id}/export` | GET | Export generation ZIP |
|
||||
| `/history/{id}/export-audio` | GET | Export audio only |
|
||||
| `/transcribe` | POST | Transcribe audio (Whisper) |
|
||||
| `/models/status` | GET | All model statuses |
|
||||
| `/models/status` | GET | All model statuses (Qwen, LuxTTS, Chatterbox, Chatterbox Turbo, Whisper) |
|
||||
| `/models/download` | POST | Trigger model download |
|
||||
| `/models/download/cancel` | POST | Cancel/dismiss download |
|
||||
| `/models/{name}` | DELETE | Delete downloaded model |
|
||||
| `/models/load` | POST | Load model into memory |
|
||||
| `/models/unload` | POST | Unload model |
|
||||
| `/models/progress/{name}` | GET | SSE download progress |
|
||||
| `/tasks/active` | GET | Active downloads/generations |
|
||||
| `/tasks/active` | GET | Active downloads/generations (with inline progress) |
|
||||
| `/stories` | POST, GET | Create/list stories |
|
||||
| `/stories/{id}` | GET, PUT, DELETE | Story CRUD |
|
||||
| `/stories/{id}/items` | POST, GET | Story items CRUD |
|
||||
@@ -458,5 +483,8 @@ This matches how PR #194 already works (Chatterbox loaded in-process alongside Q
|
||||
| `/channels` | POST, GET | Audio channel CRUD |
|
||||
| `/channels/{id}` | PUT, DELETE | Channel update/delete |
|
||||
| `/cache/clear` | POST | Clear voice prompt cache |
|
||||
| `/server/cuda/status` | GET | CUDA binary availability |
|
||||
| `/server/cuda/download` | POST | Download CUDA binary |
|
||||
| `/server/cuda/switch` | POST | Switch to CUDA backend |
|
||||
|
||||
</details>
|
||||
|
||||
+5
-1
@@ -40,5 +40,9 @@
|
||||
"engines": {
|
||||
"bun": ">=1.0.0"
|
||||
},
|
||||
"packageManager": "[email protected]"
|
||||
"packageManager": "[email protected]",
|
||||
"dependencies": {
|
||||
"loaders.css": "^0.1.2",
|
||||
"react-loaders": "^3.0.1"
|
||||
}
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,16 +1,312 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
use cpal::{SampleFormat, StreamConfig};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
/// Start capturing system audio on Linux using PulseAudio monitor sources.
|
||||
///
|
||||
/// PulseAudio exposes "monitor" devices that mirror the output of each sink,
|
||||
/// allowing us to capture whatever audio is currently playing on the system.
|
||||
/// We use `cpal` with the default host (which will be PulseAudio or PipeWire
|
||||
/// on modern Linux) and look for monitor input devices.
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
todo!("implement Linux audio capture")
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let sample_rate_arc = state.sample_rate.clone();
|
||||
let channels_arc = state.channels.clone();
|
||||
let stop_tx = state.stop_tx.clone();
|
||||
let error_arc = state.error.clone();
|
||||
|
||||
// Use AtomicBool for stop signal (works across threads)
|
||||
let stop_flag = Arc::new(AtomicBool::new(false));
|
||||
let stop_flag_clone = stop_flag.clone();
|
||||
|
||||
// Create tokio channel and spawn a task to bridge it to the AtomicBool
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
|
||||
*stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
tokio::spawn(async move {
|
||||
rx.recv().await;
|
||||
stop_flag_clone.store(true, Ordering::Relaxed);
|
||||
});
|
||||
|
||||
// Spawn capture on a dedicated thread
|
||||
thread::spawn(move || {
|
||||
let host = cpal::default_host();
|
||||
|
||||
// Try to find a monitor device for system audio capture.
|
||||
// On PulseAudio/PipeWire, monitor sources have "monitor" in their name.
|
||||
let device = {
|
||||
let mut monitor_device = None;
|
||||
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for d in devices {
|
||||
if let Ok(name) = d.name() {
|
||||
let name_lower = name.to_lowercase();
|
||||
if name_lower.contains("monitor") {
|
||||
eprintln!("Linux audio capture: Found monitor device: {}", name);
|
||||
monitor_device = Some(d);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match monitor_device {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
// Fallback to default input device (microphone)
|
||||
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
||||
match host.default_input_device() {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
let error_msg = "No audio input device available".to_string();
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
||||
eprintln!("Linux audio capture: Using device: {}", device_name);
|
||||
|
||||
// Get supported config
|
||||
let config = match device.default_input_config() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get default input config: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let sample_rate = config.sample_rate().0;
|
||||
let channels = config.channels();
|
||||
let sample_format = config.sample_format();
|
||||
|
||||
eprintln!(
|
||||
"Linux audio capture: Config - {}Hz, {} channels, format: {:?}",
|
||||
sample_rate, channels, sample_format
|
||||
);
|
||||
|
||||
*sample_rate_arc.lock().unwrap() = sample_rate;
|
||||
*channels_arc.lock().unwrap() = channels;
|
||||
|
||||
let stream_config = StreamConfig {
|
||||
channels,
|
||||
sample_rate: cpal::SampleRate(sample_rate),
|
||||
buffer_size: cpal::BufferSize::Default,
|
||||
};
|
||||
|
||||
let samples_clone = samples.clone();
|
||||
let error_arc_clone = error_arc.clone();
|
||||
let stop_flag_for_stream = stop_flag.clone();
|
||||
|
||||
let err_fn = {
|
||||
let error_arc = error_arc.clone();
|
||||
move |err: cpal::StreamError| {
|
||||
let error_msg = format!("Stream error: {}", err);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
}
|
||||
};
|
||||
|
||||
let stream = match sample_format {
|
||||
SampleFormat::F32 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[f32], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
guard.extend_from_slice(data);
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SampleFormat::I16 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[i16], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
for &s in data {
|
||||
guard.push(s as f32 / 32768.0);
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SampleFormat::U16 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[u16], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
for &s in data {
|
||||
guard.push((s as f32 / 32768.0) - 1.0);
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
let error_msg = format!("Unsupported sample format: {:?}", sample_format);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let stream = match stream {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to build input stream: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = stream.play() {
|
||||
let error_msg = format!("Failed to start stream: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
|
||||
eprintln!("Linux audio capture: Stream started successfully");
|
||||
|
||||
// Keep thread alive until stop signal
|
||||
loop {
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
|
||||
// Stream will be dropped here, stopping capture
|
||||
eprintln!("Linux audio capture: Stream stopped");
|
||||
});
|
||||
|
||||
// Spawn timeout task
|
||||
let stop_tx_clone = state.stop_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
|
||||
let tx = stop_tx_clone.lock().unwrap().take();
|
||||
if let Some(tx) = tx {
|
||||
let _ = tx.send(()).await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
todo!("implement Linux audio capture stop")
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Check if there was an error during capture
|
||||
if let Some(error) = state.error.lock().unwrap().as_ref() {
|
||||
return Err(error.clone());
|
||||
}
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err(
|
||||
"No audio samples captured. Make sure audio is playing on your system during recording."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
false
|
||||
// Check if we can find a monitor device for system audio capture
|
||||
let host = cpal::default_host();
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for d in devices {
|
||||
if let Ok(name) = d.name() {
|
||||
if name.to_lowercase().contains("monitor") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Even without a monitor, basic input capture is available
|
||||
host.default_input_device().is_some()
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer =
|
||||
WavWriter::new(cursor, spec).map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer
|
||||
.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer
|
||||
.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ struct ServerState {
|
||||
child: Mutex<Option<tauri_plugin_shell::process::CommandChild>>,
|
||||
server_pid: Mutex<Option<u32>>,
|
||||
keep_running_on_close: Mutex<bool>,
|
||||
models_dir: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
#[command]
|
||||
@@ -23,7 +24,16 @@ async fn start_server(
|
||||
app: tauri::AppHandle,
|
||||
state: State<'_, ServerState>,
|
||||
remote: Option<bool>,
|
||||
models_dir: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
// Store models_dir for use on restart (empty string means reset to default)
|
||||
if let Some(ref dir) = models_dir {
|
||||
if dir.is_empty() {
|
||||
*state.models_dir.lock().unwrap() = None;
|
||||
} else {
|
||||
*state.models_dir.lock().unwrap() = Some(dir.clone());
|
||||
}
|
||||
}
|
||||
// Check if server is already running (managed by this app instance)
|
||||
if state.child.lock().unwrap().is_some() {
|
||||
return Ok(format!("http://127.0.0.1:{}", SERVER_PORT));
|
||||
@@ -274,6 +284,12 @@ async fn start_server(
|
||||
let port_str = SERVER_PORT.to_string();
|
||||
let is_remote = remote.unwrap_or(false);
|
||||
|
||||
// Resolve the custom models directory from the parameter or stored state
|
||||
let effective_models_dir = models_dir.or_else(|| state.models_dir.lock().unwrap().clone());
|
||||
if let Some(ref dir) = effective_models_dir {
|
||||
println!("Custom models directory: {}", dir);
|
||||
}
|
||||
|
||||
// If CUDA binary exists, launch it directly instead of the bundled sidecar
|
||||
let spawn_result = if let Some(ref cuda_path) = cuda_binary {
|
||||
println!("Launching CUDA backend: {:?}", cuda_path);
|
||||
@@ -282,6 +298,9 @@ async fn start_server(
|
||||
if is_remote {
|
||||
cmd = cmd.args(["--host", "0.0.0.0"]);
|
||||
}
|
||||
if let Some(ref dir) = effective_models_dir {
|
||||
cmd = cmd.env("VOICEBOX_MODELS_DIR", dir);
|
||||
}
|
||||
cmd.spawn()
|
||||
} else {
|
||||
// Use the bundled CPU sidecar
|
||||
@@ -289,6 +308,9 @@ async fn start_server(
|
||||
if is_remote {
|
||||
sidecar = sidecar.args(["--host", "0.0.0.0"]);
|
||||
}
|
||||
if let Some(ref dir) = effective_models_dir {
|
||||
sidecar = sidecar.env("VOICEBOX_MODELS_DIR", dir);
|
||||
}
|
||||
println!("Spawning server process...");
|
||||
sidecar.spawn()
|
||||
};
|
||||
@@ -613,9 +635,19 @@ async fn stop_server(state: State<'_, ServerState>) -> Result<(), String> {
|
||||
async fn restart_server(
|
||||
app: tauri::AppHandle,
|
||||
state: State<'_, ServerState>,
|
||||
models_dir: Option<String>,
|
||||
) -> Result<String, String> {
|
||||
println!("restart_server: stopping current server...");
|
||||
|
||||
// Update stored models_dir: empty string means reset to default, non-empty means set
|
||||
if let Some(ref dir) = models_dir {
|
||||
if dir.is_empty() {
|
||||
*state.models_dir.lock().unwrap() = None;
|
||||
} else {
|
||||
*state.models_dir.lock().unwrap() = Some(dir.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the current server
|
||||
stop_server(state.clone()).await?;
|
||||
|
||||
@@ -623,9 +655,9 @@ async fn restart_server(
|
||||
println!("restart_server: waiting for port release...");
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
|
||||
|
||||
// Start server again (will auto-detect CUDA binary)
|
||||
// Start server again (will auto-detect CUDA binary and use stored models_dir)
|
||||
println!("restart_server: starting server...");
|
||||
start_server(app, state, None).await
|
||||
start_server(app, state, None, None).await
|
||||
}
|
||||
|
||||
#[command]
|
||||
@@ -686,6 +718,7 @@ pub fn run() {
|
||||
child: Mutex::new(None),
|
||||
server_pid: Mutex::new(None),
|
||||
keep_running_on_close: Mutex::new(false),
|
||||
models_dir: Mutex::new(None),
|
||||
})
|
||||
.manage(audio_capture::AudioCaptureState::new())
|
||||
.manage(audio_output::AudioOutputState::new())
|
||||
@@ -792,7 +825,9 @@ pub fn run() {
|
||||
});
|
||||
|
||||
// Wait for frontend response or timeout
|
||||
tokio::spawn(async move {
|
||||
// Use tauri::async_runtime::spawn instead of tokio::spawn to avoid
|
||||
// panics when the Tokio runtime is being dropped during app shutdown
|
||||
tauri::async_runtime::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = rx.recv() => {
|
||||
// Frontend responded, close window
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
},
|
||||
"plugins": {
|
||||
"shell": {
|
||||
"open": true
|
||||
"open": ".*"
|
||||
},
|
||||
"updater": {
|
||||
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IEUxRENBQkRBQjdBNTM1OTIKUldTU05hVzMycXZjNGJGcUxmcVVocll2QjdSaTJNdlFxR2M3VDJsMnVvbDdyZGRPMmRlOW9aWTcK",
|
||||
|
||||
@@ -5,9 +5,12 @@ import type { PlatformLifecycle } from '@/platform/types';
|
||||
class TauriLifecycle implements PlatformLifecycle {
|
||||
onServerReady?: () => void;
|
||||
|
||||
async startServer(remote = false): Promise<string> {
|
||||
async startServer(remote = false, modelsDir?: string | null): Promise<string> {
|
||||
try {
|
||||
const result = await invoke<string>('start_server', { remote });
|
||||
const result = await invoke<string>('start_server', {
|
||||
remote,
|
||||
modelsDir: modelsDir ?? undefined,
|
||||
});
|
||||
console.log('Server started:', result);
|
||||
this.onServerReady?.();
|
||||
return result;
|
||||
@@ -27,9 +30,11 @@ class TauriLifecycle implements PlatformLifecycle {
|
||||
}
|
||||
}
|
||||
|
||||
async restartServer(): Promise<string> {
|
||||
async restartServer(modelsDir?: string | null): Promise<string> {
|
||||
try {
|
||||
const result = await invoke<string>('restart_server');
|
||||
const result = await invoke<string>('restart_server', {
|
||||
modelsDir: modelsDir ?? undefined,
|
||||
});
|
||||
console.log('Server restarted:', result);
|
||||
this.onServerReady?.();
|
||||
return result;
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { PlatformLifecycle } from '@/platform/types';
|
||||
class WebLifecycle implements PlatformLifecycle {
|
||||
onServerReady?: () => void;
|
||||
|
||||
async startServer(_remote = false): Promise<string> {
|
||||
async startServer(_remote = false, _modelsDir?: string | null): Promise<string> {
|
||||
// Web assumes server is running externally
|
||||
// Return a default URL - this should be configured via env vars
|
||||
const serverUrl = import.meta.env.VITE_SERVER_URL || 'http://localhost:17493';
|
||||
@@ -15,7 +15,7 @@ class WebLifecycle implements PlatformLifecycle {
|
||||
// No-op for web - server is managed externally
|
||||
}
|
||||
|
||||
async restartServer(): Promise<string> {
|
||||
async restartServer(_modelsDir?: string | null): Promise<string> {
|
||||
// No-op for web - server is managed externally
|
||||
return import.meta.env.VITE_SERVER_URL || 'http://localhost:17493';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user