From 54d72ddfd0bfffc04113df31150e7893b8c32db7 Mon Sep 17 00:00:00 2001 From: Mriganka Date: Fri, 20 Feb 2026 23:23:38 +0530 Subject: [PATCH] fix: resolve multiple issues (#96, #119, #111, #108, #121, #125, #127) --- .../components/VoiceProfiles/ProfileForm.tsx | 17 +- app/src/lib/hooks/useAudioRecording.ts | 46 ++--- app/src/lib/utils/audio.ts | 54 ++++-- backend/backends/pytorch_backend.py | 59 ++++-- backend/config.py | 9 + backend/main.py | 168 ++++++++++++++---- backend/tts.py | 8 - web/package.json | 1 + web/vite.config.ts | 3 +- 9 files changed, 268 insertions(+), 97 deletions(-) diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index f4fc5711..8baa9672 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -43,7 +43,7 @@ import { } from '@/lib/hooks/useProfiles'; import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture'; import { useTranscription } from '@/lib/hooks/useTranscription'; -import { formatAudioDuration, getAudioDuration } from '@/lib/utils/audio'; +import { convertToWav, formatAudioDuration, getAudioDuration } from '@/lib/utils/audio'; import { usePlatform } from '@/platform/PlatformContext'; import { useServerStore } from '@/stores/serverStore'; import { type ProfileFormDraft, useUIStore } from '@/stores/uiStore'; @@ -505,10 +505,23 @@ export function ProfileForm() { language: data.language, }); + // Convert non-WAV uploads to WAV so the backend can always use soundfile. + // Recorded audio is already WAV (from useAudioRecording's convertToWav call). + let fileToUpload: File = sampleFile; + if (!sampleFile.type.includes('wav') && !sampleFile.name.toLowerCase().endsWith('.wav')) { + try { + const wavBlob = await convertToWav(sampleFile); + const wavName = sampleFile.name.replace(/\.[^.]+$/, '.wav'); + fileToUpload = new File([wavBlob], wavName, { type: 'audio/wav' }); + } catch { + // If browser can't decode the format, send the original and let the backend try. + } + } + try { await addSample.mutateAsync({ profileId: profile.id, - file: sampleFile, + file: fileToUpload, referenceText: referenceText, }); diff --git a/app/src/lib/hooks/useAudioRecording.ts b/app/src/lib/hooks/useAudioRecording.ts index 2916937c..152f90c1 100644 --- a/app/src/lib/hooks/useAudioRecording.ts +++ b/app/src/lib/hooks/useAudioRecording.ts @@ -20,11 +20,13 @@ export function useAudioRecording({ const streamRef = useRef(null); const timerRef = useRef(null); const startTimeRef = useRef(null); + const cancelledRef = useRef(false); const startRecording = useCallback(async () => { try { setError(null); chunksRef.current = []; + cancelledRef.current = false; setDuration(0); // Check if getUserMedia is available @@ -87,31 +89,34 @@ export function useAudioRecording({ }; mediaRecorder.onstop = async () => { + // Snapshot the cancellation flag and recorded duration immediately — + // cancelRecording() clears chunks and sets cancelledRef synchronously + // before this async handler runs, so we must check it first. + const wasCancelled = cancelledRef.current; + const recordedDuration = startTimeRef.current + ? (Date.now() - startTimeRef.current) / 1000 + : undefined; + const webmBlob = new Blob(chunksRef.current, { type: 'audio/webm' }); - // Convert to WAV format to avoid needing ffmpeg on backend - try { - const wavBlob = await convertToWav(webmBlob); - - // Pass the actual recorded duration - const recordedDuration = startTimeRef.current - ? (Date.now() - startTimeRef.current) / 1000 - : undefined; - onRecordingComplete?.(wavBlob, recordedDuration); - } catch (err) { - console.error('Error converting audio to WAV:', err); - // Fallback to original blob if conversion fails - const recordedDuration = startTimeRef.current - ? (Date.now() - startTimeRef.current) / 1000 - : undefined; - onRecordingComplete?.(webmBlob, recordedDuration); - } - - // Stop all tracks + // Stop all tracks now that we have the data streamRef.current?.getTracks().forEach((track) => { track.stop(); }); streamRef.current = null; + + // Don't fire completion callback if the recording was cancelled + if (wasCancelled) return; + + // Convert to WAV format to avoid needing ffmpeg on backend + try { + const wavBlob = await convertToWav(webmBlob); + onRecordingComplete?.(wavBlob, recordedDuration); + } catch (err) { + console.error('Error converting audio to WAV:', err); + // Fallback to original blob if conversion fails + onRecordingComplete?.(webmBlob, recordedDuration); + } }; mediaRecorder.onerror = (event) => { @@ -167,9 +172,10 @@ export function useAudioRecording({ const cancelRecording = useCallback(() => { if (mediaRecorderRef.current) { + cancelledRef.current = true; // Must be set before stop() triggers onstop + chunksRef.current = []; mediaRecorderRef.current.stop(); setIsRecording(false); - chunksRef.current = []; setDuration(0); } diff --git a/app/src/lib/utils/audio.ts b/app/src/lib/utils/audio.ts index 159af57d..a8ccc722 100644 --- a/app/src/lib/utils/audio.ts +++ b/app/src/lib/utils/audio.ts @@ -22,6 +22,11 @@ export function formatAudioDuration(seconds: number): string { * If the file has a recordedDuration property (from recording hooks), * use that instead of trying to read metadata. This fixes issues on Windows * where WebM files from MediaRecorder don't have proper duration metadata. + * + * For uploaded files we use AudioContext.decodeAudioData which fully decodes + * the audio and returns the exact duration. This is more reliable than + * HTMLMediaElement.duration which can return incorrect large values for VBR + * MP3 files that lack a proper XING/VBRI header. */ export async function getAudioDuration( file: File & { recordedDuration?: number }, @@ -30,26 +35,39 @@ export async function getAudioDuration( return file.recordedDuration; } - return new Promise((resolve, reject) => { - const audio = new Audio(); - const url = URL.createObjectURL(file); + // Use Web Audio API for accurate duration — avoids VBR MP3 metadata issues. + try { + const audioContext = new AudioContext(); + try { + const arrayBuffer = await file.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return audioBuffer.duration; + } finally { + await audioContext.close(); + } + } catch { + // Fallback: read duration from the media element (less accurate but works for WAV). + return new Promise((resolve, reject) => { + const audio = new Audio(); + const url = URL.createObjectURL(file); - audio.addEventListener('loadedmetadata', () => { - URL.revokeObjectURL(url); - if (Number.isFinite(audio.duration) && audio.duration > 0) { - resolve(audio.duration); - } else { - reject(new Error('Audio file has invalid duration metadata')); - } + audio.addEventListener('loadedmetadata', () => { + URL.revokeObjectURL(url); + if (Number.isFinite(audio.duration) && audio.duration > 0) { + resolve(audio.duration); + } else { + reject(new Error('Audio file has invalid duration metadata')); + } + }); + + audio.addEventListener('error', () => { + URL.revokeObjectURL(url); + reject(new Error('Failed to load audio file')); + }); + + audio.src = url; }); - - audio.addEventListener('error', () => { - URL.revokeObjectURL(url); - reject(new Error('Failed to load audio file')); - }); - - audio.src = url; - }); + } } /** diff --git a/backend/backends/pytorch_backend.py b/backend/backends/pytorch_backend.py index 26f38726..d0cba11a 100644 --- a/backend/backends/pytorch_backend.py +++ b/backend/backends/pytorch_backend.py @@ -29,9 +29,23 @@ class PyTorchTTSBackend: """Get the best available device.""" if torch.cuda.is_available(): return "cuda" - elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): - # MPS can have issues, use CPU for stability - return "cpu" + # Intel Arc / Intel Xe GPU via intel-extension-for-pytorch (IPEX) + try: + import intel_extension_for_pytorch # noqa: F401 + if hasattr(torch, 'xpu') and torch.xpu.is_available(): + return "xpu" + except ImportError: + pass + # Any GPU on Windows via DirectML (torch-directml) + try: + import torch_directml + if torch_directml.device_count() > 0: + return torch_directml.device(0) + except ImportError: + pass + # MPS (Apple Silicon) — kept for completeness but MLX backend is preferred + if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + return "cpu" # MPS disabled for stability; MLX backend handles Apple Silicon return "cpu" def is_loaded(self) -> bool: @@ -166,11 +180,21 @@ class PyTorchTTSBackend: # Load the model (tqdm is patched, but filters out non-download progress) try: - self.model = Qwen3TTSModel.from_pretrained( - model_path, - device_map=self.device, - torch_dtype=torch.float32 if self.device == "cpu" else torch.bfloat16, - ) + # Don't pass device_map on CPU: accelerate's meta-tensor mechanism + # causes "Cannot copy out of meta tensor" when moving to CPU. + # Instead load directly then call .to(device) if needed. + if self.device == "cpu": + self.model = Qwen3TTSModel.from_pretrained( + model_path, + torch_dtype=torch.float32, + low_cpu_mem_usage=False, + ) + else: + self.model = Qwen3TTSModel.from_pretrained( + model_path, + device_map=self.device, + torch_dtype=torch.bfloat16, + ) finally: # Exit the patch context tracker_context.__exit__(None, None, None) @@ -358,9 +382,22 @@ class PyTorchSTTBackend: """Get the best available device.""" if torch.cuda.is_available(): return "cuda" - elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): - # MPS support for Whisper - return "cpu" # Use CPU for stability + # Intel Arc / Intel Xe GPU via intel-extension-for-pytorch (IPEX) + try: + import intel_extension_for_pytorch # noqa: F401 + if hasattr(torch, 'xpu') and torch.xpu.is_available(): + return "xpu" + except ImportError: + pass + # Any GPU on Windows via DirectML (torch-directml) + try: + import torch_directml + if torch_directml.device_count() > 0: + return torch_directml.device(0) + except ImportError: + pass + if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + return "cpu" # MPS disabled for stability return "cpu" def is_loaded(self) -> bool: diff --git a/backend/config.py b/backend/config.py index a4718207..b5c64825 100644 --- a/backend/config.py +++ b/backend/config.py @@ -4,8 +4,17 @@ Configuration module for voicebox backend. Handles data directory configuration for production bundling. """ +import os from pathlib import Path +# Allow users to override the HuggingFace model download directory. +# Set VOICEBOX_MODELS_DIR to an absolute path before starting the server. +# This sets HF_HUB_CACHE so all huggingface_hub downloads go to that path. +_custom_models_dir = os.environ.get("VOICEBOX_MODELS_DIR") +if _custom_models_dir: + os.environ["HF_HUB_CACHE"] = _custom_models_dir + print(f"[config] Model download path set to: {_custom_models_dir}") + # Default data directory (used in development) _data_dir = Path("data") diff --git a/backend/main.py b/backend/main.py index 59fb9e18..733a0751 100644 --- a/backend/main.py +++ b/backend/main.py @@ -77,10 +77,39 @@ async def health(): tts_model = tts.get_tts_model() backend_type = get_backend_type() - # Check for GPU availability (CUDA or MPS) + # Check for GPU availability (CUDA, MPS, Intel Arc XPU, or DirectML) has_cuda = torch.cuda.is_available() has_mps = hasattr(torch.backends, 'mps') and torch.backends.mps.is_available() - gpu_available = has_cuda or has_mps + + # Intel Arc / Intel Xe via intel-extension-for-pytorch (IPEX) + has_xpu = False + xpu_name = None + try: + import intel_extension_for_pytorch as ipex # noqa: F401 + if hasattr(torch, 'xpu') and torch.xpu.is_available(): + has_xpu = True + try: + xpu_name = torch.xpu.get_device_name(0) + except Exception: + xpu_name = "Intel GPU" + except ImportError: + pass + + # DirectML backend (torch-directml) for any Windows GPU + has_directml = False + directml_name = None + try: + import torch_directml + if torch_directml.device_count() > 0: + has_directml = True + try: + directml_name = torch_directml.device_name(0) + except Exception: + directml_name = "DirectML GPU" + except ImportError: + pass + + gpu_available = has_cuda or has_mps or has_xpu or has_directml or backend_type == "mlx" gpu_type = None if has_cuda: @@ -89,6 +118,10 @@ async def health(): gpu_type = "MPS (Apple Silicon)" elif backend_type == "mlx": gpu_type = "Metal (Apple Silicon via MLX)" + elif has_xpu: + gpu_type = f"XPU ({xpu_name})" + elif has_directml: + gpu_type = f"DirectML ({directml_name})" vram_used = None if has_cuda: @@ -252,12 +285,17 @@ async def add_profile_sample( db: Session = Depends(get_db), ): """Add a sample to a voice profile.""" - # Save uploaded file to temporary location - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + # Preserve the uploaded file's extension so librosa can detect format correctly. + # Defaulting to .wav was causing soundfile to reject MP3/WebM content as invalid WAV. + _allowed_audio_exts = {'.wav', '.mp3', '.m4a', '.ogg', '.flac', '.aac', '.webm', '.opus'} + _uploaded_ext = Path(file.filename or '').suffix.lower() + file_suffix = _uploaded_ext if _uploaded_ext in _allowed_audio_exts else '.wav' + + with tempfile.NamedTemporaryFile(suffix=file_suffix, delete=False) as tmp: content = await file.read() tmp.write(content) tmp_path = tmp.name - + try: sample = await profiles.add_profile_sample( profile_id, @@ -268,6 +306,8 @@ async def add_profile_sample( return sample except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to process audio file: {str(e)}") finally: # Clean up temp file Path(tmp_path).unlink(missing_ok=True) @@ -541,48 +581,49 @@ async def generate_speech( profile = await profiles.get_profile(data.profile_id, db) if not profile: raise HTTPException(status_code=404, detail="Profile not found") - - # Create voice prompt from profile - voice_prompt = await profiles.create_voice_prompt_for_profile( - data.profile_id, - db, - ) - - # Generate audio + + # Resolve model size and load the correct model FIRST. + # This must happen before create_voice_prompt_for_profile because that + # function calls load_model_async(None), which falls back to self.model_size. + # If the model is already loaded with the right size at that point, it + # returns immediately and the voice prompt is created by the correct model. tts_model = tts.get_tts_model() - # Load the requested model size if different from current (async to not block) model_size = data.model_size or "1.7B" # Check if model needs to be downloaded first model_path = tts_model._get_model_path(model_size) - if model_path.startswith("Qwen/"): - # Model not cached - check if it exists remotely or needs download - from huggingface_hub import constants as hf_constants - repo_cache = Path(hf_constants.HF_HUB_CACHE) / ("models--" + model_path.replace("/", "--")) - if not repo_cache.exists(): - # Start download in background - model_name = f"qwen-tts-{model_size}" + if not tts_model._is_model_cached(model_size): + # Model is not fully cached — kick off a background download and tell + # the client to retry once it's ready. + model_name = f"qwen-tts-{model_size}" - async def download_model_background(): - try: - await tts_model.load_model_async(model_size) - except Exception as e: - task_manager.error_download(model_name, str(e)) + async def download_model_background(): + try: + await tts_model.load_model_async(model_size) + except Exception as e: + task_manager.error_download(model_name, str(e)) - task_manager.start_download(model_name) - asyncio.create_task(download_model_background()) + task_manager.start_download(model_name) + asyncio.create_task(download_model_background()) - # Return 202 Accepted with download info - raise HTTPException( - status_code=202, - detail={ - "message": f"Model {model_size} is being downloaded. Please wait and try again.", - "model_name": model_name, - "downloading": True - } - ) + raise HTTPException( + status_code=202, + detail={ + "message": f"Model {model_size} is being downloaded. Please wait and try again.", + "model_name": model_name, + "downloading": True, + }, + ) + # Load (or switch to) the requested model before building the voice prompt await tts_model.load_model_async(model_size) + + # Create voice prompt from profile (model is already loaded with correct size) + voice_prompt = await profiles.create_voice_prompt_for_profile( + data.profile_id, + db, + ) + audio, sample_rate = await tts_model.generate( data.text, voice_prompt, @@ -625,6 +666,59 @@ async def generate_speech( raise HTTPException(status_code=500, detail=str(e)) +@app.post("/generate/stream") +async def stream_speech( + data: models.GenerationRequest, + db: Session = Depends(get_db), +): + """ + Generate speech and stream the WAV audio directly without saving to disk. + + Returns raw WAV bytes via a StreamingResponse so the client can start + playing audio before the entire file has been received. This endpoint + does NOT create a history entry — use /generate for that. + """ + profile = await profiles.get_profile(data.profile_id, db) + if not profile: + raise HTTPException(status_code=404, detail="Profile not found") + + tts_model = tts.get_tts_model() + model_size = data.model_size or "1.7B" + + if not tts_model._is_model_cached(model_size): + raise HTTPException( + status_code=400, + detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.", + ) + + # Load the correct model before building the voice prompt (fixes issue #96) + await tts_model.load_model_async(model_size) + + voice_prompt = await profiles.create_voice_prompt_for_profile(data.profile_id, db) + + audio, sample_rate = await tts_model.generate( + data.text, + voice_prompt, + data.language, + data.seed, + data.instruct, + ) + + wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate) + + async def _wav_stream(): + # Yield in chunks so large responses don't block the event loop + chunk_size = 64 * 1024 # 64 KB + for i in range(0, len(wav_bytes), chunk_size): + yield wav_bytes[i : i + chunk_size] + + return StreamingResponse( + _wav_stream(), + media_type="audio/wav", + headers={"Content-Disposition": 'attachment; filename="speech.wav"'}, + ) + + # ============================================ # HISTORY ENDPOINTS # ============================================ diff --git a/backend/tts.py b/backend/tts.py index 98db3412..453f2f5c 100644 --- a/backend/tts.py +++ b/backend/tts.py @@ -32,11 +32,3 @@ def audio_to_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes: sf.write(buffer, audio, sample_rate, format="WAV") buffer.seek(0) return buffer.read() - - -def audio_to_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes: - """Convert audio array to WAV bytes.""" - buffer = io.BytesIO() - sf.write(buffer, audio, sample_rate, format="WAV") - buffer.seek(0) - return buffer.read() diff --git a/web/package.json b/web/package.json index 99d82c56..b8233df1 100644 --- a/web/package.json +++ b/web/package.json @@ -21,6 +21,7 @@ "@types/react-dom": "^18.3.0", "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", + "@tailwindcss/vite": "^4.0.0", "@vitejs/plugin-react": "^4.3.0", "eslint": "^8.57.0", "eslint-plugin-react-hooks": "^4.6.0", diff --git a/web/vite.config.ts b/web/vite.config.ts index 27a79a73..e3aa4013 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -1,9 +1,10 @@ import path from 'node:path'; import react from '@vitejs/plugin-react'; +import tailwindcss from '@tailwindcss/vite'; import { defineConfig } from 'vite'; export default defineConfig({ - plugins: [react()], + plugins: [react(), tailwindcss()], resolve: { alias: { '@': path.resolve(__dirname, '../app/src'),