Merge pull request #128 from mrigankad/fix/voicebox-bugs

fix: resolve multiple issues (#96, #119, #111, #108, #121, #125, #127)

app/src/components/VoiceProfiles/ProfileForm.tsx

@@ -43,7 +43,7 @@ import {
 } from '@/lib/hooks/useProfiles';
 import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
 import { useTranscription } from '@/lib/hooks/useTranscription';
-import { formatAudioDuration, getAudioDuration } from '@/lib/utils/audio';
+import { convertToWav, formatAudioDuration, getAudioDuration } from '@/lib/utils/audio';
 import { usePlatform } from '@/platform/PlatformContext';
 import { useServerStore } from '@/stores/serverStore';
 import { type ProfileFormDraft, useUIStore } from '@/stores/uiStore';
@@ -505,10 +505,23 @@ export function ProfileForm() {
           language: data.language,
         });
 
+        // Convert non-WAV uploads to WAV so the backend can always use soundfile.
+        // Recorded audio is already WAV (from useAudioRecording's convertToWav call).
+        let fileToUpload: File = sampleFile;
+        if (!sampleFile.type.includes('wav') && !sampleFile.name.toLowerCase().endsWith('.wav')) {
+          try {
+            const wavBlob = await convertToWav(sampleFile);
+            const wavName = sampleFile.name.replace(/\.[^.]+$/, '.wav');
+            fileToUpload = new File([wavBlob], wavName, { type: 'audio/wav' });
+          } catch {
+            // If browser can't decode the format, send the original and let the backend try.
+          }
+        }
+
         try {
           await addSample.mutateAsync({
             profileId: profile.id,
-            file: sampleFile,
+            file: fileToUpload,
             referenceText: referenceText,
           });
 

app/src/lib/hooks/useAudioRecording.ts

@@ -20,11 +20,13 @@ export function useAudioRecording({
   const streamRef = useRef<MediaStream | null>(null);
   const timerRef = useRef<number | null>(null);
   const startTimeRef = useRef<number | null>(null);
+  const cancelledRef = useRef<boolean>(false);
 
   const startRecording = useCallback(async () => {
     try {
       setError(null);
       chunksRef.current = [];
+      cancelledRef.current = false;
       setDuration(0);
 
       // Check if getUserMedia is available
@@ -87,31 +89,34 @@ export function useAudioRecording({
       };
 
       mediaRecorder.onstop = async () => {
+        // Snapshot the cancellation flag and recorded duration immediately —
+        // cancelRecording() clears chunks and sets cancelledRef synchronously
+        // before this async handler runs, so we must check it first.
+        const wasCancelled = cancelledRef.current;
+        const recordedDuration = startTimeRef.current
+          ? (Date.now() - startTimeRef.current) / 1000
+          : undefined;
+
         const webmBlob = new Blob(chunksRef.current, { type: 'audio/webm' });
 
-        // Convert to WAV format to avoid needing ffmpeg on backend
+        // Stop all tracks now that we have the data
-        try {
-          const wavBlob = await convertToWav(webmBlob);
-
-          // Pass the actual recorded duration
-          const recordedDuration = startTimeRef.current
-            ? (Date.now() - startTimeRef.current) / 1000
-            : undefined;
-          onRecordingComplete?.(wavBlob, recordedDuration);
-        } catch (err) {
-          console.error('Error converting audio to WAV:', err);
-          // Fallback to original blob if conversion fails
-          const recordedDuration = startTimeRef.current
-            ? (Date.now() - startTimeRef.current) / 1000
-            : undefined;
-          onRecordingComplete?.(webmBlob, recordedDuration);
-        }
-
-        // Stop all tracks
         streamRef.current?.getTracks().forEach((track) => {
           track.stop();
         });
         streamRef.current = null;
+
+        // Don't fire completion callback if the recording was cancelled
+        if (wasCancelled) return;
+
+        // Convert to WAV format to avoid needing ffmpeg on backend
+        try {
+          const wavBlob = await convertToWav(webmBlob);
+          onRecordingComplete?.(wavBlob, recordedDuration);
+        } catch (err) {
+          console.error('Error converting audio to WAV:', err);
+          // Fallback to original blob if conversion fails
+          onRecordingComplete?.(webmBlob, recordedDuration);
+        }
       };
 
       mediaRecorder.onerror = (event) => {
@@ -167,9 +172,10 @@ export function useAudioRecording({
 
   const cancelRecording = useCallback(() => {
     if (mediaRecorderRef.current) {
+      cancelledRef.current = true; // Must be set before stop() triggers onstop
+      chunksRef.current = [];
       mediaRecorderRef.current.stop();
       setIsRecording(false);
-      chunksRef.current = [];
       setDuration(0);
     }
 

app/src/lib/utils/audio.ts

@@ -22,6 +22,11 @@ export function formatAudioDuration(seconds: number): string {
  * If the file has a recordedDuration property (from recording hooks),
  * use that instead of trying to read metadata. This fixes issues on Windows
  * where WebM files from MediaRecorder don't have proper duration metadata.
+ *
+ * For uploaded files we use AudioContext.decodeAudioData which fully decodes
+ * the audio and returns the exact duration. This is more reliable than
+ * HTMLMediaElement.duration which can return incorrect large values for VBR
+ * MP3 files that lack a proper XING/VBRI header.
  */
 export async function getAudioDuration(
   file: File & { recordedDuration?: number },
@@ -30,26 +35,39 @@ export async function getAudioDuration(
     return file.recordedDuration;
   }
 
-  return new Promise((resolve, reject) => {
+  // Use Web Audio API for accurate duration — avoids VBR MP3 metadata issues.
-    const audio = new Audio();
+  try {
-    const url = URL.createObjectURL(file);
+    const audioContext = new AudioContext();
+    try {
+      const arrayBuffer = await file.arrayBuffer();
+      const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+      return audioBuffer.duration;
+    } finally {
+      await audioContext.close();
+    }
+  } catch {
+    // Fallback: read duration from the media element (less accurate but works for WAV).
+    return new Promise((resolve, reject) => {
+      const audio = new Audio();
+      const url = URL.createObjectURL(file);
 
-    audio.addEventListener('loadedmetadata', () => {
+      audio.addEventListener('loadedmetadata', () => {
-      URL.revokeObjectURL(url);
+        URL.revokeObjectURL(url);
-      if (Number.isFinite(audio.duration) && audio.duration > 0) {
+        if (Number.isFinite(audio.duration) && audio.duration > 0) {
-        resolve(audio.duration);
+          resolve(audio.duration);
-      } else {
+        } else {
-        reject(new Error('Audio file has invalid duration metadata'));
+          reject(new Error('Audio file has invalid duration metadata'));
-      }
+        }
+      });
+
+      audio.addEventListener('error', () => {
+        URL.revokeObjectURL(url);
+        reject(new Error('Failed to load audio file'));
+      });
+
+      audio.src = url;
     });
-
+  }
-    audio.addEventListener('error', () => {
-      URL.revokeObjectURL(url);
-      reject(new Error('Failed to load audio file'));
-    });
-
-    audio.src = url;
-  });
 }
 
 /**

backend/backends/pytorch_backend.py

@@ -29,9 +29,23 @@ class PyTorchTTSBackend:
         """Get the best available device."""
         if torch.cuda.is_available():
             return "cuda"
-        elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+        # Intel Arc / Intel Xe GPU via intel-extension-for-pytorch (IPEX)
-            # MPS can have issues, use CPU for stability
+        try:
-            return "cpu"
+            import intel_extension_for_pytorch  # noqa: F401
+            if hasattr(torch, 'xpu') and torch.xpu.is_available():
+                return "xpu"
+        except ImportError:
+            pass
+        # Any GPU on Windows via DirectML (torch-directml)
+        try:
+            import torch_directml
+            if torch_directml.device_count() > 0:
+                return torch_directml.device(0)
+        except ImportError:
+            pass
+        # MPS (Apple Silicon) — kept for completeness but MLX backend is preferred
+        if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+            return "cpu"  # MPS disabled for stability; MLX backend handles Apple Silicon
         return "cpu"
     
     def is_loaded(self) -> bool:
@@ -166,11 +180,21 @@ class PyTorchTTSBackend:
 
             # Load the model (tqdm is patched, but filters out non-download progress)
             try:
-                self.model = Qwen3TTSModel.from_pretrained(
+                # Don't pass device_map on CPU: accelerate's meta-tensor mechanism
-                    model_path,
+                # causes "Cannot copy out of meta tensor" when moving to CPU.
-                    device_map=self.device,
+                # Instead load directly then call .to(device) if needed.
-                    torch_dtype=torch.float32 if self.device == "cpu" else torch.bfloat16,
+                if self.device == "cpu":
-                )
+                    self.model = Qwen3TTSModel.from_pretrained(
+                        model_path,
+                        torch_dtype=torch.float32,
+                        low_cpu_mem_usage=False,
+                    )
+                else:
+                    self.model = Qwen3TTSModel.from_pretrained(
+                        model_path,
+                        device_map=self.device,
+                        torch_dtype=torch.bfloat16,
+                    )
             finally:
                 # Exit the patch context
                 tracker_context.__exit__(None, None, None)
@@ -358,9 +382,22 @@ class PyTorchSTTBackend:
         """Get the best available device."""
         if torch.cuda.is_available():
             return "cuda"
-        elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+        # Intel Arc / Intel Xe GPU via intel-extension-for-pytorch (IPEX)
-            # MPS support for Whisper
+        try:
-            return "cpu"  # Use CPU for stability
+            import intel_extension_for_pytorch  # noqa: F401
+            if hasattr(torch, 'xpu') and torch.xpu.is_available():
+                return "xpu"
+        except ImportError:
+            pass
+        # Any GPU on Windows via DirectML (torch-directml)
+        try:
+            import torch_directml
+            if torch_directml.device_count() > 0:
+                return torch_directml.device(0)
+        except ImportError:
+            pass
+        if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+            return "cpu"  # MPS disabled for stability
         return "cpu"
     
     def is_loaded(self) -> bool:

backend/config.py

@@ -4,8 +4,17 @@ Configuration module for voicebox backend.
 Handles data directory configuration for production bundling.
 """
 
+import os
 from pathlib import Path
 
+# Allow users to override the HuggingFace model download directory.
+# Set VOICEBOX_MODELS_DIR to an absolute path before starting the server.
+# This sets HF_HUB_CACHE so all huggingface_hub downloads go to that path.
+_custom_models_dir = os.environ.get("VOICEBOX_MODELS_DIR")
+if _custom_models_dir:
+    os.environ["HF_HUB_CACHE"] = _custom_models_dir
+    print(f"[config] Model download path set to: {_custom_models_dir}")
+
 # Default data directory (used in development)
 _data_dir = Path("data")
 

backend/main.py

@@ -77,10 +77,39 @@ async def health():
     tts_model = tts.get_tts_model()
     backend_type = get_backend_type()
 
-    # Check for GPU availability (CUDA or MPS)
+    # Check for GPU availability (CUDA, MPS, Intel Arc XPU, or DirectML)
     has_cuda = torch.cuda.is_available()
     has_mps = hasattr(torch.backends, 'mps') and torch.backends.mps.is_available()
-    gpu_available = has_cuda or has_mps
+
+    # Intel Arc / Intel Xe via intel-extension-for-pytorch (IPEX)
+    has_xpu = False
+    xpu_name = None
+    try:
+        import intel_extension_for_pytorch as ipex  # noqa: F401
+        if hasattr(torch, 'xpu') and torch.xpu.is_available():
+            has_xpu = True
+            try:
+                xpu_name = torch.xpu.get_device_name(0)
+            except Exception:
+                xpu_name = "Intel GPU"
+    except ImportError:
+        pass
+
+    # DirectML backend (torch-directml) for any Windows GPU
+    has_directml = False
+    directml_name = None
+    try:
+        import torch_directml
+        if torch_directml.device_count() > 0:
+            has_directml = True
+            try:
+                directml_name = torch_directml.device_name(0)
+            except Exception:
+                directml_name = "DirectML GPU"
+    except ImportError:
+        pass
+
+    gpu_available = has_cuda or has_mps or has_xpu or has_directml or backend_type == "mlx"
 
     gpu_type = None
     if has_cuda:
@@ -89,6 +118,10 @@ async def health():
         gpu_type = "MPS (Apple Silicon)"
     elif backend_type == "mlx":
         gpu_type = "Metal (Apple Silicon via MLX)"
+    elif has_xpu:
+        gpu_type = f"XPU ({xpu_name})"
+    elif has_directml:
+        gpu_type = f"DirectML ({directml_name})"
 
     vram_used = None
     if has_cuda:
@@ -252,12 +285,17 @@ async def add_profile_sample(
     db: Session = Depends(get_db),
 ):
     """Add a sample to a voice profile."""
-    # Save uploaded file to temporary location
+    # Preserve the uploaded file's extension so librosa can detect format correctly.
-    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
+    # Defaulting to .wav was causing soundfile to reject MP3/WebM content as invalid WAV.
+    _allowed_audio_exts = {'.wav', '.mp3', '.m4a', '.ogg', '.flac', '.aac', '.webm', '.opus'}
+    _uploaded_ext = Path(file.filename or '').suffix.lower()
+    file_suffix = _uploaded_ext if _uploaded_ext in _allowed_audio_exts else '.wav'
+
+    with tempfile.NamedTemporaryFile(suffix=file_suffix, delete=False) as tmp:
         content = await file.read()
         tmp.write(content)
         tmp_path = tmp.name
-    
+
     try:
         sample = await profiles.add_profile_sample(
             profile_id,
@@ -268,6 +306,8 @@ async def add_profile_sample(
         return sample
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to process audio file: {str(e)}")
     finally:
         # Clean up temp file
         Path(tmp_path).unlink(missing_ok=True)
@@ -541,48 +581,49 @@ async def generate_speech(
         profile = await profiles.get_profile(data.profile_id, db)
         if not profile:
             raise HTTPException(status_code=404, detail="Profile not found")
-        
+
-        # Create voice prompt from profile
+        # Resolve model size and load the correct model FIRST.
-        voice_prompt = await profiles.create_voice_prompt_for_profile(
+        # This must happen before create_voice_prompt_for_profile because that
-            data.profile_id,
+        # function calls load_model_async(None), which falls back to self.model_size.
-            db,
+        # If the model is already loaded with the right size at that point, it
-        )
+        # returns immediately and the voice prompt is created by the correct model.
-        
-        # Generate audio
         tts_model = tts.get_tts_model()
-        # Load the requested model size if different from current (async to not block)
         model_size = data.model_size or "1.7B"
 
         # Check if model needs to be downloaded first
         model_path = tts_model._get_model_path(model_size)
-        if model_path.startswith("Qwen/"):
+        if not tts_model._is_model_cached(model_size):
-            # Model not cached - check if it exists remotely or needs download
+            # Model is not fully cached — kick off a background download and tell
-            from huggingface_hub import constants as hf_constants
+            # the client to retry once it's ready.
-            repo_cache = Path(hf_constants.HF_HUB_CACHE) / ("models--" + model_path.replace("/", "--"))
+            model_name = f"qwen-tts-{model_size}"
-            if not repo_cache.exists():
-                # Start download in background
-                model_name = f"qwen-tts-{model_size}"
 
-                async def download_model_background():
+            async def download_model_background():
-                    try:
+                try:
-                        await tts_model.load_model_async(model_size)
+                    await tts_model.load_model_async(model_size)
-                    except Exception as e:
+                except Exception as e:
-                        task_manager.error_download(model_name, str(e))
+                    task_manager.error_download(model_name, str(e))
 
-                task_manager.start_download(model_name)
+            task_manager.start_download(model_name)
-                asyncio.create_task(download_model_background())
+            asyncio.create_task(download_model_background())
 
-                # Return 202 Accepted with download info
+            raise HTTPException(
-                raise HTTPException(
+                status_code=202,
-                    status_code=202,
+                detail={
-                    detail={
+                    "message": f"Model {model_size} is being downloaded. Please wait and try again.",
-                        "message": f"Model {model_size} is being downloaded. Please wait and try again.",
+                    "model_name": model_name,
-                        "model_name": model_name,
+                    "downloading": True,
-                        "downloading": True
+                },
-                    }
+            )
-                )
 
+        # Load (or switch to) the requested model before building the voice prompt
         await tts_model.load_model_async(model_size)
+
+        # Create voice prompt from profile (model is already loaded with correct size)
+        voice_prompt = await profiles.create_voice_prompt_for_profile(
+            data.profile_id,
+            db,
+        )
+
         audio, sample_rate = await tts_model.generate(
             data.text,
             voice_prompt,
@@ -625,6 +666,59 @@ async def generate_speech(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@app.post("/generate/stream")
+async def stream_speech(
+    data: models.GenerationRequest,
+    db: Session = Depends(get_db),
+):
+    """
+    Generate speech and stream the WAV audio directly without saving to disk.
+
+    Returns raw WAV bytes via a StreamingResponse so the client can start
+    playing audio before the entire file has been received.  This endpoint
+    does NOT create a history entry — use /generate for that.
+    """
+    profile = await profiles.get_profile(data.profile_id, db)
+    if not profile:
+        raise HTTPException(status_code=404, detail="Profile not found")
+
+    tts_model = tts.get_tts_model()
+    model_size = data.model_size or "1.7B"
+
+    if not tts_model._is_model_cached(model_size):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Model {model_size} is not downloaded yet. Use /generate to trigger a download.",
+        )
+
+    # Load the correct model before building the voice prompt (fixes issue #96)
+    await tts_model.load_model_async(model_size)
+
+    voice_prompt = await profiles.create_voice_prompt_for_profile(data.profile_id, db)
+
+    audio, sample_rate = await tts_model.generate(
+        data.text,
+        voice_prompt,
+        data.language,
+        data.seed,
+        data.instruct,
+    )
+
+    wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
+
+    async def _wav_stream():
+        # Yield in chunks so large responses don't block the event loop
+        chunk_size = 64 * 1024  # 64 KB
+        for i in range(0, len(wav_bytes), chunk_size):
+            yield wav_bytes[i : i + chunk_size]
+
+    return StreamingResponse(
+        _wav_stream(),
+        media_type="audio/wav",
+        headers={"Content-Disposition": 'attachment; filename="speech.wav"'},
+    )
+
+
 # ============================================
 # HISTORY ENDPOINTS
 # ============================================

backend/tts.py

@@ -32,11 +32,3 @@ def audio_to_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes:
     sf.write(buffer, audio, sample_rate, format="WAV")
     buffer.seek(0)
     return buffer.read()
-
-
-def audio_to_wav_bytes(audio: np.ndarray, sample_rate: int) -> bytes:
-    """Convert audio array to WAV bytes."""
-    buffer = io.BytesIO()
-    sf.write(buffer, audio, sample_rate, format="WAV")
-    buffer.seek(0)
-    return buffer.read()

web/package.json

@@ -21,6 +21,7 @@
     "@types/react-dom": "^18.3.0",
     "@typescript-eslint/eslint-plugin": "^7.0.0",
     "@typescript-eslint/parser": "^7.0.0",
+    "@tailwindcss/vite": "^4.0.0",
     "@vitejs/plugin-react": "^4.3.0",
     "eslint": "^8.57.0",
     "eslint-plugin-react-hooks": "^4.6.0",

web/vite.config.ts

@@ -1,9 +1,10 @@
 import path from 'node:path';
 import react from '@vitejs/plugin-react';
+import tailwindcss from '@tailwindcss/vite';
 import { defineConfig } from 'vite';
 
 export default defineConfig({
-  plugins: [react()],
+  plugins: [react(), tailwindcss()],
   resolve: {
     alias: {
       '@': path.resolve(__dirname, '../app/src'),