From 36031a0df51efdee22a59bf3535c7552446dcff6 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Mon, 26 Jan 2026 20:40:10 -0800 Subject: [PATCH] Enhance audio capture functionality and update dependencies - Added support for capturing system audio on Windows using WASAPI with improved error handling and thread safety. - Introduced the 'scopeguard' crate for better resource management during audio capture. - Updated Cargo.toml to include 'scopeguard' and modified Windows-specific dependencies for enhanced functionality. - Added a new test for validating audio capture output, ensuring the captured audio data is valid and non-empty. --- backend/voicebox-server.spec | 5 +- tauri/src-tauri/Cargo.lock | 1 + tauri/src-tauri/Cargo.toml | 3 +- tauri/src-tauri/src/audio_capture/windows.rs | 46 +++++++++++++-- tauri/src-tauri/src/lib.rs | 1 + tauri/src-tauri/tests/audio_capture_test.rs | 59 ++++++++++++++++++++ 6 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 tauri/src-tauri/src/lib.rs create mode 100644 tauri/src-tauri/tests/audio_capture_test.rs diff --git a/backend/voicebox-server.spec b/backend/voicebox-server.spec index 85069188..71d87865 100644 --- a/backend/voicebox-server.spec +++ b/backend/voicebox-server.spec @@ -4,15 +4,16 @@ from PyInstaller.utils.hooks import collect_submodules from PyInstaller.utils.hooks import copy_metadata datas = [] -hiddenimports = ['backend', 'backend.main', 'backend.config', 'backend.database', 'backend.models', 'backend.profiles', 'backend.history', 'backend.tts', 'backend.transcribe', 'backend.utils.audio', 'backend.utils.cache', 'backend.utils.progress', 'backend.utils.hf_progress', 'backend.utils.validation', 'torch', 'transformers', 'fastapi', 'uvicorn', 'sqlalchemy', 'librosa', 'soundfile', 'qwen_tts', 'qwen_tts.inference', 'qwen_tts.inference.qwen3_tts_model', 'qwen_tts.inference.qwen3_tts_tokenizer', 'qwen_tts.core', 'qwen_tts.cli'] +hiddenimports = ['backend', 'backend.main', 'backend.config', 'backend.database', 'backend.models', 'backend.profiles', 'backend.history', 'backend.tts', 'backend.transcribe', 'backend.utils.audio', 'backend.utils.cache', 'backend.utils.progress', 'backend.utils.hf_progress', 'backend.utils.validation', 'torch', 'transformers', 'fastapi', 'uvicorn', 'sqlalchemy', 'librosa', 'soundfile', 'qwen_tts', 'qwen_tts.inference', 'qwen_tts.inference.qwen3_tts_model', 'qwen_tts.inference.qwen3_tts_tokenizer', 'qwen_tts.core', 'qwen_tts.cli', 'pkg_resources.extern'] datas += collect_data_files('qwen_tts') datas += copy_metadata('qwen-tts') hiddenimports += collect_submodules('qwen_tts') +hiddenimports += collect_submodules('jaraco') a = Analysis( ['server.py'], - pathex=['/Users/jamespine/Projects/voice/Qwen3-TTS'], + pathex=['C:\\Users\\ijame\\Projects\\voice\\Qwen3-TTS'], binaries=[], datas=datas, hiddenimports=hiddenimports, diff --git a/tauri/src-tauri/Cargo.lock b/tauri/src-tauri/Cargo.lock index 61e1fd67..68c78b8a 100644 --- a/tauri/src-tauri/Cargo.lock +++ b/tauri/src-tauri/Cargo.lock @@ -4490,6 +4490,7 @@ dependencies = [ "coreaudio-sys", "hound", "objc", + "scopeguard", "screencapturekit", "serde", "serde_json", diff --git a/tauri/src-tauri/Cargo.toml b/tauri/src-tauri/Cargo.toml index cff60a87..880f01f5 100644 --- a/tauri/src-tauri/Cargo.toml +++ b/tauri/src-tauri/Cargo.toml @@ -22,6 +22,7 @@ serde_json = "1.0" tokio = { version = "1", features = ["full"] } hound = "3.5" base64 = "0.22" +scopeguard = "1.2.0" [target.'cfg(target_os = "macos")'.dependencies] screencapturekit = { version = "1", features = ["async"] } @@ -31,7 +32,7 @@ core-foundation-sys = "0.8" [target.'cfg(target_os = "windows")'.dependencies] wasapi = "0.22" -windows = { version = "0.62", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] } +windows = { version = "0.62", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging", "Win32_System_Com"] } [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] tauri-plugin-updater = "2.0" diff --git a/tauri/src-tauri/src/audio_capture/windows.rs b/tauri/src-tauri/src/audio_capture/windows.rs index 1f5fd40e..15270bfe 100644 --- a/tauri/src-tauri/src/audio_capture/windows.rs +++ b/tauri/src-tauri/src/audio_capture/windows.rs @@ -5,8 +5,8 @@ use std::io::Cursor; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::thread; -use std::time::Duration; use wasapi::*; +use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED}; pub async fn start_capture( state: &AudioCaptureState, @@ -37,6 +37,20 @@ pub async fn start_capture( // Spawn capture task on a dedicated thread (WASAPI COM objects are not Send) // All WASAPI objects must be created and used on the same thread thread::spawn(move || { + // Initialize COM for this thread + unsafe { + let hr = CoInitializeEx(None, COINIT_MULTITHREADED); + if hr.is_err() { + eprintln!("Failed to initialize COM: {:?}", hr); + return; + } + } + + // Ensure COM is uninitialized when thread exits + let _com_guard = scopeguard::guard((), |_| unsafe { + CoUninitialize(); + }); + // Initialize WASAPI on this thread let device = match DeviceEnumerator::new() .and_then(|enumerator| enumerator.get_default_device(&Direction::Render)) @@ -76,10 +90,21 @@ pub async fn start_capture( *sample_rate_arc.lock().unwrap() = mix_format.get_samplespersec(); *channels_arc.lock().unwrap() = mix_format.get_nchannels(); + // Get device period + let (_def_period, min_period) = match audio_client.get_device_period() { + Ok(periods) => periods, + Err(e) => { + eprintln!("Failed to get device period: {}", e); + return; + } + }; + // Initialize audio client for loopback with StreamMode + // For loopback mode: get Render device, initialize with Capture direction + // This triggers AUDCLNT_STREAMFLAGS_LOOPBACK in the wasapi crate let stream_mode = StreamMode::EventsShared { - autoconvert: false, - buffer_duration_hns: 0, // 0 = use default buffer size + autoconvert: true, // Enable automatic format conversion + buffer_duration_hns: min_period, // Use minimum period }; if let Err(e) = audio_client.initialize_client(&mix_format, &Direction::Capture, &stream_mode) { @@ -89,6 +114,15 @@ pub async fn start_capture( return; } + // Set up event handle for EventsShared mode + let h_event = match audio_client.set_get_eventhandle() { + Ok(event) => event, + Err(e) => { + eprintln!("Failed to set event handle: {}", e); + return; + } + }; + let capture_client = match audio_client.get_audiocaptureclient() { Ok(client) => client, Err(e) => { @@ -158,8 +192,10 @@ pub async fn start_capture( } } - // Sleep briefly to avoid busy-waiting - thread::sleep(Duration::from_millis(10)); + // Wait for event signal (with timeout to allow checking stop flag) + if h_event.wait_for_event(100).is_err() { + // Timeout is expected - just continue to check stop flag + } } // Stop the stream when done diff --git a/tauri/src-tauri/src/lib.rs b/tauri/src-tauri/src/lib.rs new file mode 100644 index 00000000..331d594b --- /dev/null +++ b/tauri/src-tauri/src/lib.rs @@ -0,0 +1 @@ +pub mod audio_capture; diff --git a/tauri/src-tauri/tests/audio_capture_test.rs b/tauri/src-tauri/tests/audio_capture_test.rs new file mode 100644 index 00000000..51448335 --- /dev/null +++ b/tauri/src-tauri/tests/audio_capture_test.rs @@ -0,0 +1,59 @@ +// NOTE: This test requires system audio to be playing during execution. +// To run this test successfully: +// 1. Start playing audio (music, video, etc.) +// 2. Run: cargo test --test audio_capture_test -- --nocapture +// 3. The test will capture audio for 5 seconds and verify the output + +use voicebox::audio_capture::{AudioCaptureState, start_capture, stop_capture}; +use base64::Engine; + +#[tokio::test] +async fn test_system_audio_capture() { + // Create AudioCaptureState + let state = AudioCaptureState::new(); + + println!("Starting system audio capture with 5 second max duration..."); + + // Start capture with 5 second max duration + let result = start_capture(&state, 5).await; + + if let Err(e) = result { + panic!("Failed to start capture: {}", e); + } + + println!("Capture started, waiting 5 seconds..."); + + // Wait 5 seconds for capture to complete + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + + println!("Stopping capture..."); + + // Stop capture and get the result + let audio_data = stop_capture(&state).await; + + match audio_data { + Ok(base64_wav) => { + println!("Capture stopped successfully"); + + // Validate the returned base64 WAV data + println!("Validating base64 WAV data..."); + + // Decode base64 to bytes + let decoded_bytes = base64::engine::general_purpose::STANDARD + .decode(&base64_wav) + .expect("Failed to decode base64 data"); + + // Verify bytes array is not empty + assert!(!decoded_bytes.is_empty(), "Decoded bytes array is empty"); + + // Confirm data has content (length > 0) + println!("WAV data length: {} bytes", decoded_bytes.len()); + assert!(decoded_bytes.len() > 0, "WAV data has no content"); + + println!("✓ Test passed: Audio capture produced valid WAV data"); + } + Err(e) => { + panic!("Failed to stop capture or get audio data: {}", e); + } + } +}