Merge branch 'main' into channels

tauri/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@voicebox/tauri",
   "private": true,
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "scripts": {
     "dev": "vite",

tauri/src-tauri/Cargo.lock

@@ -3836,6 +3836,16 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "tauri-plugin-process"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d55511a7bf6cd70c8767b02c97bf8134fa434daf3926cfc1be0a0f94132d165a"
+dependencies = [
+ "tauri",
+ "tauri-plugin",
+]
+
 [[package]]
 name = "tauri-plugin-shell"
 version = "2.3.4"
@@ -4483,13 +4493,14 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "voicebox"
-version = "0.1.1"
+version = "0.1.3"
 dependencies = [
  "base64 0.22.1",
  "core-foundation-sys",
  "coreaudio-sys",
  "hound",
  "objc",
+ "scopeguard",
  "screencapturekit",
  "serde",
  "serde_json",
@@ -4497,6 +4508,7 @@ dependencies = [
  "tauri-build",
  "tauri-plugin-dialog",
  "tauri-plugin-fs",
+ "tauri-plugin-process",
  "tauri-plugin-shell",
  "tauri-plugin-updater",
  "tokio",

tauri/src-tauri/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "voicebox"
-version = "0.1.1"
+version = "0.1.3"
 description = "A production-quality desktop app for Qwen3-TTS voice cloning and generation"
 authors = ["you"]
 license = ""
@@ -24,6 +24,7 @@ hound = "3.5"
 base64 = "0.22"
 cpal = "0.15"
 symphonia = { version = "0.5", features = ["wav", "pcm"] }
+scopeguard = "1.2.0"
 
 [target.'cfg(target_os = "macos")'.dependencies]
 screencapturekit = { version = "1", features = ["async"] }
@@ -33,10 +34,11 @@ core-foundation-sys = "0.8"
 
 [target.'cfg(target_os = "windows")'.dependencies]
 wasapi = "0.22"
-windows = { version = "0.62", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }
+windows = { version = "0.62", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging", "Win32_System_Com"] }
 
 [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
 tauri-plugin-updater = "2.0"
+tauri-plugin-process = "2.0"
 
 [features]
 # This feature is used for production builds or when `devPath` points to the filesystem

tauri/src-tauri/Entitlements.plist

@@ -8,5 +8,7 @@
     <true/>
     <key>com.apple.security.cs.disable-library-validation</key>
     <true/>
+    <key>com.apple.security.device.audio-input</key>
+    <true/>
 </dict>
 </plist>

tauri/src-tauri/capabilities/default.json

@@ -17,6 +17,7 @@
     "shell:allow-execute",
     "shell:allow-spawn",
     "updater:default",
+    "process:default",
     "dialog:default",
     "dialog:allow-save",
     "dialog:allow-open",

tauri/src-tauri/gen/schemas/capabilities.json

@@ -1 +1 @@
-{"default":{"identifier":"default","description":"Default permissions for voicebox","remote":{"urls":["http://localhost:*"]},"local":true,"windows":["main"],"permissions":["core:default","core:window:default","core:window:allow-start-dragging","core:webview:default","core:webview:allow-internal-toggle-devtools","shell:allow-open","shell:allow-execute","shell:allow-spawn","updater:default","dialog:default","dialog:allow-save","dialog:allow-open","fs:default","fs:read-all","fs:write-all"],"platforms":["linux","macOS","windows"]}}
+{"default":{"identifier":"default","description":"Default permissions for voicebox","remote":{"urls":["http://localhost:*"]},"local":true,"windows":["main"],"permissions":["core:default","core:window:default","core:window:allow-start-dragging","core:webview:default","core:webview:allow-internal-toggle-devtools","shell:allow-open","shell:allow-execute","shell:allow-spawn","updater:default","process:default","dialog:default","dialog:allow-save","dialog:allow-open","fs:default","fs:read-all","fs:write-all"],"platforms":["linux","macOS","windows"]}}

tauri/src-tauri/gen/schemas/desktop-schema.json

@@ -5948,6 +5948,36 @@
           "const": "fs:write-files",
           "markdownDescription": "This enables all file write related commands without any pre-configured accessible paths."
         },
+        {
+          "description": "This permission set configures which\nprocess features are by default exposed.\n\n#### Granted Permissions\n\nThis enables to quit via `allow-exit` and restart via `allow-restart`\nthe application.\n\n#### This default permission set includes:\n\n- `allow-exit`\n- `allow-restart`",
+          "type": "string",
+          "const": "process:default",
+          "markdownDescription": "This permission set configures which\nprocess features are by default exposed.\n\n#### Granted Permissions\n\nThis enables to quit via `allow-exit` and restart via `allow-restart`\nthe application.\n\n#### This default permission set includes:\n\n- `allow-exit`\n- `allow-restart`"
+        },
+        {
+          "description": "Enables the exit command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:allow-exit",
+          "markdownDescription": "Enables the exit command without any pre-configured scope."
+        },
+        {
+          "description": "Enables the restart command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:allow-restart",
+          "markdownDescription": "Enables the restart command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the exit command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:deny-exit",
+          "markdownDescription": "Denies the exit command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the restart command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:deny-restart",
+          "markdownDescription": "Denies the restart command without any pre-configured scope."
+        },
         {
           "description": "This permission set configures which\nshell functionality is exposed by default.\n\n#### Granted Permissions\n\nIt allows to use the `open` functionality with a reasonable\nscope pre-configured. It will allow opening `http(s)://`,\n`tel:` and `mailto:` links.\n\n#### This default permission set includes:\n\n- `allow-open`",
           "type": "string",

tauri/src-tauri/gen/schemas/macOS-schema.json

@@ -5948,6 +5948,36 @@
           "const": "fs:write-files",
           "markdownDescription": "This enables all file write related commands without any pre-configured accessible paths."
         },
+        {
+          "description": "This permission set configures which\nprocess features are by default exposed.\n\n#### Granted Permissions\n\nThis enables to quit via `allow-exit` and restart via `allow-restart`\nthe application.\n\n#### This default permission set includes:\n\n- `allow-exit`\n- `allow-restart`",
+          "type": "string",
+          "const": "process:default",
+          "markdownDescription": "This permission set configures which\nprocess features are by default exposed.\n\n#### Granted Permissions\n\nThis enables to quit via `allow-exit` and restart via `allow-restart`\nthe application.\n\n#### This default permission set includes:\n\n- `allow-exit`\n- `allow-restart`"
+        },
+        {
+          "description": "Enables the exit command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:allow-exit",
+          "markdownDescription": "Enables the exit command without any pre-configured scope."
+        },
+        {
+          "description": "Enables the restart command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:allow-restart",
+          "markdownDescription": "Enables the restart command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the exit command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:deny-exit",
+          "markdownDescription": "Denies the exit command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the restart command without any pre-configured scope.",
+          "type": "string",
+          "const": "process:deny-restart",
+          "markdownDescription": "Denies the restart command without any pre-configured scope."
+        },
         {
           "description": "This permission set configures which\nshell functionality is exposed by default.\n\n#### Granted Permissions\n\nIt allows to use the `open` functionality with a reasonable\nscope pre-configured. It will allow opening `http(s)://`,\n`tel:` and `mailto:` links.\n\n#### This default permission set includes:\n\n- `allow-open`",
           "type": "string",

tauri/src-tauri/src/audio_capture/macos.rs

@@ -163,25 +163,67 @@ fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, Stri
         .audio_buffer_list()
         .ok_or_else(|| "Failed to get audio buffer list".to_string())?;
 
-    let mut samples = Vec::new();
+    let buffers: Vec<_> = audio_buffer_list.iter().collect();
+    let num_buffers = buffers.len();
+    
+    if num_buffers == 0 {
+        return Ok(Vec::new());
+    }
 
-    // Iterate through audio buffers
-    for buffer in audio_buffer_list.iter() {
-        // Get raw bytes and interpret as f32 samples
+    // ScreenCaptureKit on macOS provides audio in Float32 format
+    // The audio can be either:
+    // - Interleaved (1 buffer with L,R,L,R,... samples)
+    // - Planar (2 buffers, one for L channel, one for R channel)
+    
+    if num_buffers == 1 {
+        // Interleaved stereo or mono in a single buffer
+        let buffer = &buffers[0];
         let data_bytes = buffer.data();
         let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
         
         if num_samples > 0 {
             unsafe {
-                // Interpret bytes as f32 samples
                 let data_ptr = data_bytes.as_ptr() as *const f32;
                 let data = std::slice::from_raw_parts(data_ptr, num_samples);
-                samples.extend_from_slice(data);
+                return Ok(data.to_vec());
             }
         }
+    } else {
+        // Planar format - separate buffer for each channel
+        // We need to interleave them: L0, R0, L1, R1, ...
+        let mut channel_data: Vec<Vec<f32>> = Vec::new();
+        let mut max_samples = 0;
+        
+        for buffer in &buffers {
+            let data_bytes = buffer.data();
+            let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
+            
+            if num_samples > 0 {
+                unsafe {
+                    let data_ptr = data_bytes.as_ptr() as *const f32;
+                    let data = std::slice::from_raw_parts(data_ptr, num_samples);
+                    channel_data.push(data.to_vec());
+                    max_samples = max_samples.max(num_samples);
+                }
+            }
+        }
+        
+        // Interleave the channels
+        let mut interleaved = Vec::with_capacity(max_samples * num_buffers);
+        for i in 0..max_samples {
+            for channel in &channel_data {
+                if i < channel.len() {
+                    interleaved.push(channel[i]);
+                } else {
+                    interleaved.push(0.0); // Pad with silence if needed
+                }
+            }
+        }
+        
+        return Ok(interleaved);
     }
 
-    Ok(samples)
+    Ok(Vec::new())
 }
 
 fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {

tauri/src-tauri/src/audio_capture/mod.rs

@@ -18,6 +18,7 @@ pub struct AudioCaptureState {
     pub sample_rate: Arc<Mutex<u32>>,
     pub channels: Arc<Mutex<u16>>,
     pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
+    pub error: Arc<Mutex<Option<String>>>,
     #[cfg(target_os = "macos")]
     pub stream: Arc<Mutex<Option<SCStream>>>,
 }
@@ -29,6 +30,7 @@ impl AudioCaptureState {
             sample_rate: Arc::new(Mutex::new(44100)),
             channels: Arc::new(Mutex::new(2)),
             stop_tx: Arc::new(Mutex::new(None)),
+            error: Arc::new(Mutex::new(None)),
             #[cfg(target_os = "macos")]
             stream: Arc::new(Mutex::new(None)),
         }
@@ -36,5 +38,6 @@ impl AudioCaptureState {
 
     pub fn reset(&self) {
         *self.samples.lock().unwrap() = Vec::new();
+        *self.error.lock().unwrap() = None;
     }
 }

tauri/src-tauri/src/audio_capture/windows.rs

@@ -5,8 +5,8 @@ use std::io::Cursor;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::thread;
-use std::time::Duration;
 use wasapi::*;
+use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED};
 
 pub async fn start_capture(
     state: &AudioCaptureState,
@@ -19,6 +19,7 @@ pub async fn start_capture(
     let sample_rate_arc = state.sample_rate.clone();
     let channels_arc = state.channels.clone();
     let stop_tx = state.stop_tx.clone();
+    let error_arc = state.error.clone();
 
     // Use AtomicBool for stop signal (works with non-Send types)
     let stop_flag = Arc::new(AtomicBool::new(false));
@@ -36,13 +37,29 @@ pub async fn start_capture(
     // Spawn capture task on a dedicated thread (WASAPI COM objects are not Send)
     // All WASAPI objects must be created and used on the same thread
     thread::spawn(move || {
+        // Initialize COM for this thread
+        unsafe {
+            let hr = CoInitializeEx(None, COINIT_MULTITHREADED);
+            if hr.is_err() {
+                eprintln!("Failed to initialize COM: {:?}", hr);
+                return;
+            }
+        }
+
+        // Ensure COM is uninitialized when thread exits
+        let _com_guard = scopeguard::guard((), |_| unsafe {
+            CoUninitialize();
+        });
+
         // Initialize WASAPI on this thread
         let device = match DeviceEnumerator::new()
             .and_then(|enumerator| enumerator.get_default_device(&Direction::Render))
         {
             Ok(d) => d,
             Err(e) => {
-                eprintln!("Failed to get audio device: {}", e);
+                let error_msg = format!("Failed to get audio device: {}", e);
+                eprintln!("{}", error_msg);
+                *error_arc.lock().unwrap() = Some(error_msg);
                 return;
             }
         };
@@ -50,7 +67,9 @@ pub async fn start_capture(
         let mut audio_client = match device.get_iaudioclient() {
             Ok(client) => client,
             Err(e) => {
-                eprintln!("Failed to get audio client: {}", e);
+                let error_msg = format!("Failed to get audio client: {}", e);
+                eprintln!("{}", error_msg);
+                *error_arc.lock().unwrap() = Some(error_msg);
                 return;
             }
         };
@@ -58,7 +77,9 @@ pub async fn start_capture(
         let mix_format = match audio_client.get_mixformat() {
             Ok(format) => format,
             Err(e) => {
-                eprintln!("Failed to get mix format: {}", e);
+                let error_msg = format!("Failed to get mix format: {}", e);
+                eprintln!("{}", error_msg);
+                *error_arc.lock().unwrap() = Some(error_msg);
                 return;
             }
         };
@@ -69,27 +90,53 @@ pub async fn start_capture(
         *sample_rate_arc.lock().unwrap() = mix_format.get_samplespersec();
         *channels_arc.lock().unwrap() = mix_format.get_nchannels();
 
+        // Get device period
+        let (_def_period, min_period) = match audio_client.get_device_period() {
+            Ok(periods) => periods,
+            Err(e) => {
+                eprintln!("Failed to get device period: {}", e);
+                return;
+            }
+        };
+
         // Initialize audio client for loopback with StreamMode
+        // For loopback mode: get Render device, initialize with Capture direction
+        // This triggers AUDCLNT_STREAMFLAGS_LOOPBACK in the wasapi crate
         let stream_mode = StreamMode::EventsShared {
-            autoconvert: false,
-            buffer_duration_hns: 0, // 0 = use default buffer size
+            autoconvert: true,  // Enable automatic format conversion
+            buffer_duration_hns: min_period, // Use minimum period
         };
 
         if let Err(e) = audio_client.initialize_client(&mix_format, &Direction::Capture, &stream_mode) {
-            eprintln!("Failed to initialize audio client: {}", e);
+            let error_msg = format!("Failed to initialize audio client: {}", e);
+            eprintln!("{}", error_msg);
+            *error_arc.lock().unwrap() = Some(error_msg);
             return;
         }
 
+        // Set up event handle for EventsShared mode
+        let h_event = match audio_client.set_get_eventhandle() {
+            Ok(event) => event,
+            Err(e) => {
+                eprintln!("Failed to set event handle: {}", e);
+                return;
+            }
+        };
+
         let capture_client = match audio_client.get_audiocaptureclient() {
             Ok(client) => client,
             Err(e) => {
-                eprintln!("Failed to get capture client: {}", e);
+                let error_msg = format!("Failed to get capture client: {}", e);
+                eprintln!("{}", error_msg);
+                *error_arc.lock().unwrap() = Some(error_msg);
                 return;
             }
         };
 
         if let Err(e) = audio_client.start_stream() {
-            eprintln!("Failed to start stream: {}", e);
+            let error_msg = format!("Failed to start stream: {}", e);
+            eprintln!("{}", error_msg);
+            *error_arc.lock().unwrap() = Some(error_msg);
             return;
         }
 
@@ -145,8 +192,10 @@ pub async fn start_capture(
                 }
             }
 
-            // Sleep briefly to avoid busy-waiting
-            thread::sleep(Duration::from_millis(10));
+            // Wait for event signal (with timeout to allow checking stop flag)
+            if h_event.wait_for_event(100).is_err() {
+                // Timeout is expected - just continue to check stop flag
+            }
         }
 
         // Stop the stream when done
@@ -176,13 +225,18 @@ pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
     // Wait a bit for capture to stop
     tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
 
+    // Check if there was an error during capture
+    if let Some(error) = state.error.lock().unwrap().as_ref() {
+        return Err(error.clone());
+    }
+
     // Get samples
     let samples = state.samples.lock().unwrap().clone();
     let sample_rate = *state.sample_rate.lock().unwrap();
     let channels = *state.channels.lock().unwrap();
 
     if samples.is_empty() {
-        return Err("No audio samples captured".to_string());
+        return Err("No audio samples captured. Make sure audio is playing on your system during recording.".to_string());
     }
 
     // Convert to WAV

tauri/src-tauri/src/lib.rs

@@ -0,0 +1 @@
+pub mod audio_capture;

tauri/src-tauri/src/main.rs

@@ -400,7 +400,10 @@ pub fn run() {
         .manage(audio_output::AudioOutputState::new())
         .setup(|app| {
             #[cfg(desktop)]
-            app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
+            {
+                app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
+                app.handle().plugin(tauri_plugin_process::init())?;
+            }
 
             // Hide title bar icon on Windows
             #[cfg(windows)]

tauri/src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://schema.tauri.app/config/2",
   "productName": "Voicebox",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "identifier": "sh.voicebox.app",
   "build": {
     "beforeDevCommand": "bun run dev",

tauri/src-tauri/tests/audio_capture_test.rs

@@ -0,0 +1,59 @@
+// NOTE: This test requires system audio to be playing during execution.
+// To run this test successfully:
+//   1. Start playing audio (music, video, etc.)
+//   2. Run: cargo test --test audio_capture_test -- --nocapture
+//   3. The test will capture audio for 5 seconds and verify the output
+
+use voicebox::audio_capture::{AudioCaptureState, start_capture, stop_capture};
+use base64::Engine;
+
+#[tokio::test]
+async fn test_system_audio_capture() {
+    // Create AudioCaptureState
+    let state = AudioCaptureState::new();
+
+    println!("Starting system audio capture with 5 second max duration...");
+
+    // Start capture with 5 second max duration
+    let result = start_capture(&state, 5).await;
+
+    if let Err(e) = result {
+        panic!("Failed to start capture: {}", e);
+    }
+
+    println!("Capture started, waiting 5 seconds...");
+
+    // Wait 5 seconds for capture to complete
+    tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+    println!("Stopping capture...");
+
+    // Stop capture and get the result
+    let audio_data = stop_capture(&state).await;
+
+    match audio_data {
+        Ok(base64_wav) => {
+            println!("Capture stopped successfully");
+
+            // Validate the returned base64 WAV data
+            println!("Validating base64 WAV data...");
+
+            // Decode base64 to bytes
+            let decoded_bytes = base64::engine::general_purpose::STANDARD
+                .decode(&base64_wav)
+                .expect("Failed to decode base64 data");
+
+            // Verify bytes array is not empty
+            assert!(!decoded_bytes.is_empty(), "Decoded bytes array is empty");
+
+            // Confirm data has content (length > 0)
+            println!("WAV data length: {} bytes", decoded_bytes.len());
+            assert!(decoded_bytes.len() > 0, "WAV data has no content");
+
+            println!("✓ Test passed: Audio capture produced valid WAV data");
+        }
+        Err(e) => {
+            panic!("Failed to stop capture or get audio data: {}", e);
+        }
+    }
+}