mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 06:10:43 -07:00
macos audio capture for sample creation
This commit is contained in:
Jamie Pine
@@ -0,0 +1,213 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use screencapturekit::{
|
||||
cm::CMSampleBuffer,
|
||||
shareable_content::SCShareableContent,
|
||||
stream::{
|
||||
configuration::SCStreamConfiguration,
|
||||
content_filter::SCContentFilter,
|
||||
output_trait::SCStreamOutputTrait,
|
||||
output_type::SCStreamOutputType,
|
||||
sc_stream::SCStream,
|
||||
},
|
||||
};
|
||||
use std::io::Cursor;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
// Get shareable content
|
||||
let content = SCShareableContent::get()
|
||||
.map_err(|e| format!("Failed to get shareable content: {}", e))?;
|
||||
|
||||
// Get first display
|
||||
let displays = content.displays();
|
||||
if displays.is_empty() {
|
||||
return Err("No displays available".to_string());
|
||||
}
|
||||
let display = &displays[0];
|
||||
|
||||
// Create content filter for desktop audio
|
||||
let filter = SCContentFilter::create()
|
||||
.with_display(display)
|
||||
.with_excluding_windows(&[])
|
||||
.build();
|
||||
|
||||
// Create stream configuration - audio only
|
||||
let mut config = SCStreamConfiguration::default();
|
||||
config.set_captures_audio(true);
|
||||
config.set_excludes_current_process_audio(false);
|
||||
config.set_sample_rate(48000); // Use i32 directly
|
||||
config.set_channel_count(2); // Use i32 directly
|
||||
|
||||
// Create stream using builder
|
||||
let (tx, mut rx) = mpsc::channel::<()>(1);
|
||||
*state.stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let sample_rate = state.sample_rate.clone();
|
||||
let channels = state.channels.clone();
|
||||
|
||||
// Set sample rate and channels
|
||||
*sample_rate.lock().unwrap() = 48000;
|
||||
*channels.lock().unwrap() = 2;
|
||||
|
||||
// Create output handler struct
|
||||
struct AudioHandler {
|
||||
samples: Arc<Mutex<Vec<f32>>>,
|
||||
}
|
||||
|
||||
impl SCStreamOutputTrait for AudioHandler {
|
||||
fn did_output_sample_buffer(
|
||||
&self,
|
||||
sample: CMSampleBuffer,
|
||||
_type: SCStreamOutputType,
|
||||
) {
|
||||
if _type == SCStreamOutputType::Audio {
|
||||
if let Ok(audio_samples) = extract_audio_samples(sample) {
|
||||
let mut samples_guard = self.samples.lock().unwrap();
|
||||
samples_guard.extend_from_slice(&audio_samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let handler = AudioHandler {
|
||||
samples: samples.clone(),
|
||||
};
|
||||
|
||||
// Create stream
|
||||
let mut stream = SCStream::new(&filter, &config);
|
||||
|
||||
// Add output handler for audio (order: handler, then output_type)
|
||||
stream.add_output_handler(handler, SCStreamOutputType::Audio);
|
||||
|
||||
// Store stream reference
|
||||
*state.stream.lock().unwrap() = Some(stream.clone());
|
||||
|
||||
stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?;
|
||||
|
||||
// Spawn task to stop after max duration
|
||||
let stream_clone = stream.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => {
|
||||
// Timeout reached
|
||||
}
|
||||
_ = rx.recv() => {
|
||||
// Manual stop
|
||||
}
|
||||
}
|
||||
let _ = stream_clone.stop_capture();
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Stop stream if still active
|
||||
if let Some(stream) = state.stream.lock().unwrap().take() {
|
||||
let _ = stream.stop_capture();
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err("No audio samples captured".to_string());
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
// ScreenCaptureKit requires macOS 12.3+
|
||||
// Check if we're on a supported version
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// Basic check - ScreenCaptureKit should be available on macOS 12.3+
|
||||
true
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, String> {
|
||||
// Use the crate's built-in method to get audio buffer list
|
||||
let audio_buffer_list = sample_buffer
|
||||
.audio_buffer_list()
|
||||
.ok_or_else(|| "Failed to get audio buffer list".to_string())?;
|
||||
|
||||
let mut samples = Vec::new();
|
||||
|
||||
// Iterate through audio buffers
|
||||
for buffer in audio_buffer_list.iter() {
|
||||
// Get raw bytes and interpret as f32 samples
|
||||
let data_bytes = buffer.data();
|
||||
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
|
||||
|
||||
if num_samples > 0 {
|
||||
unsafe {
|
||||
// Interpret bytes as f32 samples
|
||||
let data_ptr = data_bytes.as_ptr() as *const f32;
|
||||
let data = std::slice::from_raw_parts(data_ptr, num_samples);
|
||||
samples.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(samples)
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer = WavWriter::new(cursor, spec)
|
||||
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub use macos::*;
|
||||
#[cfg(target_os = "windows")]
|
||||
pub use windows::*;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use screencapturekit::stream::sc_stream::SCStream;
|
||||
|
||||
pub struct AudioCaptureState {
|
||||
pub samples: Arc<Mutex<Vec<f32>>>,
|
||||
pub sample_rate: Arc<Mutex<u32>>,
|
||||
pub channels: Arc<Mutex<u16>>,
|
||||
pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
|
||||
#[cfg(target_os = "macos")]
|
||||
pub stream: Arc<Mutex<Option<SCStream>>>,
|
||||
}
|
||||
|
||||
impl AudioCaptureState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
samples: Arc::new(Mutex::new(Vec::new())),
|
||||
sample_rate: Arc::new(Mutex::new(44100)),
|
||||
channels: Arc::new(Mutex::new(2)),
|
||||
stop_tx: Arc::new(Mutex::new(None)),
|
||||
#[cfg(target_os = "macos")]
|
||||
stream: Arc::new(Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(&self) {
|
||||
*self.samples.lock().unwrap() = Vec::new();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use std::io::Cursor;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::sync::mpsc;
|
||||
use wasapi::*;
|
||||
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
// Get default audio render device for loopback
|
||||
let device = DeviceEnumerator::new()
|
||||
.map_err(|e| format!("Failed to create device enumerator: {}", e))?
|
||||
.get_default_audio_endpoint(&Direction::Render)
|
||||
.map_err(|e| format!("Failed to get default render device: {}", e))?;
|
||||
|
||||
// Create audio client for loopback capture
|
||||
let audio_client = device
|
||||
.get_iaudioclient()
|
||||
.map_err(|e| format!("Failed to get audio client: {}", e))?;
|
||||
|
||||
// Get mix format
|
||||
let mix_format = audio_client
|
||||
.get_mixformat()
|
||||
.map_err(|e| format!("Failed to get mix format: {}", e))?;
|
||||
|
||||
// Set sample rate and channels
|
||||
*state.sample_rate.lock().unwrap() = mix_format.get_samples_per_sec();
|
||||
*state.channels.lock().unwrap() = mix_format.get_nchannels();
|
||||
|
||||
// Initialize audio client for loopback
|
||||
audio_client
|
||||
.initialize_client(
|
||||
&mix_format,
|
||||
0, // Buffer duration (0 = default)
|
||||
&Direction::Capture,
|
||||
ShareMode::Shared,
|
||||
true, // Loopback mode
|
||||
)
|
||||
.map_err(|e| format!("Failed to initialize audio client: {}", e))?;
|
||||
|
||||
// Get capture client
|
||||
let capture_client = audio_client
|
||||
.get_audiocaptureclient()
|
||||
.map_err(|e| format!("Failed to get capture client: {}", e))?;
|
||||
|
||||
// Start capture
|
||||
audio_client
|
||||
.start_stream()
|
||||
.map_err(|e| format!("Failed to start stream: {}", e))?;
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let stop_tx = state.stop_tx.clone();
|
||||
let (tx, mut rx) = mpsc::channel::<()>(1);
|
||||
*stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
// Spawn capture task - move audio_client and capture_client into the task
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = rx.recv() => {
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(tokio::time::Duration::from_millis(10)) => {
|
||||
// Try to get available data
|
||||
match capture_client.get_available_samples() {
|
||||
Ok(available) => {
|
||||
if available > 0 {
|
||||
match capture_client.get_buffer::<f32>() {
|
||||
Ok((data, flags)) => {
|
||||
if flags.contains(&StreamFlags::SILENT) {
|
||||
// Silent buffer, skip
|
||||
capture_client.release_buffer(available).ok();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Convert samples to f32 and store
|
||||
let mut samples_guard = samples.lock().unwrap();
|
||||
samples_guard.extend_from_slice(data);
|
||||
capture_client.release_buffer(available).ok();
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error getting buffer: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error getting available samples: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the stream when done
|
||||
audio_client.stop_stream().ok();
|
||||
});
|
||||
|
||||
// Spawn timeout task
|
||||
let stop_tx_clone = state.stop_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
|
||||
if let Some(tx) = stop_tx_clone.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err("No audio samples captured".to_string());
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
true
|
||||
}
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer = WavWriter::new(cursor, spec)
|
||||
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
mod audio_capture;
|
||||
|
||||
use std::sync::Mutex;
|
||||
use tauri::{command, State, Manager, WindowEvent, Emitter, Listener};
|
||||
use tauri_plugin_shell::ShellExt;
|
||||
@@ -165,6 +167,26 @@ async fn stop_server(state: State<'_, ServerState>) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[command]
|
||||
async fn start_system_audio_capture(
|
||||
state: State<'_, audio_capture::AudioCaptureState>,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
audio_capture::start_capture(&state, max_duration_secs).await
|
||||
}
|
||||
|
||||
#[command]
|
||||
async fn stop_system_audio_capture(
|
||||
state: State<'_, audio_capture::AudioCaptureState>,
|
||||
) -> Result<String, String> {
|
||||
audio_capture::stop_capture(&state).await
|
||||
}
|
||||
|
||||
#[command]
|
||||
fn is_system_audio_supported() -> bool {
|
||||
audio_capture::is_supported()
|
||||
}
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
tauri::Builder::default()
|
||||
@@ -174,6 +196,7 @@ pub fn run() {
|
||||
.manage(ServerState {
|
||||
child: Mutex::new(None),
|
||||
})
|
||||
.manage(audio_capture::AudioCaptureState::new())
|
||||
.setup(|app| {
|
||||
#[cfg(desktop)]
|
||||
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
|
||||
@@ -190,7 +213,13 @@ pub fn run() {
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.invoke_handler(tauri::generate_handler![start_server, stop_server])
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
start_server,
|
||||
stop_server,
|
||||
start_system_audio_capture,
|
||||
stop_system_audio_capture,
|
||||
is_system_audio_supported
|
||||
])
|
||||
.on_window_event(|window, event| {
|
||||
if let WindowEvent::CloseRequested { api, .. } = event {
|
||||
// Prevent automatic close
|
||||
|
||||
Reference in New Issue
Block a user