Initial commit (forked from jamiepine/voicebox)

This commit is contained in:
2026-08-24 19:40:39 -07:00
commit eaef8dd838
677 changed files with 129576 additions and 0 deletions
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.0 MiB

+29
View File
@@ -0,0 +1,29 @@
{
"fill": "automatic",
"groups": [
{
"layers": [
{
"image-name": "Voicebox_Microphone.png",
"name": "Voicebox_Microphone",
"position": {
"scale": 0.36,
"translation-in-points": [0.140625, 270.1875]
}
}
],
"shadow": {
"kind": "neutral",
"opacity": 0.5
},
"translucency": {
"enabled": true,
"value": 0.5
}
}
],
"supported-platforms": {
"circles": ["watchOS"],
"squares": "shared"
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 MiB

+28
View File
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>voicebox</title>
<script>
(function () {
try {
var theme = 'system';
var raw = localStorage.getItem('voicebox-ui');
if (raw) {
var parsed = JSON.parse(raw);
if (parsed && parsed.state && parsed.state.theme) theme = parsed.state.theme;
}
var resolved = theme === 'system'
? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light')
: theme;
if (resolved === 'dark') document.documentElement.classList.add('dark');
} catch (_) {}
})();
</script>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
+30
View File
@@ -0,0 +1,30 @@
{
"name": "@voicebox/tauri",
"private": true,
"version": "0.5.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"tauri": "tauri"
},
"dependencies": {
"@tauri-apps/api": "^2.0.0",
"@tauri-apps/plugin-dialog": "^2.0.0",
"@tauri-apps/plugin-fs": "^2.0.0",
"@tauri-apps/plugin-process": "^2.0.0",
"@tauri-apps/plugin-shell": "^2.0.0",
"@tauri-apps/plugin-updater": "^2.0.0"
},
"devDependencies": {
"@tailwindcss/vite": "^4.1.18",
"@tauri-apps/cli": "^2.0.0",
"@types/react": "^18.3.0",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react": "^4.3.0",
"tailwindcss": "^4.1.18",
"tailwindcss-animate": "^1.0.7",
"typescript": "^5.6.0",
"vite": "^5.4.0"
}
}
+6167
View File
File diff suppressed because it is too large Load Diff
+67
View File
@@ -0,0 +1,67 @@
[package]
name = "voicebox"
version = "0.5.0"
description = "A production-quality desktop app for Qwen3-TTS voice cloning and generation"
authors = ["you"]
license = ""
repository = ""
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[build-dependencies]
tauri-build = { version = "2.0", features = [] }
[dependencies]
tauri = { version = "2.0", features = ["macos-private-api"] }
tauri-plugin-dialog = "2.0"
tauri-plugin-fs = "2.0"
tauri-plugin-shell = "2.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["full"] }
reqwest = { version = "0.12", features = ["blocking", "json", "stream"] }
hound = "3.5"
base64 = "0.22"
cpal = "0.15"
symphonia = { version = "0.5", features = ["all"] }
scopeguard = "1.2.0"
[target.'cfg(target_os = "macos")'.dependencies]
screencapturekit = { version = "1", features = ["async"] }
coreaudio-sys = "0.2"
objc = "0.2"
core-foundation-sys = "0.8"
[target.'cfg(target_os = "windows")'.dependencies]
wasapi = "0.22"
windows = { version = "0.62", features = [
"Win32_Foundation",
"Win32_UI_WindowsAndMessaging",
"Win32_UI_Accessibility",
"Win32_UI_Input_KeyboardAndMouse",
"Win32_System_Com",
"Win32_System_DataExchange",
"Win32_System_Memory",
"Win32_System_Threading",
] }
[target.'cfg(target_os = "linux")'.dependencies]
webkit2gtk = "2.0"
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
tauri-plugin-updater = "2.0"
tauri-plugin-process = "2.0"
# Observe-only global keyboard tap. Covers macOS, Windows, and Linux
# (evdev) with left/right modifier fidelity, which is the hard
# requirement the chord engine needs. keytap is our own crate
# (jamiepine/keytap), published to crates.io, that replaces the
# abandoned Narsil/rdev we previously pinned via git — same capability
# surface, clean shutdown via Drop, no `set_is_main_thread(false)`
# dance required (the Sonoma-crashing layout-translation path simply
# isn't called).
keytap = "0.4"
[features]
# This feature is used for production builds or when `devPath` points to the filesystem
custom-protocol = ["tauri/custom-protocol"]
+16
View File
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.cs.allow-jit</key>
<true/>
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
<true/>
<key>com.apple.security.cs.disable-library-validation</key>
<true/>
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.files.user-selected.read-write</key>
<true/>
</dict>
</plist>
+14
View File
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIconFile</key>
<string>voicebox</string>
<key>CFBundleIconName</key>
<string>voicebox</string>
<key>NSMicrophoneUsageDescription</key>
<string>voicebox needs microphone access to record voice samples for voice cloning.</string>
<key>NSScreenCaptureUsageDescription</key>
<string>Voicebox needs screen capture access to record system audio for voice samples.</string>
</dict>
</plist>
+170
View File
@@ -0,0 +1,170 @@
#[cfg(target_os = "macos")]
use std::process::Command;
fn main() {
// Link Swift runtime libraries for screencapturekit crate
#[cfg(target_os = "macos")]
{
// ScreenCaptureKit does not exist on macOS 11, so weak-link it to
// allow the app to launch and gate usage at runtime instead.
println!("cargo:rustc-link-arg=-Wl,-weak_framework,ScreenCaptureKit");
// Add Swift runtime library paths to RPATH
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
println!("cargo:rustc-link-arg=-L/usr/lib/swift");
// Also try Xcode's Swift libraries
if let Ok(output) = Command::new("xcode-select").arg("-p").output() {
if output.status.success() {
let xcode_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
let swift_lib_path = format!(
"{}/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx",
xcode_path
);
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", swift_lib_path);
println!("cargo:rustc-link-arg=-L{}", swift_lib_path);
}
}
}
let project_root = env!("CARGO_MANIFEST_DIR");
let gen_dir = format!("{}/gen", project_root);
std::fs::create_dir_all(&gen_dir).expect("Failed to create gen directory");
// Compile macOS Liquid Glass icon
#[cfg(target_os = "macos")]
{
// voicebox.icon is in tauri/assets/voicebox.icon (one level up from src-tauri)
let icon_source = format!("{}/../assets/voicebox.icon", project_root);
if std::path::Path::new(&icon_source).exists() {
println!("cargo:rerun-if-changed={}", icon_source);
println!("cargo:rerun-if-changed={}/icon.json", icon_source);
println!("cargo:rerun-if-changed={}/Assets", icon_source);
let partial_plist = format!("{}/partial.plist", gen_dir);
let output = Command::new("xcrun")
.args([
"actool",
"--compile",
&gen_dir,
"--output-format",
"human-readable-text",
"--output-partial-info-plist",
&partial_plist,
"--app-icon",
"voicebox",
"--include-all-app-icons",
"--target-device",
"mac",
"--minimum-deployment-target",
"11.0",
"--platform",
"macosx",
&icon_source,
])
.output();
match output {
Ok(output) => {
if !output.status.success() {
eprintln!("actool stderr: {}", String::from_utf8_lossy(&output.stderr));
eprintln!("actool stdout: {}", String::from_utf8_lossy(&output.stdout));
panic!("actool failed to compile icon");
}
println!("Successfully compiled icon to {}", gen_dir);
}
Err(e) => {
eprintln!("Failed to execute xcrun actool: {}", e);
eprintln!("Make sure you have Xcode Command Line Tools installed");
panic!("Icon compilation failed");
}
}
// Generate voicebox.icns from the source PNG via sips + iconutil
let icns_path = format!("{}/voicebox.icns", gen_dir);
if !std::path::Path::new(&icns_path).exists() {
let source_png = format!("{}/Assets/Voicebox.png", icon_source);
if std::path::Path::new(&source_png).exists() {
let iconset_dir = format!("{}/voicebox.iconset", gen_dir);
std::fs::create_dir_all(&iconset_dir).ok();
let sizes: &[(u32, &str)] = &[
(16, "icon_16x16.png"),
(32, "[email protected]"),
(32, "icon_32x32.png"),
(64, "[email protected]"),
(128, "icon_128x128.png"),
(256, "[email protected]"),
(256, "icon_256x256.png"),
(512, "[email protected]"),
(512, "icon_512x512.png"),
(1024, "[email protected]"),
];
for (size, name) in sizes {
let dest = format!("{}/{}", iconset_dir, name);
let status = Command::new("sips")
.args([
"-z",
&size.to_string(),
&size.to_string(),
&source_png,
"--out",
&dest,
])
.output();
if let Ok(out) = status {
if !out.status.success() {
eprintln!(
"sips failed for {}: {}",
name,
String::from_utf8_lossy(&out.stderr)
);
}
}
}
let iconutil_output = Command::new("iconutil")
.args(["-c", "icns", "-o", &icns_path, &iconset_dir])
.output();
match iconutil_output {
Ok(out) if out.status.success() => {
println!("Generated voicebox.icns");
}
Ok(out) => {
eprintln!("iconutil failed: {}", String::from_utf8_lossy(&out.stderr));
}
Err(e) => {
eprintln!("Failed to run iconutil: {}", e);
}
}
// Clean up iconset directory
std::fs::remove_dir_all(&iconset_dir).ok();
}
}
} else {
println!(
"cargo:warning=Icon source not found at {}, skipping icon compilation",
icon_source
);
}
}
// Ensure all resource files exist so Tauri's bundler doesn't fail.
// On non-macOS these are always stubs. On macOS, actool may not produce
// Assets.car if the Xcode version doesn't support the .icon format.
{
let required = ["Assets.car", "voicebox.icns", "partial.plist"];
for name in required {
let path = format!("{}/{}", gen_dir, name);
if !std::path::Path::new(&path).exists() {
std::fs::write(&path, b"").ok();
}
}
}
tauri_build::build()
}
+28
View File
@@ -0,0 +1,28 @@
{
"$schema": "https://schema.tauri.app/config/2",
"identifier": "default",
"description": "Default permissions for voicebox",
"platforms": ["linux", "macOS", "windows"],
"windows": ["main", "dictate"],
"remote": {
"urls": ["http://localhost:*"]
},
"permissions": [
"core:default",
"core:window:default",
"core:window:allow-start-dragging",
"core:webview:default",
"core:webview:allow-internal-toggle-devtools",
"shell:allow-open",
"shell:allow-execute",
"shell:allow-spawn",
"updater:default",
"process:default",
"dialog:default",
"dialog:allow-save",
"dialog:allow-open",
"fs:default",
"fs:read-all",
"fs:write-all"
]
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
{"default":{"identifier":"default","description":"Default permissions for voicebox","remote":{"urls":["http://localhost:*"]},"local":true,"windows":["main","dictate"],"permissions":["core:default","core:window:default","core:window:allow-start-dragging","core:webview:default","core:webview:allow-internal-toggle-devtools","shell:allow-open","shell:allow-execute","shell:allow-spawn","updater:default","process:default","dialog:default","dialog:allow-save","dialog:allow-open","fs:default","fs:read-all","fs:write-all"],"platforms":["linux","macOS","windows"]}}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 128 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.1 KiB

@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<foreground android:drawable="@mipmap/ic_launcher_foreground"/>
<background android:drawable="@color/ic_launcher_background"/>
</adaptive-icon>
Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<color name="ic_launcher_background">#fff</color>
</resources>
Binary file not shown.
Binary file not shown.

After

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 311 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 296 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 514 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 514 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 831 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 388 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 793 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 793 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 514 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.3 KiB

+42
View File
@@ -0,0 +1,42 @@
//! Platform permission gate for the auto-paste pipeline.
//!
//! On macOS, posting synthetic keyboard events and reading focused-UI state
//! via the AX API both require the host process to be listed under System
//! Settings → Privacy & Security → Accessibility. Without that trust,
//! `CGEventPost` silently drops events and `AXUIElementCopyAttributeValue`
//! returns an error. We surface a boolean check up front so the paste
//! pipeline can short-circuit with a clear "grant permission" message
//! instead of running through the full save → write → post → restore dance
//! with nothing to show for it.
//!
//! Windows has no equivalent user-facing permission — `SendInput` and
//! UIAutomation work for any non-elevated target out of the box. (UAC /
//! UIPI still blocks sending input *into* an elevated target window from a
//! non-elevated process, but that's per-target, not a global switch, and
//! there's no Settings pane to send users to.) So the Windows branch just
//! returns `true`.
#[cfg(target_os = "macos")]
mod ffi {
#[link(name = "ApplicationServices", kind = "framework")]
extern "C" {
/// Returns true when the current process is listed in Accessibility.
/// No prompt side-effect.
pub fn AXIsProcessTrusted() -> bool;
}
}
#[cfg(target_os = "macos")]
pub fn is_trusted() -> bool {
unsafe { ffi::AXIsProcessTrusted() }
}
#[cfg(target_os = "windows")]
pub fn is_trusted() -> bool {
true
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn is_trusted() -> bool {
false
}
+382
View File
@@ -0,0 +1,382 @@
use crate::audio_capture::AudioCaptureState;
use base64::{engine::general_purpose, Engine as _};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{SampleFormat, StreamConfig};
use hound::{WavSpec, WavWriter};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread;
/// Try to find a PulseAudio/PipeWire monitor source using `pactl`.
/// Returns the source name (e.g. "alsa_output.pci-0000_0d_00.6.analog-stereo.monitor") if found.
fn find_monitor_source_via_pactl() -> Option<String> {
let output = std::process::Command::new("pactl")
.args(["list", "short", "sources"])
.output()
.ok()?;
if !output.status.success() {
return None;
}
let stdout = String::from_utf8_lossy(&output.stdout);
// First, try to find the monitor of the default sink
let default_sink = std::process::Command::new("pactl")
.args(["get-default-sink"])
.output()
.ok()
.and_then(|o| {
if o.status.success() {
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
} else {
None
}
});
// If we know the default sink, look for its .monitor specifically
if let Some(sink_name) = &default_sink {
let monitor_name = format!("{}.monitor", sink_name);
for line in stdout.lines() {
let parts: Vec<&str> = line.split('\t').collect();
if parts.len() >= 2 && parts[1] == monitor_name {
eprintln!(
"Linux audio capture: Found default sink monitor via pactl: {}",
monitor_name
);
return Some(monitor_name);
}
}
}
// Fallback: find any .monitor source
for line in stdout.lines() {
let parts: Vec<&str> = line.split('\t').collect();
if parts.len() >= 2 && parts[1].ends_with(".monitor") {
let name = parts[1].to_string();
eprintln!(
"Linux audio capture: Found monitor source via pactl: {}",
name
);
return Some(name);
}
}
None
}
/// Select the capture device: prefer an exact match against the monitor
/// source name reported by `pactl`, then fall back to any device whose name
/// contains "monitor", then the host's default input device.
fn select_capture_device(host: &cpal::Host, monitor_source: Option<&str>) -> Option<cpal::Device> {
let devices: Vec<cpal::Device> = host.input_devices().ok()?.collect();
if let Some(target) = monitor_source {
if let Some(pos) = devices
.iter()
.position(|d| d.name().map(|n| n == target).unwrap_or(false))
{
eprintln!(
"Linux audio capture: Using pactl monitor device: {}",
target
);
return devices.into_iter().nth(pos);
}
}
if let Some(pos) = devices.iter().position(|d| {
d.name()
.map(|n| n.to_lowercase().contains("monitor"))
.unwrap_or(false)
}) {
let name = devices[pos].name().unwrap_or_default();
eprintln!("Linux audio capture: Found monitor device by name: {}", name);
return devices.into_iter().nth(pos);
}
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
host.default_input_device()
}
/// Start capturing system audio on Linux using PulseAudio monitor sources.
///
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
/// monitor source via `pactl`, then select the matching cpal input device by
/// name. This avoids mutating the process environment (`PULSE_SOURCE`), which
/// is not thread-safe and would affect every thread in the process. If `pactl`
/// is unavailable, we fall back to searching cpal device names for "monitor".
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
) -> Result<(), String> {
// Reset previous samples
state.reset();
let samples = state.samples.clone();
let sample_rate_arc = state.sample_rate.clone();
let channels_arc = state.channels.clone();
let stop_tx = state.stop_tx.clone();
let error_arc = state.error.clone();
// Use AtomicBool for stop signal (works across threads)
let stop_flag = Arc::new(AtomicBool::new(false));
let stop_flag_clone = stop_flag.clone();
// Create tokio channel and spawn a task to bridge it to the AtomicBool
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
*stop_tx.lock().unwrap() = Some(tx);
tokio::spawn(async move {
rx.recv().await;
stop_flag_clone.store(true, Ordering::Relaxed);
});
// Spawn capture on a dedicated thread
thread::spawn(move || {
let host = cpal::default_host();
let monitor_source = find_monitor_source_via_pactl();
let device = match select_capture_device(&host, monitor_source.as_deref()) {
Some(d) => d,
None => {
let error_msg = "No audio input device available".to_string();
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
eprintln!("Linux audio capture: Using device: {}", device_name);
// Get supported config
let config = match device.default_input_config() {
Ok(c) => c,
Err(e) => {
let error_msg = format!("Failed to get default input config: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
let sample_rate = config.sample_rate().0;
let channels = config.channels();
let sample_format = config.sample_format();
eprintln!(
"Linux audio capture: Config - {}Hz, {} channels, format: {:?}",
sample_rate, channels, sample_format
);
*sample_rate_arc.lock().unwrap() = sample_rate;
*channels_arc.lock().unwrap() = channels;
let stream_config = StreamConfig {
channels,
sample_rate: cpal::SampleRate(sample_rate),
buffer_size: cpal::BufferSize::Default,
};
let samples_clone = samples.clone();
let error_arc_clone = error_arc.clone();
let stop_flag_for_stream = stop_flag.clone();
let err_fn = {
let error_arc = error_arc.clone();
move |err: cpal::StreamError| {
let error_msg = format!("Stream error: {}", err);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
}
};
let stream = match sample_format {
SampleFormat::F32 => {
let samples = samples_clone.clone();
let stop = stop_flag_for_stream.clone();
device.build_input_stream(
&stream_config,
move |data: &[f32], _: &cpal::InputCallbackInfo| {
if stop.load(Ordering::Relaxed) {
return;
}
let mut guard = samples.lock().unwrap();
guard.extend_from_slice(data);
},
err_fn,
None,
)
}
SampleFormat::I16 => {
let samples = samples_clone.clone();
let stop = stop_flag_for_stream.clone();
device.build_input_stream(
&stream_config,
move |data: &[i16], _: &cpal::InputCallbackInfo| {
if stop.load(Ordering::Relaxed) {
return;
}
let mut guard = samples.lock().unwrap();
for &s in data {
guard.push(s as f32 / 32768.0);
}
},
err_fn,
None,
)
}
SampleFormat::U16 => {
let samples = samples_clone.clone();
let stop = stop_flag_for_stream.clone();
device.build_input_stream(
&stream_config,
move |data: &[u16], _: &cpal::InputCallbackInfo| {
if stop.load(Ordering::Relaxed) {
return;
}
let mut guard = samples.lock().unwrap();
for &s in data {
guard.push((s as f32 / 32768.0) - 1.0);
}
},
err_fn,
None,
)
}
_ => {
let error_msg = format!("Unsupported sample format: {:?}", sample_format);
eprintln!("{}", error_msg);
*error_arc_clone.lock().unwrap() = Some(error_msg);
return;
}
};
let stream = match stream {
Ok(s) => s,
Err(e) => {
let error_msg = format!("Failed to build input stream: {}", e);
eprintln!("{}", error_msg);
*error_arc_clone.lock().unwrap() = Some(error_msg);
return;
}
};
if let Err(e) = stream.play() {
let error_msg = format!("Failed to start stream: {}", e);
eprintln!("{}", error_msg);
*error_arc_clone.lock().unwrap() = Some(error_msg);
return;
}
eprintln!("Linux audio capture: Stream started successfully");
// Keep thread alive until stop signal
loop {
if stop_flag.load(Ordering::Relaxed) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
// Stream will be dropped here, stopping capture
eprintln!("Linux audio capture: Stream stopped");
});
// Spawn timeout task
let stop_tx_clone = state.stop_tx.clone();
tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
let tx = stop_tx_clone.lock().unwrap().take();
if let Some(tx) = tx {
let _ = tx.send(()).await;
}
});
Ok(())
}
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
// Signal stop
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
let _ = tx.send(());
}
// Wait a bit for capture to stop
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Check if there was an error during capture
if let Some(error) = state.error.lock().unwrap().as_ref() {
return Err(error.clone());
}
// Get samples
let samples = state.samples.lock().unwrap().clone();
let sample_rate = *state.sample_rate.lock().unwrap();
let channels = *state.channels.lock().unwrap();
if samples.is_empty() {
return Err(
"No audio samples captured. Make sure audio is playing on your system during recording."
.to_string(),
);
}
// Convert to WAV
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
// Encode to base64
let base64_data = general_purpose::STANDARD.encode(&wav_data);
Ok(base64_data)
}
pub fn is_supported() -> bool {
// Check via pactl first (most reliable on modern Linux)
if find_monitor_source_via_pactl().is_some() {
return true;
}
// Fallback: check cpal devices
let host = cpal::default_host();
if let Ok(devices) = host.input_devices() {
for d in devices {
if let Ok(name) = d.name() {
if name.to_lowercase().contains("monitor") {
return true;
}
}
}
}
host.default_input_device().is_some()
}
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
let mut buffer = Vec::new();
let cursor = Cursor::new(&mut buffer);
let spec = WavSpec {
channels,
sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer =
WavWriter::new(cursor, spec).map_err(|e| format!("Failed to create WAV writer: {}", e))?;
// Convert f32 samples to i16
for sample in samples {
let clamped = sample.clamp(-1.0, 1.0);
let i16_sample = (clamped * 32767.0) as i16;
writer
.write_sample(i16_sample)
.map_err(|e| format!("Failed to write sample: {}", e))?;
}
writer
.finalize()
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
Ok(buffer)
}
+265
View File
@@ -0,0 +1,265 @@
use crate::audio_capture::AudioCaptureState;
use base64::{engine::general_purpose, Engine as _};
use hound::{WavSpec, WavWriter};
use screencapturekit::{
cm::CMSampleBuffer,
shareable_content::SCShareableContent,
stream::{
configuration::SCStreamConfiguration,
content_filter::SCContentFilter,
output_trait::SCStreamOutputTrait,
output_type::SCStreamOutputType,
sc_stream::SCStream,
},
};
use std::io::Cursor;
use std::process::Command;
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
) -> Result<(), String> {
if !is_supported() {
return Err("System audio capture requires macOS 12.3 or newer.".to_string());
}
// Reset previous samples
state.reset();
// Get shareable content
let content = SCShareableContent::get()
.map_err(|e| format!("Failed to get shareable content: {}", e))?;
// Get first display
let displays = content.displays();
if displays.is_empty() {
return Err("No displays available".to_string());
}
let display = &displays[0];
// Create content filter for desktop audio
let filter = SCContentFilter::create()
.with_display(display)
.with_excluding_windows(&[])
.build();
// Create stream configuration - audio only
let mut config = SCStreamConfiguration::default();
config.set_captures_audio(true);
config.set_excludes_current_process_audio(false);
config.set_sample_rate(48000); // Use i32 directly
config.set_channel_count(2); // Use i32 directly
// Create stream using builder
let (tx, mut rx) = mpsc::channel::<()>(1);
*state.stop_tx.lock().unwrap() = Some(tx);
let samples = state.samples.clone();
let sample_rate = state.sample_rate.clone();
let channels = state.channels.clone();
// Set sample rate and channels
*sample_rate.lock().unwrap() = 48000;
*channels.lock().unwrap() = 2;
// Create output handler struct
struct AudioHandler {
samples: Arc<Mutex<Vec<f32>>>,
}
impl SCStreamOutputTrait for AudioHandler {
fn did_output_sample_buffer(
&self,
sample: CMSampleBuffer,
_type: SCStreamOutputType,
) {
if _type == SCStreamOutputType::Audio {
if let Ok(audio_samples) = extract_audio_samples(sample) {
let mut samples_guard = self.samples.lock().unwrap();
samples_guard.extend_from_slice(&audio_samples);
}
}
}
}
let handler = AudioHandler {
samples: samples.clone(),
};
// Create stream
let mut stream = SCStream::new(&filter, &config);
// Add output handler for audio (order: handler, then output_type)
stream.add_output_handler(handler, SCStreamOutputType::Audio);
// Store stream reference
*state.stream.lock().unwrap() = Some(stream.clone());
stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?;
// Spawn task to stop after max duration
let stream_clone = stream.clone();
tokio::spawn(async move {
tokio::select! {
_ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => {
// Timeout reached
}
_ = rx.recv() => {
// Manual stop
}
}
let _ = stream_clone.stop_capture();
});
Ok(())
}
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
// Signal stop
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
let _ = tx.send(());
}
// Stop stream if still active
if let Some(stream) = state.stream.lock().unwrap().take() {
let _ = stream.stop_capture();
}
// Wait a bit for capture to stop
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Get samples
let samples = state.samples.lock().unwrap().clone();
let sample_rate = *state.sample_rate.lock().unwrap();
let channels = *state.channels.lock().unwrap();
if samples.is_empty() {
return Err("No audio samples captured".to_string());
}
// Convert to WAV
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
// Encode to base64
let base64_data = general_purpose::STANDARD.encode(&wav_data);
Ok(base64_data)
}
pub fn is_supported() -> bool {
macos_version_at_least(12, 3)
}
fn macos_version_at_least(required_major: u64, required_minor: u64) -> bool {
let output = match Command::new("sw_vers").arg("-productVersion").output() {
Ok(output) if output.status.success() => output,
_ => return false,
};
let version = String::from_utf8_lossy(&output.stdout);
let mut parts = version.trim().split('.');
let major = parts.next().and_then(|part| part.parse::<u64>().ok()).unwrap_or(0);
let minor = parts.next().and_then(|part| part.parse::<u64>().ok()).unwrap_or(0);
major > required_major || (major == required_major && minor >= required_minor)
}
fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, String> {
// Use the crate's built-in method to get audio buffer list
let audio_buffer_list = sample_buffer
.audio_buffer_list()
.ok_or_else(|| "Failed to get audio buffer list".to_string())?;
let buffers: Vec<_> = audio_buffer_list.iter().collect();
let num_buffers = buffers.len();
if num_buffers == 0 {
return Ok(Vec::new());
}
// ScreenCaptureKit on macOS provides audio in Float32 format
// The audio can be either:
// - Interleaved (1 buffer with L,R,L,R,... samples)
// - Planar (2 buffers, one for L channel, one for R channel)
if num_buffers == 1 {
// Interleaved stereo or mono in a single buffer
let buffer = &buffers[0];
let data_bytes = buffer.data();
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
if num_samples > 0 {
unsafe {
let data_ptr = data_bytes.as_ptr() as *const f32;
let data = std::slice::from_raw_parts(data_ptr, num_samples);
return Ok(data.to_vec());
}
}
} else {
// Planar format - separate buffer for each channel
// We need to interleave them: L0, R0, L1, R1, ...
let mut channel_data: Vec<Vec<f32>> = Vec::new();
let mut max_samples = 0;
for buffer in &buffers {
let data_bytes = buffer.data();
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
if num_samples > 0 {
unsafe {
let data_ptr = data_bytes.as_ptr() as *const f32;
let data = std::slice::from_raw_parts(data_ptr, num_samples);
channel_data.push(data.to_vec());
max_samples = max_samples.max(num_samples);
}
}
}
// Interleave the channels
let mut interleaved = Vec::with_capacity(max_samples * num_buffers);
for i in 0..max_samples {
for channel in &channel_data {
if i < channel.len() {
interleaved.push(channel[i]);
} else {
interleaved.push(0.0); // Pad with silence if needed
}
}
}
return Ok(interleaved);
}
Ok(Vec::new())
}
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
let mut buffer = Vec::new();
let cursor = Cursor::new(&mut buffer);
let spec = WavSpec {
channels,
sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer = WavWriter::new(cursor, spec)
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
// Convert f32 samples to i16
for sample in samples {
let clamped = sample.clamp(-1.0, 1.0);
let i16_sample = (clamped * 32767.0) as i16;
writer.write_sample(i16_sample)
.map_err(|e| format!("Failed to write sample: {}", e))?;
}
writer.finalize()
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
Ok(buffer)
}
+47
View File
@@ -0,0 +1,47 @@
#[cfg(target_os = "macos")]
mod macos;
#[cfg(target_os = "windows")]
mod windows;
#[cfg(target_os = "linux")]
mod linux;
#[cfg(target_os = "macos")]
pub use macos::*;
#[cfg(target_os = "windows")]
pub use windows::*;
#[cfg(target_os = "linux")]
pub use linux::*;
use std::sync::{Arc, Mutex};
#[cfg(target_os = "macos")]
use screencapturekit::stream::sc_stream::SCStream;
pub struct AudioCaptureState {
pub samples: Arc<Mutex<Vec<f32>>>,
pub sample_rate: Arc<Mutex<u32>>,
pub channels: Arc<Mutex<u16>>,
pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
pub error: Arc<Mutex<Option<String>>>,
#[cfg(target_os = "macos")]
pub stream: Arc<Mutex<Option<SCStream>>>,
}
impl AudioCaptureState {
pub fn new() -> Self {
Self {
samples: Arc::new(Mutex::new(Vec::new())),
sample_rate: Arc::new(Mutex::new(44100)),
channels: Arc::new(Mutex::new(2)),
stop_tx: Arc::new(Mutex::new(None)),
error: Arc::new(Mutex::new(None)),
#[cfg(target_os = "macos")]
stream: Arc::new(Mutex::new(None)),
}
}
pub fn reset(&self) {
*self.samples.lock().unwrap() = Vec::new();
*self.error.lock().unwrap() = None;
}
}
@@ -0,0 +1,288 @@
use crate::audio_capture::AudioCaptureState;
use base64::{engine::general_purpose, Engine as _};
use hound::{WavSpec, WavWriter};
use std::io::Cursor;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
use wasapi::*;
use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED};
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
) -> Result<(), String> {
// Reset previous samples
state.reset();
let samples = state.samples.clone();
let sample_rate_arc = state.sample_rate.clone();
let channels_arc = state.channels.clone();
let stop_tx = state.stop_tx.clone();
let error_arc = state.error.clone();
// Use AtomicBool for stop signal (works with non-Send types)
let stop_flag = Arc::new(AtomicBool::new(false));
let stop_flag_clone = stop_flag.clone();
// Create tokio channel and spawn a task to bridge it to the AtomicBool
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
*stop_tx.lock().unwrap() = Some(tx);
tokio::spawn(async move {
rx.recv().await;
stop_flag_clone.store(true, Ordering::Relaxed);
});
// Spawn capture task on a dedicated thread (WASAPI COM objects are not Send)
// All WASAPI objects must be created and used on the same thread
thread::spawn(move || {
// Initialize COM for this thread
unsafe {
let hr = CoInitializeEx(None, COINIT_MULTITHREADED);
if hr.is_err() {
eprintln!("Failed to initialize COM: {:?}", hr);
return;
}
}
// Ensure COM is uninitialized when thread exits
let _com_guard = scopeguard::guard((), |_| unsafe {
CoUninitialize();
});
// Initialize WASAPI on this thread
let device = match DeviceEnumerator::new()
.and_then(|enumerator| enumerator.get_default_device(&Direction::Render))
{
Ok(d) => d,
Err(e) => {
let error_msg = format!("Failed to get audio device: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
let mut audio_client = match device.get_iaudioclient() {
Ok(client) => client,
Err(e) => {
let error_msg = format!("Failed to get audio client: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
let mix_format = match audio_client.get_mixformat() {
Ok(format) => format,
Err(e) => {
let error_msg = format!("Failed to get mix format: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
// Set sample rate and channels
let channels = mix_format.get_nchannels() as usize;
let bytes_per_sample = (mix_format.get_bitspersample() / 8) as usize;
*sample_rate_arc.lock().unwrap() = mix_format.get_samplespersec();
*channels_arc.lock().unwrap() = mix_format.get_nchannels();
// Get device period
let (_def_period, min_period) = match audio_client.get_device_period() {
Ok(periods) => periods,
Err(e) => {
eprintln!("Failed to get device period: {}", e);
return;
}
};
// Initialize audio client for loopback with StreamMode
// For loopback mode: get Render device, initialize with Capture direction
// This triggers AUDCLNT_STREAMFLAGS_LOOPBACK in the wasapi crate
let stream_mode = StreamMode::EventsShared {
autoconvert: true, // Enable automatic format conversion
buffer_duration_hns: min_period, // Use minimum period
};
if let Err(e) = audio_client.initialize_client(&mix_format, &Direction::Capture, &stream_mode) {
let error_msg = format!("Failed to initialize audio client: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
// Set up event handle for EventsShared mode
let h_event = match audio_client.set_get_eventhandle() {
Ok(event) => event,
Err(e) => {
eprintln!("Failed to set event handle: {}", e);
return;
}
};
let capture_client = match audio_client.get_audiocaptureclient() {
Ok(client) => client,
Err(e) => {
let error_msg = format!("Failed to get capture client: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
};
if let Err(e) = audio_client.start_stream() {
let error_msg = format!("Failed to start stream: {}", e);
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
loop {
// Check if stop signal was received
if stop_flag.load(Ordering::Relaxed) {
break;
}
// Try to get available data
match capture_client.get_next_packet_size() {
Ok(Some(frames_available)) => {
if frames_available > 0 {
// Calculate buffer size needed (frames * channels * bytes_per_sample)
let buffer_size = frames_available as usize * channels * bytes_per_sample;
let mut buffer = vec![0u8; buffer_size];
match capture_client.read_from_device(&mut buffer) {
Ok((frames_read, _buffer_info)) => {
if frames_read > 0 {
// Convert bytes to f32 samples
let samples_read = (frames_read as usize * channels) as usize;
let mut samples_guard = samples.lock().unwrap();
// Assuming 32-bit float format
if bytes_per_sample == 4 {
for i in 0..samples_read {
let byte_offset = i * 4;
if byte_offset + 4 <= buffer.len() {
let sample = f32::from_le_bytes([
buffer[byte_offset],
buffer[byte_offset + 1],
buffer[byte_offset + 2],
buffer[byte_offset + 3],
]);
samples_guard.push(sample);
}
}
}
}
}
Err(e) => {
eprintln!("Error reading from device: {}", e);
}
}
}
}
Ok(None) => {
// Exclusive mode - handle differently if needed
}
Err(e) => {
eprintln!("Error getting next packet size: {}", e);
}
}
// Wait for event signal (with timeout to allow checking stop flag)
if h_event.wait_for_event(100).is_err() {
// Timeout is expected - just continue to check stop flag
}
}
// Stop the stream when done
audio_client.stop_stream().ok();
});
// Spawn timeout task
let stop_tx_clone = state.stop_tx.clone();
tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
// Take the sender out of the mutex before awaiting
let tx = stop_tx_clone.lock().unwrap().take();
if let Some(tx) = tx {
let _ = tx.send(()).await;
}
});
Ok(())
}
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
// Signal stop
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
let _ = tx.send(());
}
// Wait a bit for capture to stop
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Check if there was an error during capture
if let Some(error) = state.error.lock().unwrap().as_ref() {
return Err(error.clone());
}
// Get samples
let samples = state.samples.lock().unwrap().clone();
let sample_rate = *state.sample_rate.lock().unwrap();
let channels = *state.channels.lock().unwrap();
if samples.is_empty() {
return Err("No audio samples captured. Make sure audio is playing on your system during recording.".to_string());
}
// Convert to WAV
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
// Encode to base64
let base64_data = general_purpose::STANDARD.encode(&wav_data);
Ok(base64_data)
}
pub fn is_supported() -> bool {
#[cfg(target_os = "windows")]
{
true
}
#[cfg(not(target_os = "windows"))]
{
false
}
}
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
let mut buffer = Vec::new();
let cursor = Cursor::new(&mut buffer);
let spec = WavSpec {
channels,
sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer = WavWriter::new(cursor, spec)
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
// Convert f32 samples to i16
for sample in samples {
let clamped = sample.clamp(-1.0, 1.0);
let i16_sample = (clamped * 32767.0) as i16;
writer.write_sample(i16_sample)
.map_err(|e| format!("Failed to write sample: {}", e))?;
}
writer.finalize()
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
Ok(buffer)
}
+481
View File
@@ -0,0 +1,481 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{Device, Host, SampleFormat, StreamConfig};
use std::sync::{Arc, Mutex};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
#[derive(Debug, Clone, serde::Serialize)]
pub struct AudioOutputDevice {
pub id: String,
pub name: String,
pub is_default: bool,
}
pub struct AudioOutputState {
host: Host,
stop_flag: Arc<AtomicBool>,
}
impl AudioOutputState {
pub fn new() -> Self {
Self {
host: cpal::default_host(),
stop_flag: Arc::new(AtomicBool::new(false)),
}
}
pub fn stop_all_playback(&self) -> Result<(), String> {
eprintln!("stop_all_playback: Setting stop flag");
self.stop_flag.store(true, Ordering::Relaxed);
eprintln!("stop_all_playback: Stop flag set - active streams will output silence");
Ok(())
}
pub fn list_output_devices(&self) -> Result<Vec<AudioOutputDevice>, String> {
let devices = self
.host
.output_devices()
.map_err(|e| format!("Failed to enumerate output devices: {}", e))?;
let default_device = self.host.default_output_device();
let mut result = Vec::new();
for device in devices {
let name = device
.name()
.map_err(|e| format!("Failed to get device name: {}", e))?;
// Generate a stable ID from the device name (cpal doesn't provide stable IDs)
let id = format!("device_{}", name.replace(' ', "_").to_lowercase());
let is_default = default_device
.as_ref()
.map(|d| d.name().unwrap_or_default() == name)
.unwrap_or(false);
result.push(AudioOutputDevice {
id,
name,
is_default,
});
}
Ok(result)
}
pub async fn play_audio_to_devices(
&self,
audio_data: Vec<u8>,
device_ids: Vec<String>,
) -> Result<(), String> {
eprintln!("play_audio_to_devices called with {} bytes, {} device IDs", audio_data.len(), device_ids.len());
eprintln!("Requested device IDs: {:?}", device_ids);
// Decode audio file (assuming WAV format)
eprintln!("Decoding audio data...");
let (samples, sample_rate, channels) = self.decode_wav(&audio_data)?;
eprintln!("Audio decoded: {} samples, {}Hz, {} channels", samples.len(), sample_rate, channels);
// Find devices by ID
eprintln!("Enumerating output devices...");
let devices: Vec<Device> = self
.host
.output_devices()
.map_err(|e| format!("Failed to enumerate devices: {}", e))?
.filter_map(|device| {
let name = device.name().ok()?;
let id = format!("device_{}", name.replace(' ', "_").to_lowercase());
eprintln!("Found device: {} (id: {})", name, id);
if device_ids.contains(&id) {
eprintln!(" -> Matched! Will play to this device");
Some(device)
} else {
None
}
})
.collect();
if devices.is_empty() {
eprintln!("ERROR: No matching devices found");
return Err("No matching devices found".to_string());
}
eprintln!("Playing to {} device(s)", devices.len());
// Stop any existing playback first
self.stop_all_playback().ok();
// Reset stop flag for new playback
self.stop_flag.store(false, Ordering::Relaxed);
// Play to each device
for (i, device) in devices.iter().enumerate() {
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
eprintln!("Playing to device {}/{}: {}", i + 1, devices.len(), device_name);
self.play_to_device(device, samples.clone(), sample_rate, channels, self.stop_flag.clone())
.map_err(|e| format!("Failed to play to device {}: {}", device_name, e))?;
eprintln!("Successfully started playback on device: {}", device_name);
}
eprintln!("play_audio_to_devices completed successfully");
Ok(())
}
fn decode_wav(&self, data: &[u8]) -> Result<(Vec<f32>, u32, u16), String> {
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
eprintln!("decode_wav: Creating MediaSourceStream from {} bytes", data.len());
let mss = MediaSourceStream::new(
Box::new(std::io::Cursor::new(data.to_vec())),
Default::default(),
);
eprintln!("decode_wav: Probing audio format...");
let mut format = symphonia::default::get_probe()
.format(
&Default::default(),
mss,
&FormatOptions::default(),
&MetadataOptions::default(),
)
.map_err(|e| {
eprintln!("decode_wav: Failed to probe audio: {}", e);
format!("Failed to probe audio: {}", e)
})?
.format;
eprintln!("decode_wav: Audio format probed successfully");
eprintln!("decode_wav: Finding audio track...");
let track = format
.tracks()
.iter()
.find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL)
.ok_or_else(|| {
eprintln!("decode_wav: No audio track found");
"No audio track found".to_string()
})?;
let sample_rate = track
.codec_params
.sample_rate
.ok_or_else(|| {
eprintln!("decode_wav: No sample rate found in track");
"No sample rate found".to_string()
})?;
let channels = track
.codec_params
.channels
.ok_or_else(|| {
eprintln!("decode_wav: No channels found in track");
"No channels found".to_string()
})?
.count() as u16;
eprintln!("decode_wav: Track info - sample_rate: {}, channels: {}", sample_rate, channels);
eprintln!("decode_wav: Creating decoder...");
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &Default::default())
.map_err(|e| {
eprintln!("decode_wav: Failed to create decoder: {}", e);
format!("Failed to create decoder: {}", e)
})?;
eprintln!("decode_wav: Decoder created successfully");
let mut samples = Vec::new();
let mut packet_count = 0;
eprintln!("decode_wav: Starting packet decoding loop...");
loop {
let packet = match format.next_packet() {
Ok(packet) => packet,
Err(e) => {
eprintln!("decode_wav: End of stream or error: {:?}", e);
break;
}
};
packet_count += 1;
let decoded = decoder
.decode(&packet)
.map_err(|e| {
eprintln!("decode_wav: Decode error on packet {}: {}", packet_count, e);
format!("Decode error: {}", e)
})?;
// Convert to f32 samples by matching on the buffer type
use symphonia::core::audio::{AudioBufferRef, Signal};
use symphonia::core::conv::FromSample;
let spec = *decoded.spec();
let num_channels = spec.channels.count();
let num_frames = decoded.frames();
eprintln!("decode_wav: Packet {} - {} frames, {} channels", packet_count, num_frames, num_channels);
// Interleave samples from all channels
for frame_idx in 0..num_frames {
for ch in 0..num_channels {
let sample_f32 = match &decoded {
AudioBufferRef::U8(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::U16(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::U24(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::U32(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::S8(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::S16(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::S24(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::S32(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
AudioBufferRef::F32(buf) => buf.chan(ch)[frame_idx],
AudioBufferRef::F64(buf) => buf.chan(ch)[frame_idx] as f32,
};
samples.push(sample_f32);
}
}
}
eprintln!("decode_wav: Decoded {} packets, total {} samples", packet_count, samples.len());
eprintln!("decode_wav: Returning sample_rate={}, channels={}", sample_rate, channels);
Ok((samples, sample_rate, channels))
}
fn play_to_device(
&self,
device: &Device,
samples: Vec<f32>,
sample_rate: u32,
channels: u16,
stop_flag: Arc<AtomicBool>,
) -> Result<(), String> {
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
eprintln!("play_to_device: Starting playback to device: {}", device_name);
eprintln!("play_to_device: Input - {} samples, {}Hz, {} channels", samples.len(), sample_rate, channels);
let config = device
.default_output_config()
.map_err(|e| format!("Failed to get default config: {}", e))?;
// Prepare samples for the device's format
let device_sample_rate = config.sample_rate().0;
let device_channels = config.channels();
let device_sample_format = config.sample_format();
eprintln!("play_to_device: Device config - {}Hz, {} channels, format: {:?}",
device_sample_rate, device_channels, device_sample_format);
// Resample if needed (simple linear interpolation for now)
let resampled = if device_sample_rate != sample_rate {
eprintln!("play_to_device: Resampling from {}Hz to {}Hz", sample_rate, device_sample_rate);
let result = self.resample(&samples, sample_rate, device_sample_rate);
eprintln!("play_to_device: Resampled {} samples to {} samples", samples.len(), result.len());
result
} else {
eprintln!("play_to_device: No resampling needed");
samples
};
// Interleave/convert channels if needed
eprintln!("play_to_device: Interleaving channels from {} to {} channels", channels, device_channels);
let interleaved = self.interleave_channels(&resampled, channels, device_channels);
eprintln!("play_to_device: Interleaved to {} samples", interleaved.len());
// Create shared buffer for playback
let buffer: Arc<Mutex<Vec<f32>>> = Arc::new(Mutex::new(interleaved));
let position = Arc::new(AtomicUsize::new(0));
let buffer_clone = buffer.clone();
let position_clone = position.clone();
let err_fn = |err| eprintln!("Playback error: {}", err);
let stream_config = StreamConfig {
channels: device_channels,
sample_rate: cpal::SampleRate(device_sample_rate),
buffer_size: cpal::BufferSize::Default,
};
let stop_flag_clone = stop_flag.clone();
let stream = match config.sample_format() {
SampleFormat::F32 => {
let buffer = buffer_clone.clone();
let pos = position_clone.clone();
device
.build_output_stream(
&stream_config,
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
// Check stop flag - if set, output silence
if stop_flag_clone.load(Ordering::Relaxed) {
for sample in data.iter_mut() {
*sample = 0.0;
}
return;
}
let mut idx = pos.load(Ordering::Relaxed);
let buf = buffer.lock().unwrap();
for sample in data.iter_mut() {
if idx < buf.len() {
*sample = buf[idx];
idx += 1;
} else {
*sample = 0.0;
}
}
pos.store(idx, Ordering::Relaxed);
},
err_fn,
None,
)
.map_err(|e| format!("Failed to build stream: {}", e))?
}
SampleFormat::I16 => {
let buffer = buffer_clone.clone();
let pos = position_clone.clone();
device
.build_output_stream(
&stream_config,
move |data: &mut [i16], _: &cpal::OutputCallbackInfo| {
// Check stop flag - if set, output silence
if stop_flag_clone.load(Ordering::Relaxed) {
for sample in data.iter_mut() {
*sample = 0;
}
return;
}
let mut idx = pos.load(Ordering::Relaxed);
let buf = buffer.lock().unwrap();
for sample in data.iter_mut() {
if idx < buf.len() {
*sample = (buf[idx] * 32767.0) as i16;
idx += 1;
} else {
*sample = 0;
}
}
pos.store(idx, Ordering::Relaxed);
},
err_fn,
None,
)
.map_err(|e| format!("Failed to build stream: {}", e))?
}
SampleFormat::U16 => {
let buffer = buffer_clone.clone();
let pos = position_clone.clone();
device
.build_output_stream(
&stream_config,
move |data: &mut [u16], _: &cpal::OutputCallbackInfo| {
// Check stop flag - if set, output silence
if stop_flag_clone.load(Ordering::Relaxed) {
for sample in data.iter_mut() {
*sample = 32768;
}
return;
}
let mut idx = pos.load(Ordering::Relaxed);
let buf = buffer.lock().unwrap();
for sample in data.iter_mut() {
if idx < buf.len() {
*sample = ((buf[idx] + 1.0) * 32767.5) as u16;
idx += 1;
} else {
*sample = 32768;
}
}
pos.store(idx, Ordering::Relaxed);
},
err_fn,
None,
)
.map_err(|e| format!("Failed to build stream: {}", e))?
}
_ => return Err("Unsupported sample format".to_string()),
};
eprintln!("play_to_device: Starting stream playback...");
stream.play().map_err(|e| {
eprintln!("play_to_device: Failed to play stream: {}", e);
format!("Failed to play stream: {}", e)
})?;
eprintln!("play_to_device: Stream started successfully");
// Keep the stream alive until playback finishes.
// Previously the stream was dropped immediately on function return,
// causing silent playback (cpal stops output when its Stream is dropped).
let total_samples = {
buffer.lock().unwrap().len()
};
loop {
let pos = position.load(std::sync::atomic::Ordering::Relaxed);
if pos >= total_samples || stop_flag.load(std::sync::atomic::Ordering::Relaxed) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
// stream is dropped here, after audio has finished playing
drop(stream);
eprintln!("play_to_device: Function completed successfully");
Ok(())
}
fn resample(&self, samples: &[f32], from_rate: u32, to_rate: u32) -> Vec<f32> {
if from_rate == to_rate {
return samples.to_vec();
}
let ratio = to_rate as f64 / from_rate as f64;
let new_len = (samples.len() as f64 * ratio) as usize;
let mut resampled = Vec::with_capacity(new_len);
for i in 0..new_len {
let src_idx = (i as f64 / ratio) as usize;
if src_idx < samples.len() {
resampled.push(samples[src_idx]);
} else {
resampled.push(0.0);
}
}
resampled
}
fn interleave_channels(
&self,
samples: &[f32],
src_channels: u16,
dst_channels: u16,
) -> Vec<f32> {
if src_channels == dst_channels {
return samples.to_vec();
}
let mut interleaved = Vec::new();
let samples_per_channel = samples.len() / src_channels as usize;
for i in 0..samples_per_channel {
for ch in 0..dst_channels {
let src_ch = if ch < src_channels { ch } else { src_channels - 1 };
let idx = (i * src_channels as usize) + src_ch as usize;
if idx < samples.len() {
interleaved.push(samples[idx]);
} else {
interleaved.push(0.0);
}
}
}
interleaved
}
}
impl Default for AudioOutputState {
fn default() -> Self {
Self::new()
}
}
+718
View File
@@ -0,0 +1,718 @@
//! Snapshot / write / restore helpers around the system clipboard.
//!
//! Used by the auto-paste flow: before synthesising the paste accelerator
//! into a foreign app we need to (1) remember what the user had on the
//! clipboard, (2) stage our transcribed text, (3) paste, (4) put the
//! original contents back. Missing step 4 turns every dictation into a
//! silent clipboard-stomp.
//!
//! On **macOS** the snapshot walks `NSPasteboard.pasteboardItems` and
//! copies every `(UTI, data)` pair into an owned `Vec<u8>`, so restore
//! rebuilds the full multi-type payload — not just the plain-text
//! fallback. Images, styled text, file-reference lists all survive the
//! round-trip.
//!
//! On **Windows** the snapshot walks `EnumClipboardFormats` and copies the
//! HGLOBAL payload for every advertised format. GDI-handle formats (DIB
//! bitmap, metafile, enhanced metafile, palette), owner-display variants,
//! and the private-/GDI-object format ranges are skipped — those can't be
//! round-tripped across processes without synthesising the underlying
//! kernel/GDI objects, which isn't worth the complexity for a dictation
//! clipboard guard. CF_UNICODETEXT, CF_HDROP, CF_DIB (bitmap data in
//! memory, not a handle), CF_DIBV5, and every registered format (HTML
//! Format, Rich Text Format, FileGroupDescriptor, etc.) all survive.
//!
//! On **macOS** every entry point manages its own `NSAutoreleasePool`
//! because the Tauri command runtime threads don't have one by default —
//! without it, every autoreleased `NSString` / `NSData` we touch would
//! leak for the life of the process. On Windows, HGLOBAL ownership
//! transfers to the clipboard on `SetClipboardData` success, so we only
//! free handles we allocated but didn't hand off.
#[cfg(target_os = "macos")]
use objc::runtime::Object;
#[cfg(target_os = "macos")]
use objc::{class, msg_send, sel, sel_impl};
/// One full-fidelity snapshot of the general pasteboard. Hold on to the value
/// until the paste has landed, then pass it to [`restore_clipboard`].
#[derive(Debug, Clone)]
pub struct ClipboardSnapshot {
/// Outer vec: pasteboard items. Inner: `(uti, raw bytes)` per type. We
/// store the raw UTI string and the raw `NSData` payload so we can rebuild
/// the item with `setData:forType:` without interpreting the contents.
items: Vec<Vec<(String, Vec<u8>)>>,
/// `NSPasteboard.changeCount` at the moment of capture. Incremented by AppKit
/// on every mutation from any process, so a caller can decide whether a
/// restore is still safe (change_count == expected) or whether someone
/// else wrote to the clipboard in the interim and we should back off.
change_count: i64,
}
impl ClipboardSnapshot {
pub fn change_count(&self) -> i64 {
self.change_count
}
pub fn item_count(&self) -> usize {
self.items.len()
}
}
#[cfg(target_os = "macos")]
type Id = *mut Object;
/// RAII wrapper so the pool drains even on early return / `?` propagation.
#[cfg(target_os = "macos")]
struct AutoreleasePool {
pool: Id,
}
#[cfg(target_os = "macos")]
impl AutoreleasePool {
unsafe fn new() -> Self {
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
let pool: Id = msg_send![pool, init];
Self { pool }
}
}
#[cfg(target_os = "macos")]
impl Drop for AutoreleasePool {
fn drop(&mut self) {
unsafe {
let _: () = msg_send![self.pool, drain];
}
}
}
/// Build an autoreleased `NSString` from a Rust `&str` without scanning for
/// interior nulls (which is what `initWithUTF8String:` would require).
#[cfg(target_os = "macos")]
unsafe fn ns_string(s: &str) -> Id {
// NSUTF8StringEncoding = 4.
let obj: Id = msg_send![class!(NSString), alloc];
let obj: Id = msg_send![
obj,
initWithBytes: s.as_ptr()
length: s.len()
encoding: 4u64
];
let _: () = msg_send![obj, autorelease];
obj
}
#[cfg(target_os = "macos")]
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
if s.is_null() {
return None;
}
let bytes: *const i8 = msg_send![s, UTF8String];
if bytes.is_null() {
return None;
}
std::ffi::CStr::from_ptr(bytes)
.to_str()
.ok()
.map(|x| x.to_owned())
}
#[cfg(target_os = "macos")]
unsafe fn general_pasteboard() -> Result<Id, String> {
let pb: Id = msg_send![class!(NSPasteboard), generalPasteboard];
if pb.is_null() {
return Err("NSPasteboard generalPasteboard returned nil".into());
}
Ok(pb)
}
/// Read the pasteboard's current change count without snapshotting contents.
///
/// AppKit increments this every time any process writes to the general
/// pasteboard, so it's a cheap way to detect "did someone clobber my staged
/// text before the paste landed?".
#[cfg(target_os = "macos")]
pub fn current_change_count() -> Result<i64, String> {
unsafe {
let _pool = AutoreleasePool::new();
let pb = general_pasteboard()?;
let c: i64 = msg_send![pb, changeCount];
Ok(c)
}
}
/// Capture every item on the general pasteboard into an owned snapshot.
#[cfg(target_os = "macos")]
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
unsafe {
let _pool = AutoreleasePool::new();
let pb = general_pasteboard()?;
let change_count: i64 = msg_send![pb, changeCount];
let items: Id = msg_send![pb, pasteboardItems];
if items.is_null() {
return Ok(ClipboardSnapshot {
items: Vec::new(),
change_count,
});
}
let count: usize = msg_send![items, count];
let mut saved: Vec<Vec<(String, Vec<u8>)>> = Vec::with_capacity(count);
for i in 0..count {
let item: Id = msg_send![items, objectAtIndex: i];
if item.is_null() {
continue;
}
let types: Id = msg_send![item, types];
if types.is_null() {
continue;
}
let type_count: usize = msg_send![types, count];
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(type_count);
for j in 0..type_count {
let t: Id = msg_send![types, objectAtIndex: j];
let Some(type_str) = ns_string_to_rust(t) else {
continue;
};
let data: Id = msg_send![item, dataForType: t];
if data.is_null() {
// Type advertised but no concrete data (lazy provider).
// Skipping is safer than trying to force it to materialise.
continue;
}
let length: usize = msg_send![data, length];
let bytes_ptr: *const u8 = msg_send![data, bytes];
let bytes = if bytes_ptr.is_null() || length == 0 {
Vec::new()
} else {
std::slice::from_raw_parts(bytes_ptr, length).to_vec()
};
pairs.push((type_str, bytes));
}
saved.push(pairs);
}
Ok(ClipboardSnapshot {
items: saved,
change_count,
})
}
}
/// Replace the pasteboard contents with a single plain-text string. Returns
/// the post-write change count so a later restore can verify nothing else
/// touched the clipboard in between.
#[cfg(target_os = "macos")]
pub fn write_text(text: &str) -> Result<i64, String> {
unsafe {
let _pool = AutoreleasePool::new();
let pb = general_pasteboard()?;
let _new_count: i64 = msg_send![pb, clearContents];
let ns_text = ns_string(text);
// `public.utf8-plain-text` is the raw UTI behind `NSPasteboardTypeString`
// and works for every text-aware paste target we care about.
let ns_type = ns_string("public.utf8-plain-text");
let ok: bool = msg_send![pb, setString: ns_text forType: ns_type];
if !ok {
return Err("NSPasteboard setString:forType: returned NO".into());
}
let after: i64 = msg_send![pb, changeCount];
Ok(after)
}
}
/// Rebuild the pasteboard from a snapshot, replacing whatever is on it now.
///
/// Does not consult the change count — callers that want safe restore should
/// compare [`current_change_count`] against the value returned by
/// [`write_text`] first.
#[cfg(target_os = "macos")]
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
unsafe {
let _pool = AutoreleasePool::new();
let pb = general_pasteboard()?;
let _: i64 = msg_send![pb, clearContents];
if snapshot.items.is_empty() {
return Ok(());
}
let array: Id = msg_send![class!(NSMutableArray), array];
for pairs in &snapshot.items {
let item: Id = msg_send![class!(NSPasteboardItem), alloc];
let item: Id = msg_send![item, init];
let _: () = msg_send![item, autorelease];
for (uti, bytes) in pairs {
let ns_type = ns_string(uti);
let data: Id = msg_send![
class!(NSData),
dataWithBytes: bytes.as_ptr()
length: bytes.len()
];
let _ok: bool = msg_send![item, setData: data forType: ns_type];
}
let _: () = msg_send![array, addObject: item];
}
let ok: bool = msg_send![pb, writeObjects: array];
if !ok {
return Err("NSPasteboard writeObjects: returned NO".into());
}
Ok(())
}
}
#[cfg(target_os = "windows")]
mod win {
//! Windows clipboard implementation.
//!
//! The snapshot is structured so it mirrors the macOS `Vec<Vec<_>>`
//! shape: a single outer "item" holding one `(format-name, bytes)`
//! pair per enumerated format. Windows has no notion of multiple
//! pasteboard items, so there's always exactly one or zero outer
//! entries — enough to keep `item_count()` meaningful without
//! fan-out.
//!
//! Format IDs are serialised as strings so the snapshot type can stay
//! platform-neutral. Predefined formats use their canonical
//! identifier (`"CF_UNICODETEXT"`, `"CF_HDROP"`, `"CF_DIB"`, …);
//! registered formats use their string name from
//! `GetClipboardFormatNameW` (`"HTML Format"`, `"Rich Text
//! Format"`, …). Restore reverses the mapping with a lookup table
//! for the predefined IDs and `RegisterClipboardFormatW` for the
//! rest.
//!
//! Skipped format classes:
//! - CF_BITMAP (2), CF_METAFILEPICT (3), CF_PALETTE (9),
//! CF_ENHMETAFILE (14) — HGLOBAL's actually an HBITMAP /
//! HENHMETAFILE, not raw memory. Rebuilding them across processes
//! is possible but not worth it for clipboard stashing.
//! - CF_OWNERDISPLAY (0x80) and the CF_DSPxxx variants (0x810x8E) —
//! the owner draws these on demand. No data to snapshot.
//! - CF_PRIVATEFIRST..CF_PRIVATELAST (0x2000x2FF) — app-private,
//! meaningless to restore from a different process.
//! - CF_GDIOBJFIRST..CF_GDIOBJLAST (0x3000x3FF) — GDI handles.
//!
//! Text formats that Windows auto-synthesises (CF_TEXT, CF_OEMTEXT,
//! CF_LOCALE) are also skipped during save: `SetClipboardData` on
//! CF_UNICODETEXT regenerates them lazily on restore.
use std::thread;
use std::time::Duration;
use windows::core::PCWSTR;
use windows::Win32::Foundation::{GlobalFree, HANDLE, HGLOBAL, HWND};
use windows::Win32::System::DataExchange::{
CloseClipboard, EmptyClipboard, EnumClipboardFormats, GetClipboardData,
GetClipboardFormatNameW, GetClipboardSequenceNumber, OpenClipboard,
RegisterClipboardFormatW, SetClipboardData,
};
use windows::Win32::System::Memory::{
GlobalAlloc, GlobalLock, GlobalSize, GlobalUnlock, GLOBAL_ALLOC_FLAGS,
};
// `windows` 0.62 doesn't re-export every predefined clipboard format
// under a stable feature flag, so the values are pinned inline.
// These numbers are ABI-stable back to Windows 3.1 — verified against
// winuser.h.
pub const CF_TEXT: u32 = 1;
pub const CF_BITMAP: u32 = 2;
pub const CF_METAFILEPICT: u32 = 3;
pub const CF_SYLK: u32 = 4;
pub const CF_DIF: u32 = 5;
pub const CF_TIFF: u32 = 6;
pub const CF_OEMTEXT: u32 = 7;
pub const CF_DIB: u32 = 8;
pub const CF_PALETTE: u32 = 9;
pub const CF_PENDATA: u32 = 10;
pub const CF_RIFF: u32 = 11;
pub const CF_WAVE: u32 = 12;
pub const CF_UNICODETEXT: u32 = 13;
pub const CF_ENHMETAFILE: u32 = 14;
pub const CF_HDROP: u32 = 15;
pub const CF_LOCALE: u32 = 16;
pub const CF_DIBV5: u32 = 17;
pub const CF_OWNERDISPLAY: u32 = 0x0080;
pub const CF_DSPTEXT: u32 = 0x0081;
pub const CF_DSPBITMAP: u32 = 0x0082;
pub const CF_DSPMETAFILEPICT: u32 = 0x0083;
pub const CF_DSPENHMETAFILE: u32 = 0x008E;
pub const CF_PRIVATEFIRST: u32 = 0x0200;
pub const CF_PRIVATELAST: u32 = 0x02FF;
pub const CF_GDIOBJFIRST: u32 = 0x0300;
pub const CF_GDIOBJLAST: u32 = 0x03FF;
/// `GlobalAlloc` movable-memory flag — `GMEM_MOVEABLE` (0x0002).
/// Required for HGLOBAL handles destined for `SetClipboardData`; fixed
/// allocations are rejected.
const GMEM_MOVEABLE: GLOBAL_ALLOC_FLAGS = GLOBAL_ALLOC_FLAGS(0x0002);
/// Map a predefined clipboard format ID to its canonical identifier
/// string. Registered formats (IDs >= 0xC000) aren't handled here —
/// the caller resolves those via `GetClipboardFormatNameW`.
pub fn predefined_name(id: u32) -> Option<&'static str> {
Some(match id {
CF_TEXT => "CF_TEXT",
CF_BITMAP => "CF_BITMAP",
CF_METAFILEPICT => "CF_METAFILEPICT",
CF_SYLK => "CF_SYLK",
CF_DIF => "CF_DIF",
CF_TIFF => "CF_TIFF",
CF_OEMTEXT => "CF_OEMTEXT",
CF_DIB => "CF_DIB",
CF_PALETTE => "CF_PALETTE",
CF_PENDATA => "CF_PENDATA",
CF_RIFF => "CF_RIFF",
CF_WAVE => "CF_WAVE",
CF_UNICODETEXT => "CF_UNICODETEXT",
CF_ENHMETAFILE => "CF_ENHMETAFILE",
CF_HDROP => "CF_HDROP",
CF_LOCALE => "CF_LOCALE",
CF_DIBV5 => "CF_DIBV5",
CF_OWNERDISPLAY => "CF_OWNERDISPLAY",
CF_DSPTEXT => "CF_DSPTEXT",
CF_DSPBITMAP => "CF_DSPBITMAP",
CF_DSPMETAFILEPICT => "CF_DSPMETAFILEPICT",
CF_DSPENHMETAFILE => "CF_DSPENHMETAFILE",
_ => return None,
})
}
/// Reverse of [`predefined_name`].
pub fn predefined_id(name: &str) -> Option<u32> {
Some(match name {
"CF_TEXT" => CF_TEXT,
"CF_BITMAP" => CF_BITMAP,
"CF_METAFILEPICT" => CF_METAFILEPICT,
"CF_SYLK" => CF_SYLK,
"CF_DIF" => CF_DIF,
"CF_TIFF" => CF_TIFF,
"CF_OEMTEXT" => CF_OEMTEXT,
"CF_DIB" => CF_DIB,
"CF_PALETTE" => CF_PALETTE,
"CF_PENDATA" => CF_PENDATA,
"CF_RIFF" => CF_RIFF,
"CF_WAVE" => CF_WAVE,
"CF_UNICODETEXT" => CF_UNICODETEXT,
"CF_ENHMETAFILE" => CF_ENHMETAFILE,
"CF_HDROP" => CF_HDROP,
"CF_LOCALE" => CF_LOCALE,
"CF_DIBV5" => CF_DIBV5,
"CF_OWNERDISPLAY" => CF_OWNERDISPLAY,
"CF_DSPTEXT" => CF_DSPTEXT,
"CF_DSPBITMAP" => CF_DSPBITMAP,
"CF_DSPMETAFILEPICT" => CF_DSPMETAFILEPICT,
"CF_DSPENHMETAFILE" => CF_DSPENHMETAFILE,
_ => return None,
})
}
/// Returns true for predefined formats whose payload is a GDI handle
/// or owner-display sentinel rather than plain memory — callers must
/// skip these during snapshot because GlobalSize/GlobalLock wouldn't
/// return usable bytes.
pub fn is_skipped_format(id: u32) -> bool {
matches!(
id,
CF_BITMAP
| CF_METAFILEPICT
| CF_PALETTE
| CF_ENHMETAFILE
| CF_OWNERDISPLAY
| CF_DSPTEXT
| CF_DSPBITMAP
| CF_DSPMETAFILEPICT
| CF_DSPENHMETAFILE
) || (CF_PRIVATEFIRST..=CF_PRIVATELAST).contains(&id)
|| (CF_GDIOBJFIRST..=CF_GDIOBJLAST).contains(&id)
}
/// Auto-synthesised formats that Windows regenerates from
/// CF_UNICODETEXT on demand. Safe to skip during save; restore
/// lets `SetClipboardData(CF_UNICODETEXT)` re-derive them.
pub fn is_auto_synthesised(id: u32) -> bool {
matches!(id, CF_TEXT | CF_OEMTEXT | CF_LOCALE)
}
/// RAII wrapper around `OpenClipboard` / `CloseClipboard`.
///
/// The clipboard is a global exclusive resource — only one process at
/// a time holds the handle. `OpenClipboard` fails with
/// ERROR_ACCESS_DENIED when another process is mid-paste; the retry
/// loop here absorbs the common transient case without bubbling a
/// user-visible error.
pub struct ClipboardGuard;
impl ClipboardGuard {
pub fn open() -> Result<Self, String> {
const MAX_ATTEMPTS: usize = 10;
const RETRY_DELAY: Duration = Duration::from_millis(10);
let mut last_err: Option<windows::core::Error> = None;
for _ in 0..MAX_ATTEMPTS {
let result = unsafe { OpenClipboard(Some(HWND(std::ptr::null_mut()))) };
match result {
Ok(()) => return Ok(Self),
Err(e) => {
last_err = Some(e);
thread::sleep(RETRY_DELAY);
}
}
}
Err(format!(
"OpenClipboard failed after {} retries ({:?}). Another process likely holds the clipboard open.",
MAX_ATTEMPTS, last_err
))
}
}
impl Drop for ClipboardGuard {
fn drop(&mut self) {
unsafe {
let _ = CloseClipboard();
}
}
}
/// Read the full payload for `format` from the currently open
/// clipboard into an owned `Vec<u8>`. Returns `Ok(None)` when the
/// clipboard advertises the format but provides no concrete data
/// (delay-rendered format that's never been realised).
pub fn read_format_bytes(format: u32) -> Result<Option<Vec<u8>>, String> {
unsafe {
let handle = GetClipboardData(format)
.map_err(|e| format!("GetClipboardData({format}) failed: {e}"))?;
if handle.is_invalid() {
return Ok(None);
}
let hglobal = HGLOBAL(handle.0);
let size = GlobalSize(hglobal);
if size == 0 {
return Ok(Some(Vec::new()));
}
let ptr = GlobalLock(hglobal);
if ptr.is_null() {
return Err(format!(
"GlobalLock returned null for format {format} (size {size})"
));
}
let bytes = std::slice::from_raw_parts(ptr as *const u8, size).to_vec();
let _ = GlobalUnlock(hglobal);
Ok(Some(bytes))
}
}
/// Look up the name for a registered format ID (>= 0xC000). Returns
/// `None` for unnamed predefined IDs — the caller should have used
/// [`predefined_name`] first.
pub fn registered_name(id: u32) -> Option<String> {
let mut buf = [0u16; 256];
let len = unsafe { GetClipboardFormatNameW(id, &mut buf) };
if len <= 0 {
return None;
}
String::from_utf16(&buf[..len as usize]).ok()
}
/// Allocate a movable HGLOBAL, copy `bytes` in, return the handle
/// ready for `SetClipboardData`. On success ownership transfers to
/// the clipboard; on failure the caller must `GlobalFree`.
pub fn allocate_global(bytes: &[u8]) -> Result<HGLOBAL, String> {
if bytes.is_empty() {
// `GlobalAlloc(_, 0)` returns NULL, which `SetClipboardData`
// would then reject as an invalid handle. Pad to one byte so
// the format still round-trips (the receiving app already
// has to handle zero-content payloads via GlobalSize).
return allocate_global(&[0u8]);
}
unsafe {
let hglobal = GlobalAlloc(GMEM_MOVEABLE, bytes.len())
.map_err(|e| format!("GlobalAlloc({}) failed: {e}", bytes.len()))?;
let ptr = GlobalLock(hglobal);
if ptr.is_null() {
let _ = GlobalFree(Some(hglobal));
return Err("GlobalLock returned null after GlobalAlloc".into());
}
std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr as *mut u8, bytes.len());
let _ = GlobalUnlock(hglobal);
Ok(hglobal)
}
}
/// Push one format's payload onto the currently open clipboard.
/// On `SetClipboardData` success the HGLOBAL becomes the clipboard's
/// responsibility — do not free. On failure, free it ourselves.
pub fn put_format(format: u32, bytes: &[u8]) -> Result<(), String> {
let hglobal = allocate_global(bytes)?;
let handle = HANDLE(hglobal.0);
unsafe {
match SetClipboardData(format, Some(handle)) {
Ok(_) => Ok(()),
Err(e) => {
let _ = GlobalFree(Some(hglobal));
Err(format!("SetClipboardData({format}) failed: {e}"))
}
}
}
}
/// UTF-16 encode `s` with a trailing null code unit and push it as
/// CF_UNICODETEXT.
pub fn put_unicode_text(s: &str) -> Result<(), String> {
let mut utf16: Vec<u16> = s.encode_utf16().collect();
utf16.push(0);
let bytes: &[u8] = unsafe {
std::slice::from_raw_parts(
utf16.as_ptr() as *const u8,
utf16.len() * std::mem::size_of::<u16>(),
)
};
put_format(CF_UNICODETEXT, bytes)
}
/// Walk every format currently on the clipboard. `EnumClipboardFormats(0)`
/// returns the first; each subsequent call with the previous format
/// returns the next, until it returns 0 (or an error).
pub fn enumerate_formats() -> Vec<u32> {
let mut out = Vec::new();
let mut current = 0u32;
loop {
let next = unsafe { EnumClipboardFormats(current) };
if next == 0 {
break;
}
out.push(next);
current = next;
}
out
}
/// Resolve a snapshot's format name back to the u32 format ID.
/// Registered names (anything not predefined) go through
/// `RegisterClipboardFormatW`, which is idempotent — the same name
/// yields the same ID within a Windows session.
pub fn resolve_format_id(name: &str) -> Result<u32, String> {
if let Some(id) = predefined_id(name) {
return Ok(id);
}
let wide: Vec<u16> = name.encode_utf16().chain(std::iter::once(0)).collect();
let id = unsafe { RegisterClipboardFormatW(PCWSTR(wide.as_ptr())) };
if id == 0 {
return Err(format!("RegisterClipboardFormatW failed for {name:?}"));
}
Ok(id)
}
pub fn sequence_number() -> u32 {
unsafe { GetClipboardSequenceNumber() }
}
pub fn empty() -> Result<(), String> {
unsafe { EmptyClipboard().map_err(|e| format!("EmptyClipboard failed: {e}")) }
}
}
#[cfg(target_os = "windows")]
pub fn current_change_count() -> Result<i64, String> {
Ok(win::sequence_number() as i64)
}
#[cfg(target_os = "windows")]
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
let change_count = win::sequence_number() as i64;
let _guard = win::ClipboardGuard::open()?;
let formats = win::enumerate_formats();
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(formats.len());
for id in formats {
if win::is_skipped_format(id) || win::is_auto_synthesised(id) {
continue;
}
let name = match win::predefined_name(id) {
Some(n) => n.to_string(),
None => match win::registered_name(id) {
Some(n) => n,
None => continue,
},
};
match win::read_format_bytes(id) {
Ok(Some(bytes)) => pairs.push((name, bytes)),
Ok(None) => {}
Err(_) => {
// Single-format read failure (delay-render that never
// materialises, ACL-restricted format, etc.) shouldn't
// abort the whole snapshot — drop this format and keep
// going so the user's other clipboard contents still
// survive the round-trip.
continue;
}
}
}
let items = if pairs.is_empty() {
Vec::new()
} else {
vec![pairs]
};
Ok(ClipboardSnapshot {
items,
change_count,
})
}
#[cfg(target_os = "windows")]
pub fn write_text(text: &str) -> Result<i64, String> {
let _guard = win::ClipboardGuard::open()?;
win::empty()?;
win::put_unicode_text(text)?;
// `GetClipboardSequenceNumber` reflects the post-write value as soon
// as `SetClipboardData` returns.
Ok(win::sequence_number() as i64)
}
#[cfg(target_os = "windows")]
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
let _guard = win::ClipboardGuard::open()?;
win::empty()?;
for pairs in &snapshot.items {
for (name, bytes) in pairs {
let id = match win::resolve_format_id(name) {
Ok(id) => id,
Err(_) => continue,
};
// Per-format failures here also don't abort the whole
// restore — better to get the user's text content back even
// if a weird custom format can't be rehydrated.
let _ = win::put_format(id, bytes);
}
}
Ok(())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn current_change_count() -> Result<i64, String> {
Err("clipboard snapshot is not yet implemented on this platform".into())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
Err("clipboard snapshot is not yet implemented on this platform".into())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn write_text(_text: &str) -> Result<i64, String> {
Err("clipboard snapshot is not yet implemented on this platform".into())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn restore_clipboard(_snapshot: &ClipboardSnapshot) -> Result<(), String> {
Err("clipboard snapshot is not yet implemented on this platform".into())
}
+535
View File
@@ -0,0 +1,535 @@
//! Captures the focused-UI snapshot at chord-start so auto-paste can land
//! in the user's original text field even after focus drifts during
//! transcription / refinement.
//!
//! We don't try to re-focus a specific sub-element on restore — many apps
//! expose complex focus hierarchies that don't respond consistently to
//! programmatic focus pokes. Bringing the owning *window* to the
//! foreground is enough: the window's own focus manager restores its
//! last-focused field, which is what every well-behaved paste-buffer tool
//! does and what users expect.
//!
//! - **macOS** — `AXUIElementCopyAttributeValue(kAXFocusedUIElement)` +
//! `AXUIElementGetPid` + NSRunningApplication activation. Activation
//! uses the cooperative-activation pattern on macOS 14+ (the caller
//! `yieldActivationToApplication:`s, then the target `activate`s) and
//! falls back to the pre-Sonoma `activateWithOptions:` on 1113. See
//! `activate_pid` for the rationale.
//! - **Windows** — `GetForegroundWindow` + `GetWindowThreadProcessId` for
//! the top-level HWND and PID; UIAutomation's `IUIAutomation::GetFocusedElement`
//! for best-effort control-class (skipped silently if COM isn't usable).
//! Activation walks top-level windows for the saved PID and calls
//! `SetForegroundWindow`, bracketed by the `AttachThreadInput` dance
//! so Windows' foreground-lock rules don't silently swallow the
//! activation into a taskbar flash.
//!
//! PID + bundle id + role are all captured for diagnostics — the bundle
//! id lets step 6 (internal direct injection) detect "focus was inside
//! Voicebox itself" and short-circuit the synthetic-paste path. On
//! Windows, `bundle_id` holds the lowercased exe basename (`"voicebox.exe"`)
//! since there's no equivalent of macOS' reverse-DNS bundle identifier.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct FocusSnapshot {
pub pid: i32,
pub bundle_id: Option<String>,
pub role: Option<String>,
}
#[cfg(target_os = "macos")]
use core_foundation_sys::base::{kCFAllocatorDefault, CFRelease};
#[cfg(target_os = "macos")]
use core_foundation_sys::string::{
kCFStringEncodingUTF8, CFStringCreateWithCString, CFStringGetCString, CFStringGetLength,
CFStringRef,
};
#[cfg(target_os = "macos")]
use objc::runtime::Object;
#[cfg(target_os = "macos")]
use objc::{class, msg_send, sel, sel_impl};
#[cfg(target_os = "macos")]
type Id = *mut Object;
#[cfg(target_os = "macos")]
mod ffi {
use core_foundation_sys::base::CFTypeRef;
use core_foundation_sys::string::CFStringRef;
pub type AXError = i32;
pub const AX_ERROR_SUCCESS: AXError = 0;
pub type AXUIElementRef = *const std::ffi::c_void;
pub type Pid = i32;
#[link(name = "ApplicationServices", kind = "framework")]
extern "C" {
pub fn AXUIElementCreateSystemWide() -> AXUIElementRef;
pub fn AXUIElementCopyAttributeValue(
element: AXUIElementRef,
attribute: CFStringRef,
value: *mut CFTypeRef,
) -> AXError;
pub fn AXUIElementGetPid(element: AXUIElementRef, pid: *mut Pid) -> AXError;
}
// AX attribute keys are exposed as C macros that expand to CFSTR(...)
// literals, not as linkable symbols — build the CFStrings at runtime
// instead (see `cf_string_const` in focus_capture.rs).
}
#[cfg(target_os = "macos")]
struct AutoreleasePool {
pool: Id,
}
#[cfg(target_os = "macos")]
impl AutoreleasePool {
unsafe fn new() -> Self {
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
let pool: Id = msg_send![pool, init];
Self { pool }
}
}
#[cfg(target_os = "macos")]
impl Drop for AutoreleasePool {
fn drop(&mut self) {
unsafe {
let _: () = msg_send![self.pool, drain];
}
}
}
#[cfg(target_os = "macos")]
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
if s.is_null() {
return None;
}
let bytes: *const i8 = msg_send![s, UTF8String];
if bytes.is_null() {
return None;
}
std::ffi::CStr::from_ptr(bytes)
.to_str()
.ok()
.map(|x| x.to_owned())
}
/// Build a `+1` retained CFString from an ASCII constant. Caller owns the
/// returned reference and must `CFRelease` it. Used for AX attribute keys
/// (`"AXFocusedUIElement"`, `"AXRole"`) because those aren't exported as
/// linker symbols — Apple ships them as `CFSTR(...)` macros.
#[cfg(target_os = "macos")]
unsafe fn cf_string_const(s: &str) -> Option<CFStringRef> {
let cstr = std::ffi::CString::new(s).ok()?;
let result = CFStringCreateWithCString(kCFAllocatorDefault, cstr.as_ptr(), kCFStringEncodingUTF8);
if result.is_null() {
None
} else {
Some(result)
}
}
#[cfg(target_os = "macos")]
unsafe fn cfstring_to_rust(s: CFStringRef) -> Option<String> {
if s.is_null() {
return None;
}
let len = CFStringGetLength(s);
if len == 0 {
return Some(String::new());
}
// CFStringGetLength is in UTF-16 code units; UTF-8 can need up to 4
// bytes per unit plus the trailing NUL.
let max_bytes = (len * 4 + 1) as usize;
let mut buf = vec![0u8; max_bytes];
let ok = CFStringGetCString(
s,
buf.as_mut_ptr() as *mut i8,
max_bytes as isize,
kCFStringEncodingUTF8,
);
if ok == 0 {
return None;
}
let cstr = std::ffi::CStr::from_ptr(buf.as_ptr() as *const i8);
cstr.to_str().ok().map(|x| x.to_owned())
}
#[cfg(target_os = "macos")]
unsafe fn bundle_id_for_pid(pid: i32) -> Option<String> {
let _pool = AutoreleasePool::new();
let app: Id = msg_send![
class!(NSRunningApplication),
runningApplicationWithProcessIdentifier: pid
];
if app.is_null() {
return None;
}
let bundle: Id = msg_send![app, bundleIdentifier];
ns_string_to_rust(bundle)
}
/// Read the system-wide focused UI element's PID, bundle id, and AX role.
///
/// Returns an error when no element is focused (e.g. Dock has focus) or
/// when Accessibility permission is missing — `AXUIElementCopyAttributeValue`
/// returns `-25204 kAXErrorAPIDisabled` in that case.
#[cfg(target_os = "macos")]
pub fn capture_focus() -> Result<FocusSnapshot, String> {
use ffi::*;
unsafe {
let system_wide = AXUIElementCreateSystemWide();
if system_wide.is_null() {
return Err("AXUIElementCreateSystemWide returned null".into());
}
let _sys_guard = scopeguard::guard(system_wide, |e| {
CFRelease(e as *const std::ffi::c_void)
});
let focused_attr = cf_string_const("AXFocusedUIElement")
.ok_or("Failed to build AXFocusedUIElement CFString")?;
let _focused_attr_guard =
scopeguard::guard(focused_attr, |s| CFRelease(s as *const std::ffi::c_void));
let mut focused: *const std::ffi::c_void = std::ptr::null();
let err = AXUIElementCopyAttributeValue(
system_wide,
focused_attr,
&mut focused as *mut _,
);
if err != AX_ERROR_SUCCESS || focused.is_null() {
return Err(format!(
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
err
));
}
let _focus_guard = scopeguard::guard(focused, |e| CFRelease(e));
let focused_elem = focused as AXUIElementRef;
let mut pid: Pid = 0;
let err = AXUIElementGetPid(focused_elem, &mut pid);
if err != AX_ERROR_SUCCESS {
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
}
let role = {
let role_attr = cf_string_const("AXRole");
match role_attr {
Some(role_attr) => {
let _role_attr_guard = scopeguard::guard(role_attr, |s| {
CFRelease(s as *const std::ffi::c_void)
});
let mut role_value: *const std::ffi::c_void = std::ptr::null();
let err = AXUIElementCopyAttributeValue(
focused_elem,
role_attr,
&mut role_value as *mut _,
);
if err == AX_ERROR_SUCCESS && !role_value.is_null() {
let _role_guard = scopeguard::guard(role_value, |e| CFRelease(e));
cfstring_to_rust(role_value as CFStringRef)
} else {
None
}
}
None => None,
}
};
let bundle_id = bundle_id_for_pid(pid);
Ok(FocusSnapshot {
pid,
bundle_id,
role,
})
}
}
/// Bring the app owning `pid` to the foreground, re-activating its
/// last-focused window. Paired with [`capture_focus`] at chord-start so a
/// post-transcription synthetic ⌘V lands where the user started, not
/// wherever focus drifted to during the transcribe / refine window.
///
/// macOS 14 (Sonoma) deprecated `activateWithOptions:` in favour of a
/// cooperative-activation pattern: the caller first invokes
/// `yieldActivationToApplication:` on its own `NSRunningApplication` to
/// grant the target activation rights, then the target's `activate`
/// succeeds against the tightened Sonoma foreground rules. Without the
/// yield, `activate` on 14+ sometimes silently fails or only bounces the
/// dock icon — exactly the "paste lands in the wrong app" symptom we're
/// trying to prevent. The yield is discovered at runtime via
/// `respondsToSelector:` so we don't need an operatingSystemVersion probe
/// and the pre-Sonoma path stays identical.
///
/// The BOOL return of both `activate` and `activateWithOptions:` is now
/// propagated — if the system refuses activation (target quit mid-
/// transcription, trust revoked, cooperative-activation refused) the
/// caller aborts before clobbering the clipboard.
#[cfg(target_os = "macos")]
pub fn activate_pid(pid: i32) -> Result<(), String> {
unsafe {
let _pool = AutoreleasePool::new();
let target: Id = msg_send![
class!(NSRunningApplication),
runningApplicationWithProcessIdentifier: pid
];
if target.is_null() {
return Err(format!("No running application for PID {}", pid));
}
let activated: bool = if can_yield_activation() {
let current: Id =
msg_send![class!(NSRunningApplication), currentApplication];
if !current.is_null() {
let _: () = msg_send![current, yieldActivationToApplication: target];
}
msg_send![target, activate]
} else {
// NSApplicationActivateIgnoringOtherApps = 1 << 1 = 2.
msg_send![target, activateWithOptions: 2u64]
};
if !activated {
return Err(format!(
"NSRunningApplication activate returned false for PID {} — the target may have quit mid-transcription, Accessibility is no longer trusted, or the system refused cooperative activation.",
pid
));
}
Ok(())
}
}
/// `true` when `NSRunningApplication` responds to
/// `yieldActivationToApplication:` — the macOS 14+ discriminator for the
/// cooperative-activation APIs. Cached since the answer doesn't change
/// over a process's lifetime and the objc_msgSend probe is otherwise
/// repeated on every paste.
#[cfg(target_os = "macos")]
fn can_yield_activation() -> bool {
use std::sync::OnceLock;
static CACHED: OnceLock<bool> = OnceLock::new();
*CACHED.get_or_init(|| unsafe {
let current: Id = msg_send![class!(NSRunningApplication), currentApplication];
if current.is_null() {
return false;
}
let responds: bool = msg_send![
current,
respondsToSelector: sel!(yieldActivationToApplication:)
];
responds
})
}
#[cfg(target_os = "windows")]
mod win {
use std::path::Path;
use windows::core::{IUnknown, BOOL, BSTR, PWSTR};
use windows::Win32::Foundation::{CloseHandle, HWND, LPARAM};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
};
use windows::Win32::System::Threading::{
AttachThreadInput, GetCurrentThreadId, OpenProcess, QueryFullProcessImageNameW,
PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION,
};
use windows::Win32::UI::Accessibility::{CUIAutomation, IUIAutomation, IUIAutomationElement};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetForegroundWindow, GetWindow, GetWindowThreadProcessId, IsWindowVisible,
SetForegroundWindow, GW_OWNER,
};
/// Read the PID that owns `hwnd`. Returns 0 on failure.
pub unsafe fn hwnd_pid(hwnd: HWND) -> u32 {
let mut pid: u32 = 0;
let _ = GetWindowThreadProcessId(hwnd, Some(&mut pid as *mut _));
pid
}
/// Query a PID's executable path and return its lowercased basename
/// (e.g. `"voicebox.exe"`). This is the Windows analogue of macOS'
/// `bundleIdentifier`, just less globally unique — two apps with the
/// same exe name can collide, but that's rare enough to accept for
/// the self-paste short-circuit.
pub fn exe_basename(pid: u32) -> Option<String> {
unsafe {
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
let mut buf = [0u16; 1024];
let mut size = buf.len() as u32;
let ok = QueryFullProcessImageNameW(
handle,
PROCESS_NAME_FORMAT(0),
PWSTR(buf.as_mut_ptr()),
&mut size,
);
let _ = CloseHandle(handle);
if ok.is_err() || size == 0 {
return None;
}
let full = String::from_utf16(&buf[..size as usize]).ok()?;
let basename = Path::new(&full)
.file_name()
.and_then(|s| s.to_str())
.map(|s| s.to_ascii_lowercase())?;
Some(basename)
}
}
/// Best-effort `UIAutomation::GetFocusedElement().CurrentClassName()`.
/// Returns `None` when COM init, CoCreateInstance, or any UIA call
/// fails — role info is nice-to-have, not load-bearing for paste.
pub fn focused_control_class() -> Option<String> {
unsafe {
// MTA per-thread init. Ignore HRESULT: S_OK / S_FALSE /
// RPC_E_CHANGED_MODE are all benign for our uses here, and
// we deliberately never call CoUninitialize (the Tauri
// runtime thread lives for the life of the process, so
// leaving COM init in place is fine).
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
let automation: IUIAutomation =
CoCreateInstance(&CUIAutomation, None::<&IUnknown>, CLSCTX_INPROC_SERVER).ok()?;
let element: IUIAutomationElement = automation.GetFocusedElement().ok()?;
// UIAutomationElement's CurrentClassName allocates a BSTR
// the caller has to drop. `BSTR` in `windows` crate is a
// Drop-wrapped owned string, so just returning `.to_string()`
// is safe.
let class: BSTR = element.CurrentClassName().ok()?;
let s = class.to_string();
if s.is_empty() {
None
} else {
Some(s)
}
}
}
/// Find a visible top-level window owned by `pid`. Returns the first
/// match via `EnumWindows`. Top-level ≡ no owner window.
pub fn find_top_level_window(pid: u32) -> Option<HWND> {
struct Ctx {
target_pid: u32,
found: Option<HWND>,
}
let mut ctx = Ctx {
target_pid: pid,
found: None,
};
unsafe extern "system" fn callback(hwnd: HWND, lparam: LPARAM) -> BOOL {
let ctx = &mut *(lparam.0 as *mut Ctx);
if hwnd_pid(hwnd) != ctx.target_pid {
return BOOL(1);
}
// Skip tool windows / invisible shells. `GetWindow(GW_OWNER)`
// is non-null for modal dialogs and other secondary windows;
// we want the real app frame, which has no owner.
if !IsWindowVisible(hwnd).as_bool() {
return BOOL(1);
}
if !GetWindow(hwnd, GW_OWNER).unwrap_or(HWND(std::ptr::null_mut())).is_invalid() {
return BOOL(1);
}
ctx.found = Some(hwnd);
BOOL(0)
}
unsafe {
let _ = EnumWindows(
Some(callback),
LPARAM(&mut ctx as *mut _ as isize),
);
}
ctx.found
}
/// Bring `hwnd` to the foreground reliably.
///
/// Plain `SetForegroundWindow` loses to Windows' foreground-lock
/// rules — when our process isn't already foreground it can't hand
/// focus to another app. The documented workaround is to attach the
/// current thread's input queue to the current foreground window's
/// thread for the duration of the call, which temporarily lets us
/// share that thread's "last user activity" stamp.
pub fn activate_hwnd(hwnd: HWND) -> Result<(), String> {
unsafe {
let fg = GetForegroundWindow();
if fg == hwnd {
return Ok(());
}
let our_thread = GetCurrentThreadId();
let fg_thread = if fg.is_invalid() {
0
} else {
let mut _pid: u32 = 0;
GetWindowThreadProcessId(fg, Some(&mut _pid as *mut _))
};
let attached = fg_thread != 0
&& fg_thread != our_thread
&& AttachThreadInput(our_thread, fg_thread, true).as_bool();
let ok = SetForegroundWindow(hwnd).as_bool();
if attached {
let _ = AttachThreadInput(our_thread, fg_thread, false);
}
if !ok {
return Err(format!(
"SetForegroundWindow failed for HWND {:?} — Windows foreground-lock may have denied the activation.",
hwnd.0
));
}
Ok(())
}
}
}
#[cfg(target_os = "windows")]
pub fn capture_focus() -> Result<FocusSnapshot, String> {
use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
unsafe {
let hwnd = GetForegroundWindow();
if hwnd.is_invalid() {
return Err(
"GetForegroundWindow returned null — the desktop has no focused window (secure attention sequence, lock screen, or no user session)."
.into(),
);
}
let pid = win::hwnd_pid(hwnd);
if pid == 0 {
return Err("GetWindowThreadProcessId returned PID 0 for the foreground window".into());
}
let bundle_id = win::exe_basename(pid);
let role = win::focused_control_class();
Ok(FocusSnapshot {
pid: pid as i32,
bundle_id,
role,
})
}
}
#[cfg(target_os = "windows")]
pub fn activate_pid(pid: i32) -> Result<(), String> {
if pid <= 0 {
return Err(format!("Cannot activate invalid PID {pid}"));
}
let hwnd = win::find_top_level_window(pid as u32)
.ok_or_else(|| format!("No visible top-level window for PID {pid}"))?;
win::activate_hwnd(hwnd)
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn capture_focus() -> Result<FocusSnapshot, String> {
Err("focus capture is not yet implemented on this platform".into())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn activate_pid(_pid: i32) -> Result<(), String> {
Err("app activation is not yet implemented on this platform".into())
}
+290
View File
@@ -0,0 +1,290 @@
//! Global hotkey → dictation effect bridge.
//!
//! Thin adapter from `keytap::chord::ChordMatcher` to Tauri events. keytap
//! owns the OS event tap + the chord state machine (Momentary vs Toggle,
//! longest-match resolution, sticky-toggle semantics); this module's only
//! job is:
//!
//! 1. Build a `ChordMatcher` from the user's saved PTT + Toggle chords.
//! 2. Translate `ChordEvent` → voicebox's [`Effect`] on a dispatcher
//! thread.
//! 3. Fan [`Effect`]s out into Tauri events + dictate-window show/hide.
//!
//! The [`Effect::RestartRecording`] signal is emitted when keytap fires
//! `End(PTT)` and `Start(Toggle)` with the *same* [`Instant`] — which
//! happens when the held set upgrades from a shorter chord to a longer
//! superset in a single event (the classic PTT→hands-free transition).
//! We detect the pair with a 5 ms peek on the matcher's receiver and
//! coalesce into one `Restart` so hosts can discard the transition-
//! moment audio rather than treat it as an unrelated Stop+Start pair.
//!
//! Left- and right-hand modifier variants are kept distinct all the way
//! down to the OS event tap (keytap's core promise). Defaults bind to
//! right-hand Cmd + right-hand Option on macOS / right-hand Ctrl +
//! right-hand Shift on Windows so the usual left-hand shortcuts stay
//! with the OS / app.
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::thread::{self, JoinHandle};
use std::time::Duration;
use keytap::chord::{Chord, ChordEvent, ChordMatcher};
use keytap::{Key, RecvTimeoutError};
use tauri::{AppHandle, Emitter, Manager};
use crate::focus_capture;
use crate::DICTATE_WINDOW_LABEL;
// ========================================================================
// Public types
// ========================================================================
/// Semantic action a chord can be bound to. `PushToTalk` = hold chord to
/// record, release to stop. `ToggleToTalk` = press chord to start recording,
/// press again to stop.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ChordAction {
PushToTalk,
ToggleToTalk,
}
/// Effect produced after the chord matcher resolves an event. Hosts
/// translate these into UI / recorder calls.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Effect {
StartRecording(ChordAction),
StopRecording(ChordAction),
/// Emitted when a push-to-talk chord is "upgraded" into the toggle
/// chord mid-hold — hosts may want to discard the captured audio and
/// restart so the transition moment isn't in the recording.
RestartRecording(ChordAction),
}
/// Chord key sets from capture settings. Both actions use the same
/// `HashSet<Key>` shape so callers don't need to know about keytap's
/// `Chord` type.
pub type Bindings = HashMap<ChordAction, HashSet<Key>>;
// ========================================================================
// Monitor
// ========================================================================
pub struct HotkeyMonitor {
app: AppHandle,
active: Option<Active>,
}
struct Active {
dispatcher: JoinHandle<()>,
shutdown: Arc<AtomicBool>,
}
impl HotkeyMonitor {
/// Build the monitor with initial bindings. Equivalent to constructing
/// an empty monitor and calling [`Self::update_bindings`] once.
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
let mut m = Self { app, active: None };
m.apply(bindings);
m
}
/// Swap in a fresh set of chord bindings. Tears down the existing
/// `ChordMatcher` (which stops keytap's chord worker thread and
/// closes the OS tap) and spawns a new one. No-op for the "all
/// empty" case so "disable hotkey" doesn't keep a tap running for
/// no reason.
pub fn update_bindings(&mut self, bindings: Bindings) {
self.apply(bindings);
}
fn apply(&mut self, bindings: Bindings) {
// Tear down any existing matcher + dispatcher first. The
// dispatcher sees the shutdown flag on its next recv_timeout
// (≤100ms) and returns; joining waits for that. Dropping the
// ChordMatcher stops keytap's chord-worker thread and the
// underlying Tap.
if let Some(active) = self.active.take() {
active.shutdown.store(true, Ordering::Relaxed);
let _ = active.dispatcher.join();
}
if bindings.values().all(|set| set.is_empty()) {
return;
}
let matcher = match build_matcher(&bindings) {
Ok(m) => m,
Err(err) => {
eprintln!(
"HotkeyMonitor: ChordMatcher build failed ({err}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch."
);
return;
}
};
let shutdown = Arc::new(AtomicBool::new(false));
let shutdown_for_thread = shutdown.clone();
let app = self.app.clone();
let dispatcher = thread::Builder::new()
.name("voicebox-hotkey-dispatcher".into())
.spawn(move || dispatcher_loop(app, matcher, shutdown_for_thread))
.expect("spawn hotkey dispatcher thread");
self.active = Some(Active { dispatcher, shutdown });
}
}
impl Drop for HotkeyMonitor {
fn drop(&mut self) {
if let Some(active) = self.active.take() {
active.shutdown.store(true, Ordering::Relaxed);
let _ = active.dispatcher.join();
}
}
}
// ========================================================================
// Matcher construction + dispatch
// ========================================================================
fn build_matcher(bindings: &Bindings) -> Result<ChordMatcher<ChordAction>, keytap::Error> {
let mut builder = ChordMatcher::builder();
if let Some(keys) = bindings.get(&ChordAction::PushToTalk) {
if !keys.is_empty() {
builder = builder.add(
ChordAction::PushToTalk,
Chord::of(keys.iter().copied()),
);
}
}
if let Some(keys) = bindings.get(&ChordAction::ToggleToTalk) {
if !keys.is_empty() {
builder = builder.add_toggle(
ChordAction::ToggleToTalk,
Chord::of(keys.iter().copied()),
);
}
}
builder.build()
}
fn dispatcher_loop(
app: AppHandle,
matcher: ChordMatcher<ChordAction>,
shutdown: Arc<AtomicBool>,
) {
while !shutdown.load(Ordering::Relaxed) {
match matcher.recv_timeout(Duration::from_millis(100)) {
Ok(event) => process_event(&app, &matcher, event),
Err(RecvTimeoutError::Timeout) => continue,
Err(RecvTimeoutError::Disconnected) => break,
}
}
}
/// Turn a single [`ChordEvent`] into zero or one [`Effect`]s, peeking at
/// the matcher once for a same-Instant follow-up so upgrade transitions
/// coalesce into [`Effect::RestartRecording`] instead of a Stop+Start
/// pair.
fn process_event(
app: &AppHandle,
matcher: &ChordMatcher<ChordAction>,
event: ChordEvent<ChordAction>,
) {
match event {
ChordEvent::Start { id, .. } => {
apply_effect(app, Effect::StartRecording(id));
}
ChordEvent::End { id: end_id, time: end_time } => {
// Peek for an immediately-following Start. keytap emits
// End+Start atomically (same Instant) when the held set
// transitions between registered chords — our 5 ms window
// is well under perceptible latency but far longer than the
// channel hop between keytap's chord worker and our
// dispatcher.
match matcher.recv_timeout(Duration::from_millis(5)) {
Ok(ChordEvent::Start { id: start_id, time: start_time })
if start_time == end_time =>
{
apply_effect(app, Effect::RestartRecording(start_id));
}
Ok(other) => {
apply_effect(app, Effect::StopRecording(end_id));
// The peeked event wasn't a transition partner;
// process it in its own right. Recursion depth is
// bounded by the number of back-to-back chord
// events, in practice 12.
process_event(app, matcher, other);
}
Err(_) => {
apply_effect(app, Effect::StopRecording(end_id));
}
}
}
}
}
// ========================================================================
// Effect → Tauri
// ========================================================================
fn apply_effect(app: &AppHandle, effect: Effect) {
match effect {
Effect::StartRecording(_) => {
// Snapshot focus BEFORE we touch the window — any AppKit
// reshuffle triggered by set_position / show could in principle
// steal key focus and poison the reading. In practice those
// calls leave keyWindow alone, but capturing first is free.
let focus = focus_capture::capture_focus().ok();
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
// The previous hide-cycle parked the window off-screen and
// made it click-through — undo both before showing, so the
// pill lands at top-center and the user can actually click
// the error pill / stop button.
//
// `current_monitor()` returns None when the window is off
// any display (our hide handler parks it at -10_000, -10_000
// precisely so it never intercepts clicks), so fall back to
// the primary monitor for the reposition.
let monitor = window
.current_monitor()
.ok()
.flatten()
.or_else(|| window.primary_monitor().ok().flatten());
if let Some(monitor) = monitor {
let monitor_pos = monitor.position();
let monitor_size = monitor.size();
if let Ok(win_size) = window.outer_size() {
let x = monitor_pos.x
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
}
}
// Skip on Linux: aborts if the window was never realized
// (see show_dictate_window in main.rs).
#[cfg(not(target_os = "linux"))]
let _ = window.set_ignore_cursor_events(false);
// Deliberately no set_focus() — taking key focus would yank
// it out of whatever app the user was typing in, which is
// the opposite of what a dictation overlay should do.
let _ = window.show();
let payload = serde_json::json!({ "focus": focus });
let _ = window.emit("dictate:start", payload);
}
}
Effect::StopRecording(_) => {
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
let _ = window.emit("dictate:stop", ());
}
}
Effect::RestartRecording(_) => {
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
let _ = window.emit("dictate:restart", ());
}
}
}
}
+82
View File
@@ -0,0 +1,82 @@
//! Platform permission gate for the global keyboard tap.
//!
//! On macOS 10.15+, creating a CGEventTap that observes keyboard events
//! requires the host process to be listed under System Settings → Privacy &
//! Security → Input Monitoring. Without that trust, keytap's `Tap` returns
//! a permission error and no key events ever flow through the chord engine.
//!
//! The relevant TCC pair lives in IOKit, mirroring `AXIsProcessTrusted` /
//! `AXIsProcessTrustedWithOptions` on the Accessibility side:
//!
//! - `IOHIDCheckAccess(kIOHIDRequestTypeListenEvent)` — read the current
//! grant without prompting. We call this from the Captures settings UI
//! so the row can show "granted" / "missing" without surprising the user.
//! - `IOHIDRequestAccess(kIOHIDRequestTypeListenEvent)` — fire the
//! "Voicebox would like to receive keystrokes from any application"
//! dialog and add Voicebox to the Input Monitoring pane (toggle off).
//! Returns true when access is already granted; otherwise returns false
//! and queues the prompt. The user still has to flip the toggle on; this
//! just gets us into the list.
//!
//! `enable_hotkey` calls `request` on first invocation so the prompt fires
//! from a deterministic, user-initiated point (the Captures toggle) instead
//! of as a side-effect of keytap's `Tap` creating its CGEventTap.
//!
//! Windows / Linux don't gate keyboard taps behind a TCC-style permission,
//! so those branches return `true`.
#[cfg(target_os = "macos")]
mod ffi {
use std::os::raw::c_uint;
/// `kIOHIDRequestTypeListenEvent` from `<IOKit/hidsystem/IOHIDLib.h>` —
/// the request-type discriminator for "I want to read keyboard / mouse
/// events created by other processes."
pub const REQUEST_TYPE_LISTEN_EVENT: c_uint = 1;
/// `kIOHIDAccessTypeGranted` from `IOHIDLib.h`. The other values are
/// `Denied = 1` and `Unknown = 2`; we only ever care about the granted
/// case so they don't get their own constants.
pub const ACCESS_TYPE_GRANTED: c_uint = 0;
#[link(name = "IOKit", kind = "framework")]
extern "C" {
/// Returns the current access state as an `IOHIDAccessType` enum
/// (Granted=0, Denied=1, Unknown=2). No prompt side-effect.
///
/// Declared as `c_uint` rather than `bool`: the C signature returns
/// the full enum, and reading a 3-valued enum into Rust's 1-bit
/// `bool` is undefined behaviour that silently inverts our gate.
pub fn IOHIDCheckAccess(request_type: c_uint) -> c_uint;
/// Returns true when access is already granted; otherwise queues
/// the system prompt and returns false synchronously. Safe to call
/// repeatedly — once the entry exists in the Input Monitoring pane
/// macOS won't re-prompt. Real `Boolean` (UInt8) return on the C
/// side, so `bool` here is correct.
pub fn IOHIDRequestAccess(request_type: c_uint) -> bool;
}
}
#[cfg(target_os = "macos")]
pub fn is_trusted() -> bool {
unsafe { ffi::IOHIDCheckAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) == ffi::ACCESS_TYPE_GRANTED }
}
/// Fire the Input Monitoring prompt if not already granted. Returns the
/// current grant state; a `false` here means the prompt was queued and the
/// user needs to flip the toggle in System Settings before key events flow.
#[cfg(target_os = "macos")]
pub fn request() -> bool {
unsafe { ffi::IOHIDRequestAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) }
}
#[cfg(not(target_os = "macos"))]
pub fn is_trusted() -> bool {
true
}
#[cfg(not(target_os = "macos"))]
pub fn request() -> bool {
true
}
+95
View File
@@ -0,0 +1,95 @@
//! Stable string ↔ `keytap::Key` mapping for chord persistence.
//!
//! The frontend captures keypresses through the browser keyboard API (which
//! exposes `event.code` like `"MetaRight"`, `"AltRight"`, `"Space"`, `"KeyA"`)
//! and stores chords in capture_settings as JSON arrays of canonical names.
//! On the way back the same names need to round-trip into `keytap::Key`
//! variants the chord engine actually matches against.
//!
//! Input strings follow the W3C `KeyboardEvent.code` identifiers exactly —
//! `"MetaRight"`, `"AltRight"`, `"KeyA"`, `"Digit0"`, `"ArrowUp"`, … —
//! which is also what the browser emits natively, so on-disk chords
//! round-trip without translation on the frontend side. Legacy aliases
//! (`"Alt"` / `"AltGr"` / `"Num0"` / `"UpArrow"` / …) are accepted too so
//! older capture_settings rows written before the keytap swap keep working.
use keytap::Key;
/// Resolve a canonical key name to its `keytap::Key`. Returns `None` for
/// names that don't have a corresponding variant — the command surface
/// rejects those so we never silently drop keys from a chord.
pub fn key_from_str(name: &str) -> Option<Key> {
Some(match name {
// Modifiers — left/right distinction matters for chord defaults.
"AltLeft" | "Alt" => Key::AltLeft,
"AltRight" | "AltGr" => Key::AltRight,
"ControlLeft" => Key::ControlLeft,
"ControlRight" => Key::ControlRight,
"MetaLeft" => Key::MetaLeft,
"MetaRight" => Key::MetaRight,
"ShiftLeft" => Key::ShiftLeft,
"ShiftRight" => Key::ShiftRight,
"CapsLock" => Key::CapsLock,
"Function" => Key::Function,
// Whitespace / navigation
"Space" => Key::Space,
"Tab" => Key::Tab,
"Enter" | "Return" => Key::Enter,
"Backspace" => Key::Backspace,
"Delete" => Key::Delete,
"Escape" => Key::Escape,
"Insert" => Key::Insert,
"Home" => Key::Home,
"End" => Key::End,
"PageUp" => Key::PageUp,
"PageDown" => Key::PageDown,
"ArrowUp" | "UpArrow" => Key::ArrowUp,
"ArrowDown" | "DownArrow" => Key::ArrowDown,
"ArrowLeft" | "LeftArrow" => Key::ArrowLeft,
"ArrowRight" | "RightArrow" => Key::ArrowRight,
// Function row
"F1" => Key::F1, "F2" => Key::F2, "F3" => Key::F3, "F4" => Key::F4,
"F5" => Key::F5, "F6" => Key::F6, "F7" => Key::F7, "F8" => Key::F8,
"F9" => Key::F9, "F10" => Key::F10, "F11" => Key::F11, "F12" => Key::F12,
// Digits
"Digit0" | "Num0" => Key::Digit0,
"Digit1" | "Num1" => Key::Digit1,
"Digit2" | "Num2" => Key::Digit2,
"Digit3" | "Num3" => Key::Digit3,
"Digit4" | "Num4" => Key::Digit4,
"Digit5" | "Num5" => Key::Digit5,
"Digit6" | "Num6" => Key::Digit6,
"Digit7" | "Num7" => Key::Digit7,
"Digit8" | "Num8" => Key::Digit8,
"Digit9" | "Num9" => Key::Digit9,
// Letters — browser emits "KeyA"; keytap uses the bare letter.
"KeyA" => Key::A, "KeyB" => Key::B, "KeyC" => Key::C,
"KeyD" => Key::D, "KeyE" => Key::E, "KeyF" => Key::F,
"KeyG" => Key::G, "KeyH" => Key::H, "KeyI" => Key::I,
"KeyJ" => Key::J, "KeyK" => Key::K, "KeyL" => Key::L,
"KeyM" => Key::M, "KeyN" => Key::N, "KeyO" => Key::O,
"KeyP" => Key::P, "KeyQ" => Key::Q, "KeyR" => Key::R,
"KeyS" => Key::S, "KeyT" => Key::T, "KeyU" => Key::U,
"KeyV" => Key::V, "KeyW" => Key::W, "KeyX" => Key::X,
"KeyY" => Key::Y, "KeyZ" => Key::Z,
// Punctuation / symbols
"Backquote" | "BackQuote" => Key::Backtick,
"Minus" => Key::Minus,
"Equal" => Key::Equal,
"BracketLeft" | "LeftBracket" => Key::BracketLeft,
"BracketRight" | "RightBracket" => Key::BracketRight,
"Semicolon" | "SemiColon" => Key::Semicolon,
"Quote" => Key::Quote,
"Backslash" | "BackSlash" => Key::Backslash,
"Comma" => Key::Comma,
"Period" | "Dot" => Key::Period,
"Slash" => Key::Slash,
_ => return None,
})
}
+187
View File
@@ -0,0 +1,187 @@
//! Layout-aware resolution of the keycode whose current-layout translation
//! is `'v'`. Drives [`crate::synthetic_keys::send_paste`] so the synthetic
//! Cmd+V it posts is interpreted as Paste by the focused app regardless of
//! the user's active keyboard layout (Dvorak, Colemak, AZERTY, …).
//!
//! macOS apps process Cmd+V via NSMenu key equivalents, which match against
//! `[NSEvent charactersIgnoringModifiers]` — i.e. the layout-translated
//! character, not the raw keycode. Posting `kVK_ANSI_V` (= 9, the QWERTY V
//! position) on Dvorak therefore produces Cmd+. and never triggers Paste.
//!
//! All TIS calls happen on the main thread: once at startup via [`init`]
//! from Tauri's setup hook, and again from the
//! `kTISNotifySelectedKeyboardInputSourceChanged` distributed notification
//! (delivered to the main runloop). The hot path ([`paste_keycode_v`])
//! only reads an [`AtomicU16`], so paste latency is unchanged.
//!
//! Windows is intentionally not covered here. `SendInput` with
//! `wVk = VK_V` delivers `WM_KEYDOWN` to the target with `wParam = VK_V`
//! regardless of the active layout — most Windows apps treat that as
//! Ctrl+V. AutoHotkey relies on the same behaviour.
#[cfg(target_os = "macos")]
use std::sync::atomic::{AtomicU16, Ordering};
/// `kVK_ANSI_V` — the keycode for the physical V key on a US QWERTY
/// layout. Used as the fallback whenever live resolution can't produce a
/// better answer (no Unicode key layout data, lookup failure, non-macOS).
#[cfg(target_os = "macos")]
const FALLBACK_V_KEYCODE: u16 = 9;
#[cfg(target_os = "macos")]
static V_KEYCODE: AtomicU16 = AtomicU16::new(FALLBACK_V_KEYCODE);
/// Returns the keycode whose current-layout translation is `'v'`. Falls
/// back to `kVK_ANSI_V` when resolution hasn't run, the active input
/// source carries no Unicode key layout data, or no keycode in the layout
/// produces `v`.
#[cfg(target_os = "macos")]
pub fn paste_keycode_v() -> u16 {
V_KEYCODE.load(Ordering::Relaxed)
}
#[cfg(target_os = "macos")]
pub fn init() {
macos::init();
}
#[cfg(not(target_os = "macos"))]
pub fn init() {}
#[cfg(target_os = "macos")]
mod macos {
use super::{FALLBACK_V_KEYCODE, V_KEYCODE};
use core_foundation_sys::base::CFRelease;
use core_foundation_sys::data::{CFDataGetBytePtr, CFDataRef};
use core_foundation_sys::dictionary::CFDictionaryRef;
use core_foundation_sys::notification_center::{
CFNotificationCenterAddObserver, CFNotificationCenterGetDistributedCenter,
CFNotificationCenterRef, CFNotificationName,
CFNotificationSuspensionBehaviorDeliverImmediately,
};
use core_foundation_sys::string::CFStringRef;
use std::ffi::c_void;
use std::ptr;
use std::sync::atomic::Ordering;
type TISInputSourceRef = *mut c_void;
/// `kUCKeyActionDown`.
const K_UC_KEY_ACTION_DOWN: u16 = 0;
/// `kUCKeyTranslateNoDeadKeysMask` — collapse dead-key state machine so
/// a single call gives us the bare character. V is never a dead key on
/// any layout we care about, but the flag costs nothing and removes
/// any chance of ambiguous output.
const K_UC_KEY_TRANSLATE_NO_DEAD_KEYS_MASK: u32 = 1;
/// Standard US-style virtual keycodes occupy 0..0x7F. We iterate the
/// full range so non-US-extended layouts (ISO, JIS) can still be
/// resolved if their `v` lives outside the ANSI range.
const MAX_KEYCODE: u16 = 127;
const TARGET_CHAR: u16 = b'v' as u16;
#[link(name = "Carbon", kind = "framework")]
extern "C" {
fn TISCopyCurrentKeyboardLayoutInputSource() -> TISInputSourceRef;
fn TISGetInputSourceProperty(
source: TISInputSourceRef,
key: CFStringRef,
) -> *mut c_void;
fn LMGetKbdType() -> u8;
fn UCKeyTranslate(
keyboard_layout: *const u8,
virtual_key_code: u16,
key_action: u16,
modifier_key_state: u32,
keyboard_type: u32,
key_translate_options: u32,
dead_key_state: *mut u32,
max_string_length: usize,
actual_string_length: *mut usize,
unicode_string: *mut u16,
) -> i32;
static kTISPropertyUnicodeKeyLayoutData: CFStringRef;
static kTISNotifySelectedKeyboardInputSourceChanged: CFStringRef;
}
pub fn init() {
resolve_into_cache();
register_layout_change_observer();
}
fn resolve_into_cache() {
let kc = resolve_v_keycode().unwrap_or(FALLBACK_V_KEYCODE);
V_KEYCODE.store(kc, Ordering::Relaxed);
}
fn resolve_v_keycode() -> Option<u16> {
unsafe {
let source = TISCopyCurrentKeyboardLayoutInputSource();
if source.is_null() {
return None;
}
let _src_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
let layout_data_ptr =
TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
if layout_data_ptr.is_null() {
return None;
}
let layout_bytes = CFDataGetBytePtr(layout_data_ptr as CFDataRef);
if layout_bytes.is_null() {
return None;
}
let kbd_type = LMGetKbdType() as u32;
for keycode in 0..=MAX_KEYCODE {
let mut dead_key_state: u32 = 0;
let mut chars: [u16; 4] = [0; 4];
let mut actual_len: usize = 0;
let status = UCKeyTranslate(
layout_bytes,
keycode,
K_UC_KEY_ACTION_DOWN,
0, // no modifiers
kbd_type,
K_UC_KEY_TRANSLATE_NO_DEAD_KEYS_MASK,
&mut dead_key_state,
chars.len(),
&mut actual_len,
chars.as_mut_ptr(),
);
if status == 0 && actual_len == 1 && chars[0] == TARGET_CHAR {
return Some(keycode);
}
}
None
}
}
extern "C" fn layout_changed(
_center: CFNotificationCenterRef,
_observer: *mut c_void,
_name: CFNotificationName,
_object: *const c_void,
_user_info: CFDictionaryRef,
) {
resolve_into_cache();
}
fn register_layout_change_observer() {
unsafe {
let center = CFNotificationCenterGetDistributedCenter();
if center.is_null() {
return;
}
CFNotificationCenterAddObserver(
center,
ptr::null(),
layout_changed,
kTISNotifySelectedKeyboardInputSourceChanged,
ptr::null(),
CFNotificationSuspensionBehaviorDeliverImmediately,
);
}
}
}
+1
View File
@@ -0,0 +1 @@
pub mod audio_capture;
File diff suppressed because it is too large Load Diff
+180
View File
@@ -0,0 +1,180 @@
//! Rust-side subscriber for the backend `/events/speak` SSE stream.
//!
//! Owns the pill-window lifecycle for agent-initiated speech. The dictate
//! webview used to do this itself via `EventSource`, but hidden WebKit
//! windows on macOS throttle long-lived network connections, so speak events
//! never reached the pill. Tauri's event bus, on the other hand, reliably
//! delivers events to hidden webviews (the chord path proves it), so we
//! subscribe here and fan out via `emit`.
//!
//! Flow:
//! backend speak-start → show dictate window + emit("dictate:speak-start")
//! backend speak-end → emit("dictate:speak-end")
//! The pill webview handles the rest (audio playback, then emits
//! `dictate:hide` back to Rust when the audio element's `ended` fires).
//!
//! Reconnect policy: idle-timeout + escalating backoff. The stream is
//! infinite by design, so a successful round means "we were receiving
//! frames and then the backend closed the connection" (typically a
//! server restart) — reset backoff and reconnect quickly. A failure or
//! a round that produced no frames escalates backoff up to a 30 s cap so
//! long-term outages stop filling stderr with reconnect log lines.
//!
//! The idle timeout guards against the worst silent-failure mode: a
//! backend that accepts the TCP connection but stops producing frames
//! (deadlocked SSE endpoint, zombie process). Without a timeout the
//! `chunk().await` blocks forever and the task never notices. The
//! backend emits a `:ping` comment every 15 s, so 45 s without any data
//! is a reliable signal the stream is dead.
use std::time::Duration;
use tauri::{AppHandle, Emitter};
use crate::{ensure_dictate_window, SERVER_PORT};
const INITIAL_BACKOFF: Duration = Duration::from_millis(500);
const MAX_BACKOFF: Duration = Duration::from_secs(30);
/// Backend emits a `:ping` heartbeat every 15 s. Giving the stream 45 s
/// of idle budget absorbs one missed heartbeat (slow GC pause, brief
/// backend stall) without being so long that a truly dead stream blocks
/// the pill from surfacing for minutes.
const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(45);
pub fn spawn_speak_monitor(app: AppHandle) {
tauri::async_runtime::spawn(async move {
run(app).await;
});
}
async fn run(app: AppHandle) {
let url = format!("http://127.0.0.1:{}/events/speak", SERVER_PORT);
let client = match reqwest::Client::builder().build() {
Ok(c) => c,
Err(e) => {
eprintln!("speak_monitor: failed to build HTTP client: {e}");
return;
}
};
let mut backoff = INITIAL_BACKOFF;
let mut attempt: u32 = 0;
loop {
let stream_result = stream_once(&client, &url, &app).await;
let had_success = matches!(stream_result, Ok(true));
if had_success {
backoff = INITIAL_BACKOFF;
attempt = 0;
} else {
attempt += 1;
let reason = match stream_result {
Ok(_) => "stream closed without data".to_string(),
Err(e) => format!("stream err: {e}"),
};
eprintln!(
"speak_monitor: {reason} (attempt {attempt}, retry in {:?})",
backoff
);
}
tokio::time::sleep(backoff).await;
if !had_success {
backoff = (backoff * 2).min(MAX_BACKOFF);
}
}
}
/// Consume the SSE stream until it closes or errors. Returns `Ok(true)`
/// if at least one frame was received (the connection was genuinely
/// productive), `Ok(false)` on a clean but empty close, and `Err` for
/// any connection or parse failure.
async fn stream_once(
client: &reqwest::Client,
url: &str,
app: &AppHandle,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let mut resp = client
.get(url)
.header("Accept", "text/event-stream")
.send()
.await?;
if !resp.status().is_success() {
return Err(format!("speak_monitor: backend returned {}", resp.status()).into());
}
let mut buf = String::new();
let mut saw_data = false;
loop {
let chunk = match tokio::time::timeout(STREAM_IDLE_TIMEOUT, resp.chunk()).await {
Ok(Ok(Some(chunk))) => chunk,
Ok(Ok(None)) => return Ok(saw_data),
Ok(Err(e)) => return Err(Box::new(e)),
Err(_) => {
return Err(format!(
"no data for {:?} (heartbeat should arrive every 15 s)",
STREAM_IDLE_TIMEOUT
)
.into())
}
};
saw_data = true;
buf.push_str(std::str::from_utf8(&chunk)?);
// sse-starlette emits CRLF framing; the spec also permits LF, so
// handle either. Drain whichever separator appears first.
loop {
let crlf = buf.find("\r\n\r\n");
let lf = buf.find("\n\n");
let (end, sep_len) = match (crlf, lf) {
(Some(c), Some(l)) if c <= l => (c, 4),
(Some(c), None) => (c, 4),
(_, Some(l)) => (l, 2),
(None, None) => break,
};
let frame: String = buf.drain(..end + sep_len).collect();
if let Some((event, data)) = parse_frame(&frame) {
dispatch(app, &event, &data);
}
}
}
}
/// Parse a single SSE frame into (event_name, data_json).
///
/// Returns None for comment-only frames (lines starting with `:`) and
/// for frames without a recognizable `event:` or `data:` line.
fn parse_frame(frame: &str) -> Option<(String, String)> {
let mut event: Option<String> = None;
let mut data_lines: Vec<&str> = Vec::new();
for line in frame.lines() {
if line.is_empty() || line.starts_with(':') {
continue;
}
if let Some(rest) = line.strip_prefix("event:") {
event = Some(rest.trim().to_string());
} else if let Some(rest) = line.strip_prefix("data:") {
data_lines.push(rest.trim_start());
}
}
let event = event?;
let data = data_lines.join("\n");
Some((event, data))
}
fn dispatch(app: &AppHandle, event: &str, data: &str) {
match event {
"speak-start" => {
// Defensive for dev/restart paths where the setup-created pill
// is not present — but don't *show* it here. The pill
// surfaces itself from `audio.onplaying` via `dictate:show`, so
// users never see the empty-silent generation window.
ensure_dictate_window(app);
let _ = app.emit("dictate:speak-start", data.to_string());
}
"speak-end" => {
let _ = app.emit("dictate:speak-end", data.to_string());
}
// `ready` and `ping` are heartbeats; ignore.
_ => {}
}
}
+220
View File
@@ -0,0 +1,220 @@
//! Synthetic keyboard event posting for the auto-paste pipeline.
//!
//! `send_paste` fires the four-event paste sequence onto the OS input
//! pipeline so the focused app performs its native paste action against
//! whatever the clipboard module has just staged.
//!
//! - **macOS** — Cmd down with Cmd flag, V down with Cmd flag, V up with
//! Cmd flag, Cmd up via `CGEventPost` at `kCGHIDEventTap`. The Cmd-down
//! event carries the Command flag so its `flagsChanged` representation
//! matches hardware — Electron/Chromium tracks modifier state from that
//! flag and drops the paste otherwise (see the note on the event table).
//! Accessibility permission is load-bearing: without it the system
//! swallows the events silently, so callers must gate on
//! [`crate::accessibility::is_trusted`].
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
//! permission gate, but UAC/UIPI blocks delivery into elevated target
//! windows when we run non-elevated — nothing we can do short of also
//! running elevated.
//!
//! On macOS the V keycode is resolved per-layout by
//! [`crate::keyboard_layout`] — Cmd+V is matched against the layout-
//! translated character via NSMenu key equivalents, so hardcoding
//! `kVK_ANSI_V` (the QWERTY V position) would fire Cmd+. on Dvorak. The
//! resolved keycode is read once per paste from an atomic; the cache is
//! primed at startup and refreshed on layout change.
//!
//! Windows hardcodes `VK_V`. `SendInput` with `wVk = VK_V` makes the
//! target receive `WM_KEYDOWN` with `wParam = VK_V` regardless of the
//! active layout, and most Windows apps treat that as Ctrl+V (the same
//! reason `Send "^v"` works in AutoHotkey on Dvorak Windows).
#[cfg(target_os = "macos")]
use std::ffi::c_void;
#[cfg(target_os = "macos")]
mod ffi {
use std::ffi::c_void;
#[repr(C)]
pub struct CGEvent {
_opaque: [u8; 0],
}
pub type CGEventRef = *mut CGEvent;
#[repr(C)]
pub struct CGEventSource {
_opaque: [u8; 0],
}
pub type CGEventSourceRef = *mut CGEventSource;
pub type CGEventTapLocation = u32;
pub type CGKeyCode = u16;
pub type CGEventFlags = u64;
pub type CGEventSourceStateID = i32;
/// `kCGHIDEventTap` — posted events enter at the HID level so every
/// downstream tap (including the target app) sees them exactly as if the
/// hardware had produced them.
pub const K_CG_HID_EVENT_TAP: CGEventTapLocation = 0;
/// `kCGEventSourceStateHIDSystemState` — mimics hardware, which is what
/// we want: modifier bookkeeping inside target apps stays consistent.
pub const K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE: CGEventSourceStateID = 1;
/// `kCGEventFlagMaskCommand` — the Cmd modifier bit inside `CGEventFlags`.
pub const K_CG_EVENT_FLAG_MASK_COMMAND: CGEventFlags = 0x00100000;
/// `kVK_Command` (left Cmd).
pub const KEYCODE_LEFT_CMD: CGKeyCode = 0x37;
#[link(name = "CoreGraphics", kind = "framework")]
extern "C" {
pub fn CGEventSourceCreate(state_id: CGEventSourceStateID) -> CGEventSourceRef;
pub fn CGEventCreateKeyboardEvent(
source: CGEventSourceRef,
virtual_key: CGKeyCode,
key_down: bool,
) -> CGEventRef;
pub fn CGEventSetFlags(event: CGEventRef, flags: CGEventFlags);
pub fn CGEventPost(tap: CGEventTapLocation, event: CGEventRef);
}
#[link(name = "CoreFoundation", kind = "framework")]
extern "C" {
pub fn CFRelease(cf: *const c_void);
}
}
/// Post the four-event Cmd+V sequence to the HID event tap.
///
/// Returns after the events are queued — there's no completion callback,
/// so callers should sleep briefly afterwards to let the target app
/// process the paste before any follow-up (e.g. clipboard restore).
#[cfg(target_os = "macos")]
pub fn send_paste() -> Result<(), String> {
use ffi::*;
let v_keycode = crate::keyboard_layout::paste_keycode_v();
unsafe {
let source = CGEventSourceCreate(K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE);
if source.is_null() {
return Err("CGEventSourceCreate returned null".into());
}
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
let events = [
// The Cmd-down event must carry the Command flag itself. On real
// hardware the Cmd keyDown is a flagsChanged event whose flags
// already include Command; Chromium/Electron builds its tracked
// modifier state from that flag. Posting Cmd-down with flags = 0
// leaves that tracker showing "Command up", so the following V —
// even though its own flags carry Command — matches neither the
// Cmd+V accelerator (tracker says no modifier) nor plain-text
// insertion (event flags say Command), and Electron drops it
// silently. AppKit reads the V event's own flags and pastes
// regardless, which is why native apps worked but Electron
// targets (Slack, VS Code) silently no-op'd.
(KEYCODE_LEFT_CMD, true, K_CG_EVENT_FLAG_MASK_COMMAND),
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
(KEYCODE_LEFT_CMD, false, 0),
];
// Build the four events up front so CFRelease happens after all posts.
// Posting in a loop that interleaved create → post → release would
// work, but keeping the events alive for the full sequence matches
// the pattern CGEventPost's docs show and is easier to reason about.
let mut guards = Vec::with_capacity(events.len());
let mut created = Vec::with_capacity(events.len());
for (key, down, flags) in events {
let event = CGEventCreateKeyboardEvent(source, key, down);
if event.is_null() {
return Err(format!(
"CGEventCreateKeyboardEvent(key={}, down={}) returned null",
key, down
));
}
let guard = scopeguard::guard(event, |e| CFRelease(e as *const c_void));
if flags != 0 {
CGEventSetFlags(event, flags);
}
created.push(event);
guards.push(guard);
}
for event in created {
CGEventPost(K_CG_HID_EVENT_TAP, event);
}
drop(guards);
Ok(())
}
}
#[cfg(target_os = "windows")]
mod win {
use windows::Win32::UI::Input::KeyboardAndMouse::{
INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_KEYUP,
VIRTUAL_KEY,
};
pub fn make_key(vk: VIRTUAL_KEY, up: bool) -> INPUT {
let flags = if up {
KEYEVENTF_KEYUP
} else {
KEYBD_EVENT_FLAGS(0)
};
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: vk,
wScan: 0,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
}
}
}
#[cfg(target_os = "windows")]
pub fn send_paste() -> Result<(), String> {
use windows::Win32::UI::Input::KeyboardAndMouse::{
SendInput, INPUT, VK_CONTROL, VK_V,
};
// Four-event Ctrl+V sequence. Matches the macOS CGEvent pattern: the
// modifier brackets the letter so the target app sees a fully formed
// accelerator rather than a lone V. `dwExtraInfo` is zero — we're not
// tagging these as "ours" because no consumer in the paste path needs
// to distinguish synthetic events from hardware ones.
let events = [
win::make_key(VK_CONTROL, false),
win::make_key(VK_V, false),
win::make_key(VK_V, true),
win::make_key(VK_CONTROL, true),
];
unsafe {
let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);
if sent as usize != events.len() {
return Err(format!(
"SendInput delivered {} of {} events — the input desktop may be locked (secure attention sequence) or a higher-integrity window is intercepting.",
sent,
events.len()
));
}
}
Ok(())
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
pub fn send_paste() -> Result<(), String> {
Err("synthetic paste is not yet implemented on this platform".into())
}
+67
View File
@@ -0,0 +1,67 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "Voicebox",
"version": "0.5.0",
"identifier": "sh.voicebox.app",
"build": {
"beforeDevCommand": "bun run dev",
"beforeBuildCommand": "bun run build",
"frontendDist": "../dist",
"devUrl": "http://localhost:5173"
},
"bundle": {
"active": true,
"targets": "all",
"createUpdaterArtifacts": "v1Compatible",
"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"],
"icon": [
"icons/32x32.png",
"icons/128x128.png",
"icons/[email protected]",
"icons/icon.icns",
"icons/icon.ico"
],
"macOS": {
"frameworks": [],
"minimumSystemVersion": "11.0",
"infoPlist": "Info.plist",
"entitlements": "Entitlements.plist"
},
"resources": {
"gen/Assets.car": "./",
"gen/voicebox.icns": "./",
"gen/partial.plist": "./"
}
},
"app": {
"macOSPrivateApi": true,
"security": {
"csp": null,
"capabilities": ["default"]
},
"windows": [
{
"title": "",
"width": 1200,
"height": 800,
"minWidth": 800,
"minHeight": 600,
"resizable": true,
"fullscreen": false,
"devtools": true,
"userAgent": null,
"titleBarStyle": "Overlay"
}
],
"withGlobalTauri": true
},
"plugins": {
"shell": {
"open": ".*"
},
"updater": {
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IEUxRENBQkRBQjdBNTM1OTIKUldTU05hVzMycXZjNGJGcUxmcVVocll2QjdSaTJNdlFxR2M3VDJsMnVvbDdyZGRPMmRlOW9aWTcK",
"endpoints": ["https://github.com/jamiepine/voicebox/releases/latest/download/latest.json"]
}
}
}
@@ -0,0 +1,59 @@
// NOTE: This test requires system audio to be playing during execution.
// To run this test successfully:
// 1. Start playing audio (music, video, etc.)
// 2. Run: cargo test --test audio_capture_test -- --nocapture
// 3. The test will capture audio for 5 seconds and verify the output
use voicebox::audio_capture::{AudioCaptureState, start_capture, stop_capture};
use base64::Engine;
#[tokio::test]
async fn test_system_audio_capture() {
// Create AudioCaptureState
let state = AudioCaptureState::new();
println!("Starting system audio capture with 5 second max duration...");
// Start capture with 5 second max duration
let result = start_capture(&state, 5).await;
if let Err(e) = result {
panic!("Failed to start capture: {}", e);
}
println!("Capture started, waiting 5 seconds...");
// Wait 5 seconds for capture to complete
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
println!("Stopping capture...");
// Stop capture and get the result
let audio_data = stop_capture(&state).await;
match audio_data {
Ok(base64_wav) => {
println!("Capture stopped successfully");
// Validate the returned base64 WAV data
println!("Validating base64 WAV data...");
// Decode base64 to bytes
let decoded_bytes = base64::engine::general_purpose::STANDARD
.decode(&base64_wav)
.expect("Failed to decode base64 data");
// Verify bytes array is not empty
assert!(!decoded_bytes.is_empty(), "Decoded bytes array is empty");
// Confirm data has content (length > 0)
println!("WAV data length: {} bytes", decoded_bytes.len());
assert!(decoded_bytes.len() > 0, "WAV data has no content");
println!("✓ Test passed: Audio capture produced valid WAV data");
}
Err(e) => {
panic!("Failed to stop capture or get audio data: {}", e);
}
}
}
+31
View File
@@ -0,0 +1,31 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
// import { ReactQueryDevtools } from '@tanstack/react-query-devtools';
import App from '@/App';
// Import CSS from app directory using alias so Tailwind can scan the source files
import '@/index.css';
import { PlatformProvider } from '@/platform/PlatformContext';
import { tauriPlatform } from './platform';
const queryClient = new QueryClient({
defaultOptions: {
queries: {
staleTime: 1000 * 60 * 5, // 5 minutes
gcTime: 1000 * 60 * 10, // 10 minutes
retry: 1,
refetchOnWindowFocus: false,
},
},
});
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<QueryClientProvider client={queryClient}>
<PlatformProvider platform={tauriPlatform}>
<App />
{/* <ReactQueryDevtools initialIsOpen={false} /> */}
</PlatformProvider>
</QueryClientProvider>
</React.StrictMode>,
);
+44
View File
@@ -0,0 +1,44 @@
import { invoke } from '@tauri-apps/api/core';
import type { PlatformAudio, AudioDevice } from '@/platform/types';
export const tauriAudio: PlatformAudio = {
async isSystemAudioSupported(): Promise<boolean> {
return await invoke<boolean>('is_system_audio_supported');
},
async startSystemAudioCapture(maxDurationSecs: number): Promise<void> {
await invoke('start_system_audio_capture', {
maxDurationSecs,
});
},
async stopSystemAudioCapture(): Promise<Blob> {
const base64Data = await invoke<string>('stop_system_audio_capture');
// Convert base64 to Blob
const binaryString = atob(base64Data);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return new Blob([bytes], { type: 'audio/wav' });
},
async listOutputDevices(): Promise<AudioDevice[]> {
return await invoke<AudioDevice[]>('list_audio_output_devices');
},
async playToDevices(audioData: Uint8Array, deviceIds: string[]): Promise<void> {
await invoke('play_audio_to_devices', {
audioData: Array.from(audioData),
deviceIds,
});
},
stopPlayback(): void {
invoke('stop_audio_playback').catch((error) => {
console.error('Failed to stop audio playback:', error);
});
},
};
+38
View File
@@ -0,0 +1,38 @@
import type { FileFilter, PlatformFilesystem } from '@/platform/types';
export const tauriFilesystem: PlatformFilesystem = {
async saveFile(filename: string, blob: Blob, filters?: FileFilter[]) {
const { save } = await import('@tauri-apps/plugin-dialog');
const { writeFile } = await import('@tauri-apps/plugin-fs');
const filePath = await save({
defaultPath: filename,
filters: filters || [],
});
if (!filePath) return; // User cancelled the dialog
const resolvedPath =
typeof filePath === 'string' ? filePath : (filePath as { path: string }).path;
if (!resolvedPath) {
throw new Error('Failed to resolve save path from dialog');
}
const arrayBuffer = await blob.arrayBuffer();
await writeFile(resolvedPath, new Uint8Array(arrayBuffer));
},
async openPath(path: string) {
const { open } = await import('@tauri-apps/plugin-shell');
await open(path);
},
async pickDirectory(title: string) {
const { open } = await import('@tauri-apps/plugin-dialog');
const selected = await open({ directory: true, title });
if (!selected) return null;
const dir = typeof selected === 'string' ? selected : (selected as { path: string }).path;
return dir || null;
},
};
+14
View File
@@ -0,0 +1,14 @@
import type { Platform } from '@/platform/types';
import { tauriFilesystem } from './filesystem';
import { tauriUpdater } from './updater';
import { tauriAudio } from './audio';
import { tauriLifecycle } from './lifecycle';
import { tauriMetadata } from './metadata';
export const tauriPlatform: Platform = {
filesystem: tauriFilesystem,
updater: tauriUpdater,
audio: tauriAudio,
lifecycle: tauriLifecycle,
metadata: tauriMetadata,
};
+125
View File
@@ -0,0 +1,125 @@
import { invoke } from '@tauri-apps/api/core';
import { emit, listen } from '@tauri-apps/api/event';
import type { PlatformLifecycle, ServerLogEntry } from '@/platform/types';
class TauriLifecycle implements PlatformLifecycle {
onServerReady?: () => void;
async startServer(remote = false, modelsDir?: string | null): Promise<string> {
try {
const result = await invoke<string>('start_server', {
remote,
modelsDir: modelsDir ?? undefined,
});
console.log('Server started:', result);
this.onServerReady?.();
return result;
} catch (error) {
console.error('Failed to start server:', error);
throw error;
}
}
async stopServer(): Promise<void> {
try {
await invoke('stop_server');
console.log('Server stopped');
} catch (error) {
console.error('Failed to stop server:', error);
throw error;
}
}
async restartServer(modelsDir?: string | null): Promise<string> {
try {
const result = await invoke<string>('restart_server', {
modelsDir: modelsDir ?? undefined,
});
console.log('Server restarted:', result);
this.onServerReady?.();
return result;
} catch (error) {
console.error('Failed to restart server:', error);
throw error;
}
}
async setKeepServerRunning(keepRunning: boolean): Promise<void> {
try {
await invoke('set_keep_server_running', { keepRunning });
} catch (error) {
console.error('Failed to set keep server running setting:', error);
}
}
async setBackendOverride(backend?: string | null): Promise<void> {
try {
await invoke('set_backend_override', { backend: backend ?? undefined });
} catch (error) {
console.error('Failed to set backend override:', error);
throw error;
}
}
async setupWindowCloseHandler(): Promise<void> {
try {
// Listen for window close request from Rust
await listen<null>('window-close-requested', async () => {
// Import store here to avoid circular dependency
const { useServerStore } = await import('@/stores/serverStore');
const keepRunning = useServerStore.getState().keepServerRunningOnClose;
// Check if server was started by this app instance
// @ts-expect-error - accessing module-level variable from another module
const serverStartedByApp = window.__voiceboxServerStartedByApp ?? false;
console.log(
'[lifecycle] window-close-requested: keepRunning=%s, serverStartedByApp=%s',
keepRunning,
serverStartedByApp,
);
if (!keepRunning && serverStartedByApp) {
// Stop server before closing (only if we started it)
try {
await this.stopServer();
} catch (error) {
console.error('Failed to stop server on close:', error);
}
}
// Emit event back to Rust to allow close
await emit('window-close-allowed');
});
} catch (error) {
console.error('Failed to setup window close handler:', error);
}
}
subscribeToServerLogs(callback: (entry: ServerLogEntry) => void): () => void {
let disposed = false;
let unlisten: (() => void) | null = null;
void listen<ServerLogEntry>('server-log', (event) => {
callback(event.payload);
})
.then((fn) => {
if (disposed) {
fn();
return;
}
unlisten = fn;
})
.catch((error) => {
console.error('Failed to subscribe to server logs:', error);
});
return () => {
disposed = true;
unlisten?.();
unlisten = null;
};
}
}
export const tauriLifecycle = new TauriLifecycle();
+14
View File
@@ -0,0 +1,14 @@
import { getVersion } from '@tauri-apps/api/app';
import type { PlatformMetadata } from '@/platform/types';
export const tauriMetadata: PlatformMetadata = {
async getVersion(): Promise<string> {
try {
return await getVersion();
} catch (error) {
console.error('Failed to get version:', error);
return '0.1.0';
}
},
isTauri: true,
};

Some files were not shown because too many files have changed in this diff Show More