Initial commit (forked from jamiepine/voicebox)
|
After Width: | Height: | Size: 4.0 MiB |
|
After Width: | Height: | Size: 4.0 MiB |
|
After Width: | Height: | Size: 4.0 MiB |
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"fill": "automatic",
|
||||
"groups": [
|
||||
{
|
||||
"layers": [
|
||||
{
|
||||
"image-name": "Voicebox_Microphone.png",
|
||||
"name": "Voicebox_Microphone",
|
||||
"position": {
|
||||
"scale": 0.36,
|
||||
"translation-in-points": [0.140625, 270.1875]
|
||||
}
|
||||
}
|
||||
],
|
||||
"shadow": {
|
||||
"kind": "neutral",
|
||||
"opacity": 0.5
|
||||
},
|
||||
"translucency": {
|
||||
"enabled": true,
|
||||
"value": 0.5
|
||||
}
|
||||
}
|
||||
],
|
||||
"supported-platforms": {
|
||||
"circles": ["watchOS"],
|
||||
"squares": "shared"
|
||||
}
|
||||
}
|
||||
|
After Width: | Height: | Size: 2.1 MiB |
|
After Width: | Height: | Size: 1.8 MiB |
|
After Width: | Height: | Size: 2.8 MiB |
|
After Width: | Height: | Size: 2.5 MiB |
|
After Width: | Height: | Size: 1.0 MiB |
|
After Width: | Height: | Size: 2.2 MiB |
|
After Width: | Height: | Size: 2.6 MiB |
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>voicebox</title>
|
||||
<script>
|
||||
(function () {
|
||||
try {
|
||||
var theme = 'system';
|
||||
var raw = localStorage.getItem('voicebox-ui');
|
||||
if (raw) {
|
||||
var parsed = JSON.parse(raw);
|
||||
if (parsed && parsed.state && parsed.state.theme) theme = parsed.state.theme;
|
||||
}
|
||||
var resolved = theme === 'system'
|
||||
? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light')
|
||||
: theme;
|
||||
if (resolved === 'dark') document.documentElement.classList.add('dark');
|
||||
} catch (_) {}
|
||||
})();
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "@voicebox/tauri",
|
||||
"private": true,
|
||||
"version": "0.5.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"tauri": "tauri"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tauri-apps/api": "^2.0.0",
|
||||
"@tauri-apps/plugin-dialog": "^2.0.0",
|
||||
"@tauri-apps/plugin-fs": "^2.0.0",
|
||||
"@tauri-apps/plugin-process": "^2.0.0",
|
||||
"@tauri-apps/plugin-shell": "^2.0.0",
|
||||
"@tauri-apps/plugin-updater": "^2.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.1.18",
|
||||
"@tauri-apps/cli": "^2.0.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react": "^4.3.0",
|
||||
"tailwindcss": "^4.1.18",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"typescript": "^5.6.0",
|
||||
"vite": "^5.4.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
[package]
|
||||
name = "voicebox"
|
||||
version = "0.5.0"
|
||||
description = "A production-quality desktop app for Qwen3-TTS voice cloning and generation"
|
||||
authors = ["you"]
|
||||
license = ""
|
||||
repository = ""
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[build-dependencies]
|
||||
tauri-build = { version = "2.0", features = [] }
|
||||
|
||||
[dependencies]
|
||||
tauri = { version = "2.0", features = ["macos-private-api"] }
|
||||
tauri-plugin-dialog = "2.0"
|
||||
tauri-plugin-fs = "2.0"
|
||||
tauri-plugin-shell = "2.0"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
reqwest = { version = "0.12", features = ["blocking", "json", "stream"] }
|
||||
hound = "3.5"
|
||||
base64 = "0.22"
|
||||
cpal = "0.15"
|
||||
symphonia = { version = "0.5", features = ["all"] }
|
||||
scopeguard = "1.2.0"
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
screencapturekit = { version = "1", features = ["async"] }
|
||||
coreaudio-sys = "0.2"
|
||||
objc = "0.2"
|
||||
core-foundation-sys = "0.8"
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
wasapi = "0.22"
|
||||
windows = { version = "0.62", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_UI_WindowsAndMessaging",
|
||||
"Win32_UI_Accessibility",
|
||||
"Win32_UI_Input_KeyboardAndMouse",
|
||||
"Win32_System_Com",
|
||||
"Win32_System_DataExchange",
|
||||
"Win32_System_Memory",
|
||||
"Win32_System_Threading",
|
||||
] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
webkit2gtk = "2.0"
|
||||
|
||||
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
|
||||
tauri-plugin-updater = "2.0"
|
||||
tauri-plugin-process = "2.0"
|
||||
# Observe-only global keyboard tap. Covers macOS, Windows, and Linux
|
||||
# (evdev) with left/right modifier fidelity, which is the hard
|
||||
# requirement the chord engine needs. keytap is our own crate
|
||||
# (jamiepine/keytap), published to crates.io, that replaces the
|
||||
# abandoned Narsil/rdev we previously pinned via git — same capability
|
||||
# surface, clean shutdown via Drop, no `set_is_main_thread(false)`
|
||||
# dance required (the Sonoma-crashing layout-translation path simply
|
||||
# isn't called).
|
||||
keytap = "0.4"
|
||||
|
||||
[features]
|
||||
# This feature is used for production builds or when `devPath` points to the filesystem
|
||||
custom-protocol = ["tauri/custom-protocol"]
|
||||
@@ -0,0 +1,16 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>com.apple.security.cs.allow-jit</key>
|
||||
<true/>
|
||||
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
|
||||
<true/>
|
||||
<key>com.apple.security.cs.disable-library-validation</key>
|
||||
<true/>
|
||||
<key>com.apple.security.device.audio-input</key>
|
||||
<true/>
|
||||
<key>com.apple.security.files.user-selected.read-write</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIconFile</key>
|
||||
<string>voicebox</string>
|
||||
<key>CFBundleIconName</key>
|
||||
<string>voicebox</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>voicebox needs microphone access to record voice samples for voice cloning.</string>
|
||||
<key>NSScreenCaptureUsageDescription</key>
|
||||
<string>Voicebox needs screen capture access to record system audio for voice samples.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,170 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
// Link Swift runtime libraries for screencapturekit crate
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// ScreenCaptureKit does not exist on macOS 11, so weak-link it to
|
||||
// allow the app to launch and gate usage at runtime instead.
|
||||
println!("cargo:rustc-link-arg=-Wl,-weak_framework,ScreenCaptureKit");
|
||||
|
||||
// Add Swift runtime library paths to RPATH
|
||||
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
|
||||
println!("cargo:rustc-link-arg=-L/usr/lib/swift");
|
||||
|
||||
// Also try Xcode's Swift libraries
|
||||
if let Ok(output) = Command::new("xcode-select").arg("-p").output() {
|
||||
if output.status.success() {
|
||||
let xcode_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
let swift_lib_path = format!(
|
||||
"{}/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx",
|
||||
xcode_path
|
||||
);
|
||||
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", swift_lib_path);
|
||||
println!("cargo:rustc-link-arg=-L{}", swift_lib_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let project_root = env!("CARGO_MANIFEST_DIR");
|
||||
let gen_dir = format!("{}/gen", project_root);
|
||||
std::fs::create_dir_all(&gen_dir).expect("Failed to create gen directory");
|
||||
|
||||
// Compile macOS Liquid Glass icon
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// voicebox.icon is in tauri/assets/voicebox.icon (one level up from src-tauri)
|
||||
let icon_source = format!("{}/../assets/voicebox.icon", project_root);
|
||||
|
||||
if std::path::Path::new(&icon_source).exists() {
|
||||
println!("cargo:rerun-if-changed={}", icon_source);
|
||||
println!("cargo:rerun-if-changed={}/icon.json", icon_source);
|
||||
println!("cargo:rerun-if-changed={}/Assets", icon_source);
|
||||
|
||||
let partial_plist = format!("{}/partial.plist", gen_dir);
|
||||
let output = Command::new("xcrun")
|
||||
.args([
|
||||
"actool",
|
||||
"--compile",
|
||||
&gen_dir,
|
||||
"--output-format",
|
||||
"human-readable-text",
|
||||
"--output-partial-info-plist",
|
||||
&partial_plist,
|
||||
"--app-icon",
|
||||
"voicebox",
|
||||
"--include-all-app-icons",
|
||||
"--target-device",
|
||||
"mac",
|
||||
"--minimum-deployment-target",
|
||||
"11.0",
|
||||
"--platform",
|
||||
"macosx",
|
||||
&icon_source,
|
||||
])
|
||||
.output();
|
||||
|
||||
match output {
|
||||
Ok(output) => {
|
||||
if !output.status.success() {
|
||||
eprintln!("actool stderr: {}", String::from_utf8_lossy(&output.stderr));
|
||||
eprintln!("actool stdout: {}", String::from_utf8_lossy(&output.stdout));
|
||||
panic!("actool failed to compile icon");
|
||||
}
|
||||
println!("Successfully compiled icon to {}", gen_dir);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Failed to execute xcrun actool: {}", e);
|
||||
eprintln!("Make sure you have Xcode Command Line Tools installed");
|
||||
panic!("Icon compilation failed");
|
||||
}
|
||||
}
|
||||
|
||||
// Generate voicebox.icns from the source PNG via sips + iconutil
|
||||
let icns_path = format!("{}/voicebox.icns", gen_dir);
|
||||
if !std::path::Path::new(&icns_path).exists() {
|
||||
let source_png = format!("{}/Assets/Voicebox.png", icon_source);
|
||||
if std::path::Path::new(&source_png).exists() {
|
||||
let iconset_dir = format!("{}/voicebox.iconset", gen_dir);
|
||||
std::fs::create_dir_all(&iconset_dir).ok();
|
||||
|
||||
let sizes: &[(u32, &str)] = &[
|
||||
(16, "icon_16x16.png"),
|
||||
(32, "[email protected]"),
|
||||
(32, "icon_32x32.png"),
|
||||
(64, "[email protected]"),
|
||||
(128, "icon_128x128.png"),
|
||||
(256, "[email protected]"),
|
||||
(256, "icon_256x256.png"),
|
||||
(512, "[email protected]"),
|
||||
(512, "icon_512x512.png"),
|
||||
(1024, "[email protected]"),
|
||||
];
|
||||
|
||||
for (size, name) in sizes {
|
||||
let dest = format!("{}/{}", iconset_dir, name);
|
||||
let status = Command::new("sips")
|
||||
.args([
|
||||
"-z",
|
||||
&size.to_string(),
|
||||
&size.to_string(),
|
||||
&source_png,
|
||||
"--out",
|
||||
&dest,
|
||||
])
|
||||
.output();
|
||||
if let Ok(out) = status {
|
||||
if !out.status.success() {
|
||||
eprintln!(
|
||||
"sips failed for {}: {}",
|
||||
name,
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let iconutil_output = Command::new("iconutil")
|
||||
.args(["-c", "icns", "-o", &icns_path, &iconset_dir])
|
||||
.output();
|
||||
|
||||
match iconutil_output {
|
||||
Ok(out) if out.status.success() => {
|
||||
println!("Generated voicebox.icns");
|
||||
}
|
||||
Ok(out) => {
|
||||
eprintln!("iconutil failed: {}", String::from_utf8_lossy(&out.stderr));
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Failed to run iconutil: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up iconset directory
|
||||
std::fs::remove_dir_all(&iconset_dir).ok();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!(
|
||||
"cargo:warning=Icon source not found at {}, skipping icon compilation",
|
||||
icon_source
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure all resource files exist so Tauri's bundler doesn't fail.
|
||||
// On non-macOS these are always stubs. On macOS, actool may not produce
|
||||
// Assets.car if the Xcode version doesn't support the .icon format.
|
||||
{
|
||||
let required = ["Assets.car", "voicebox.icns", "partial.plist"];
|
||||
for name in required {
|
||||
let path = format!("{}/{}", gen_dir, name);
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
std::fs::write(&path, b"").ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tauri_build::build()
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"identifier": "default",
|
||||
"description": "Default permissions for voicebox",
|
||||
"platforms": ["linux", "macOS", "windows"],
|
||||
"windows": ["main", "dictate"],
|
||||
"remote": {
|
||||
"urls": ["http://localhost:*"]
|
||||
},
|
||||
"permissions": [
|
||||
"core:default",
|
||||
"core:window:default",
|
||||
"core:window:allow-start-dragging",
|
||||
"core:webview:default",
|
||||
"core:webview:allow-internal-toggle-devtools",
|
||||
"shell:allow-open",
|
||||
"shell:allow-execute",
|
||||
"shell:allow-spawn",
|
||||
"updater:default",
|
||||
"process:default",
|
||||
"dialog:default",
|
||||
"dialog:allow-save",
|
||||
"dialog:allow-open",
|
||||
"fs:default",
|
||||
"fs:read-all",
|
||||
"fs:write-all"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"default":{"identifier":"default","description":"Default permissions for voicebox","remote":{"urls":["http://localhost:*"]},"local":true,"windows":["main","dictate"],"permissions":["core:default","core:window:default","core:window:allow-start-dragging","core:webview:default","core:webview:allow-internal-toggle-devtools","shell:allow-open","shell:allow-execute","shell:allow-spawn","updater:default","process:default","dialog:default","dialog:allow-save","dialog:allow-open","fs:default","fs:read-all","fs:write-all"],"platforms":["linux","macOS","windows"]}}
|
||||
|
After Width: | Height: | Size: 27 KiB |
|
After Width: | Height: | Size: 90 KiB |
|
After Width: | Height: | Size: 3.4 KiB |
|
After Width: | Height: | Size: 8.9 KiB |
|
After Width: | Height: | Size: 20 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 36 KiB |
|
After Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 3.1 KiB |
|
After Width: | Height: | Size: 128 KiB |
|
After Width: | Height: | Size: 5.1 KiB |
|
After Width: | Height: | Size: 10 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 6.1 KiB |
@@ -0,0 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<foreground android:drawable="@mipmap/ic_launcher_foreground"/>
|
||||
<background android:drawable="@color/ic_launcher_background"/>
|
||||
</adaptive-icon>
|
||||
|
After Width: | Height: | Size: 11 KiB |
|
After Width: | Height: | Size: 11 KiB |
|
After Width: | Height: | Size: 11 KiB |
|
After Width: | Height: | Size: 5.8 KiB |
|
After Width: | Height: | Size: 5.8 KiB |
|
After Width: | Height: | Size: 5.8 KiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 54 KiB |
|
After Width: | Height: | Size: 54 KiB |
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<color name="ic_launcher_background">#fff</color>
|
||||
</resources>
|
||||
|
After Width: | Height: | Size: 95 KiB |
|
After Width: | Height: | Size: 311 KiB |
|
After Width: | Height: | Size: 296 B |
|
After Width: | Height: | Size: 514 B |
|
After Width: | Height: | Size: 514 B |
|
After Width: | Height: | Size: 831 B |
|
After Width: | Height: | Size: 388 B |
|
After Width: | Height: | Size: 793 B |
|
After Width: | Height: | Size: 793 B |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 514 B |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 1.1 KiB |
|
After Width: | Height: | Size: 1.6 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 1.6 KiB |
|
After Width: | Height: | Size: 2.5 KiB |
|
After Width: | Height: | Size: 1.0 KiB |
|
After Width: | Height: | Size: 2.1 KiB |
|
After Width: | Height: | Size: 2.3 KiB |
@@ -0,0 +1,42 @@
|
||||
//! Platform permission gate for the auto-paste pipeline.
|
||||
//!
|
||||
//! On macOS, posting synthetic keyboard events and reading focused-UI state
|
||||
//! via the AX API both require the host process to be listed under System
|
||||
//! Settings → Privacy & Security → Accessibility. Without that trust,
|
||||
//! `CGEventPost` silently drops events and `AXUIElementCopyAttributeValue`
|
||||
//! returns an error. We surface a boolean check up front so the paste
|
||||
//! pipeline can short-circuit with a clear "grant permission" message
|
||||
//! instead of running through the full save → write → post → restore dance
|
||||
//! with nothing to show for it.
|
||||
//!
|
||||
//! Windows has no equivalent user-facing permission — `SendInput` and
|
||||
//! UIAutomation work for any non-elevated target out of the box. (UAC /
|
||||
//! UIPI still blocks sending input *into* an elevated target window from a
|
||||
//! non-elevated process, but that's per-target, not a global switch, and
|
||||
//! there's no Settings pane to send users to.) So the Windows branch just
|
||||
//! returns `true`.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
/// Returns true when the current process is listed in Accessibility.
|
||||
/// No prompt side-effect.
|
||||
pub fn AXIsProcessTrusted() -> bool;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn is_trusted() -> bool {
|
||||
unsafe { ffi::AXIsProcessTrusted() }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn is_trusted() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn is_trusted() -> bool {
|
||||
false
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
use cpal::{SampleFormat, StreamConfig};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
/// Try to find a PulseAudio/PipeWire monitor source using `pactl`.
|
||||
/// Returns the source name (e.g. "alsa_output.pci-0000_0d_00.6.analog-stereo.monitor") if found.
|
||||
fn find_monitor_source_via_pactl() -> Option<String> {
|
||||
let output = std::process::Command::new("pactl")
|
||||
.args(["list", "short", "sources"])
|
||||
.output()
|
||||
.ok()?;
|
||||
|
||||
if !output.status.success() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
|
||||
// First, try to find the monitor of the default sink
|
||||
let default_sink = std::process::Command::new("pactl")
|
||||
.args(["get-default-sink"])
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| {
|
||||
if o.status.success() {
|
||||
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
// If we know the default sink, look for its .monitor specifically
|
||||
if let Some(sink_name) = &default_sink {
|
||||
let monitor_name = format!("{}.monitor", sink_name);
|
||||
for line in stdout.lines() {
|
||||
let parts: Vec<&str> = line.split('\t').collect();
|
||||
if parts.len() >= 2 && parts[1] == monitor_name {
|
||||
eprintln!(
|
||||
"Linux audio capture: Found default sink monitor via pactl: {}",
|
||||
monitor_name
|
||||
);
|
||||
return Some(monitor_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: find any .monitor source
|
||||
for line in stdout.lines() {
|
||||
let parts: Vec<&str> = line.split('\t').collect();
|
||||
if parts.len() >= 2 && parts[1].ends_with(".monitor") {
|
||||
let name = parts[1].to_string();
|
||||
eprintln!(
|
||||
"Linux audio capture: Found monitor source via pactl: {}",
|
||||
name
|
||||
);
|
||||
return Some(name);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Select the capture device: prefer an exact match against the monitor
|
||||
/// source name reported by `pactl`, then fall back to any device whose name
|
||||
/// contains "monitor", then the host's default input device.
|
||||
fn select_capture_device(host: &cpal::Host, monitor_source: Option<&str>) -> Option<cpal::Device> {
|
||||
let devices: Vec<cpal::Device> = host.input_devices().ok()?.collect();
|
||||
|
||||
if let Some(target) = monitor_source {
|
||||
if let Some(pos) = devices
|
||||
.iter()
|
||||
.position(|d| d.name().map(|n| n == target).unwrap_or(false))
|
||||
{
|
||||
eprintln!(
|
||||
"Linux audio capture: Using pactl monitor device: {}",
|
||||
target
|
||||
);
|
||||
return devices.into_iter().nth(pos);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(pos) = devices.iter().position(|d| {
|
||||
d.name()
|
||||
.map(|n| n.to_lowercase().contains("monitor"))
|
||||
.unwrap_or(false)
|
||||
}) {
|
||||
let name = devices[pos].name().unwrap_or_default();
|
||||
eprintln!("Linux audio capture: Found monitor device by name: {}", name);
|
||||
return devices.into_iter().nth(pos);
|
||||
}
|
||||
|
||||
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
||||
host.default_input_device()
|
||||
}
|
||||
|
||||
/// Start capturing system audio on Linux using PulseAudio monitor sources.
|
||||
///
|
||||
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
|
||||
/// monitor source via `pactl`, then select the matching cpal input device by
|
||||
/// name. This avoids mutating the process environment (`PULSE_SOURCE`), which
|
||||
/// is not thread-safe and would affect every thread in the process. If `pactl`
|
||||
/// is unavailable, we fall back to searching cpal device names for "monitor".
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let sample_rate_arc = state.sample_rate.clone();
|
||||
let channels_arc = state.channels.clone();
|
||||
let stop_tx = state.stop_tx.clone();
|
||||
let error_arc = state.error.clone();
|
||||
|
||||
// Use AtomicBool for stop signal (works across threads)
|
||||
let stop_flag = Arc::new(AtomicBool::new(false));
|
||||
let stop_flag_clone = stop_flag.clone();
|
||||
|
||||
// Create tokio channel and spawn a task to bridge it to the AtomicBool
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
|
||||
*stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
tokio::spawn(async move {
|
||||
rx.recv().await;
|
||||
stop_flag_clone.store(true, Ordering::Relaxed);
|
||||
});
|
||||
|
||||
// Spawn capture on a dedicated thread
|
||||
thread::spawn(move || {
|
||||
let host = cpal::default_host();
|
||||
let monitor_source = find_monitor_source_via_pactl();
|
||||
|
||||
let device = match select_capture_device(&host, monitor_source.as_deref()) {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
let error_msg = "No audio input device available".to_string();
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
||||
eprintln!("Linux audio capture: Using device: {}", device_name);
|
||||
|
||||
// Get supported config
|
||||
let config = match device.default_input_config() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get default input config: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let sample_rate = config.sample_rate().0;
|
||||
let channels = config.channels();
|
||||
let sample_format = config.sample_format();
|
||||
|
||||
eprintln!(
|
||||
"Linux audio capture: Config - {}Hz, {} channels, format: {:?}",
|
||||
sample_rate, channels, sample_format
|
||||
);
|
||||
|
||||
*sample_rate_arc.lock().unwrap() = sample_rate;
|
||||
*channels_arc.lock().unwrap() = channels;
|
||||
|
||||
let stream_config = StreamConfig {
|
||||
channels,
|
||||
sample_rate: cpal::SampleRate(sample_rate),
|
||||
buffer_size: cpal::BufferSize::Default,
|
||||
};
|
||||
|
||||
let samples_clone = samples.clone();
|
||||
let error_arc_clone = error_arc.clone();
|
||||
let stop_flag_for_stream = stop_flag.clone();
|
||||
|
||||
let err_fn = {
|
||||
let error_arc = error_arc.clone();
|
||||
move |err: cpal::StreamError| {
|
||||
let error_msg = format!("Stream error: {}", err);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
}
|
||||
};
|
||||
|
||||
let stream = match sample_format {
|
||||
SampleFormat::F32 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[f32], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
guard.extend_from_slice(data);
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SampleFormat::I16 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[i16], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
for &s in data {
|
||||
guard.push(s as f32 / 32768.0);
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
SampleFormat::U16 => {
|
||||
let samples = samples_clone.clone();
|
||||
let stop = stop_flag_for_stream.clone();
|
||||
device.build_input_stream(
|
||||
&stream_config,
|
||||
move |data: &[u16], _: &cpal::InputCallbackInfo| {
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let mut guard = samples.lock().unwrap();
|
||||
for &s in data {
|
||||
guard.push((s as f32 / 32768.0) - 1.0);
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
let error_msg = format!("Unsupported sample format: {:?}", sample_format);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let stream = match stream {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to build input stream: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = stream.play() {
|
||||
let error_msg = format!("Failed to start stream: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc_clone.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
|
||||
eprintln!("Linux audio capture: Stream started successfully");
|
||||
|
||||
// Keep thread alive until stop signal
|
||||
loop {
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
|
||||
// Stream will be dropped here, stopping capture
|
||||
eprintln!("Linux audio capture: Stream stopped");
|
||||
});
|
||||
|
||||
// Spawn timeout task
|
||||
let stop_tx_clone = state.stop_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
|
||||
let tx = stop_tx_clone.lock().unwrap().take();
|
||||
if let Some(tx) = tx {
|
||||
let _ = tx.send(()).await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Check if there was an error during capture
|
||||
if let Some(error) = state.error.lock().unwrap().as_ref() {
|
||||
return Err(error.clone());
|
||||
}
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err(
|
||||
"No audio samples captured. Make sure audio is playing on your system during recording."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
// Check via pactl first (most reliable on modern Linux)
|
||||
if find_monitor_source_via_pactl().is_some() {
|
||||
return true;
|
||||
}
|
||||
// Fallback: check cpal devices
|
||||
let host = cpal::default_host();
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for d in devices {
|
||||
if let Ok(name) = d.name() {
|
||||
if name.to_lowercase().contains("monitor") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
host.default_input_device().is_some()
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer =
|
||||
WavWriter::new(cursor, spec).map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer
|
||||
.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer
|
||||
.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use screencapturekit::{
|
||||
cm::CMSampleBuffer,
|
||||
shareable_content::SCShareableContent,
|
||||
stream::{
|
||||
configuration::SCStreamConfiguration,
|
||||
content_filter::SCContentFilter,
|
||||
output_trait::SCStreamOutputTrait,
|
||||
output_type::SCStreamOutputType,
|
||||
sc_stream::SCStream,
|
||||
},
|
||||
};
|
||||
use std::io::Cursor;
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
if !is_supported() {
|
||||
return Err("System audio capture requires macOS 12.3 or newer.".to_string());
|
||||
}
|
||||
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
// Get shareable content
|
||||
let content = SCShareableContent::get()
|
||||
.map_err(|e| format!("Failed to get shareable content: {}", e))?;
|
||||
|
||||
// Get first display
|
||||
let displays = content.displays();
|
||||
if displays.is_empty() {
|
||||
return Err("No displays available".to_string());
|
||||
}
|
||||
let display = &displays[0];
|
||||
|
||||
// Create content filter for desktop audio
|
||||
let filter = SCContentFilter::create()
|
||||
.with_display(display)
|
||||
.with_excluding_windows(&[])
|
||||
.build();
|
||||
|
||||
// Create stream configuration - audio only
|
||||
let mut config = SCStreamConfiguration::default();
|
||||
config.set_captures_audio(true);
|
||||
config.set_excludes_current_process_audio(false);
|
||||
config.set_sample_rate(48000); // Use i32 directly
|
||||
config.set_channel_count(2); // Use i32 directly
|
||||
|
||||
// Create stream using builder
|
||||
let (tx, mut rx) = mpsc::channel::<()>(1);
|
||||
*state.stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let sample_rate = state.sample_rate.clone();
|
||||
let channels = state.channels.clone();
|
||||
|
||||
// Set sample rate and channels
|
||||
*sample_rate.lock().unwrap() = 48000;
|
||||
*channels.lock().unwrap() = 2;
|
||||
|
||||
// Create output handler struct
|
||||
struct AudioHandler {
|
||||
samples: Arc<Mutex<Vec<f32>>>,
|
||||
}
|
||||
|
||||
impl SCStreamOutputTrait for AudioHandler {
|
||||
fn did_output_sample_buffer(
|
||||
&self,
|
||||
sample: CMSampleBuffer,
|
||||
_type: SCStreamOutputType,
|
||||
) {
|
||||
if _type == SCStreamOutputType::Audio {
|
||||
if let Ok(audio_samples) = extract_audio_samples(sample) {
|
||||
let mut samples_guard = self.samples.lock().unwrap();
|
||||
samples_guard.extend_from_slice(&audio_samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let handler = AudioHandler {
|
||||
samples: samples.clone(),
|
||||
};
|
||||
|
||||
// Create stream
|
||||
let mut stream = SCStream::new(&filter, &config);
|
||||
|
||||
// Add output handler for audio (order: handler, then output_type)
|
||||
stream.add_output_handler(handler, SCStreamOutputType::Audio);
|
||||
|
||||
// Store stream reference
|
||||
*state.stream.lock().unwrap() = Some(stream.clone());
|
||||
|
||||
stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?;
|
||||
|
||||
// Spawn task to stop after max duration
|
||||
let stream_clone = stream.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => {
|
||||
// Timeout reached
|
||||
}
|
||||
_ = rx.recv() => {
|
||||
// Manual stop
|
||||
}
|
||||
}
|
||||
let _ = stream_clone.stop_capture();
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Stop stream if still active
|
||||
if let Some(stream) = state.stream.lock().unwrap().take() {
|
||||
let _ = stream.stop_capture();
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err("No audio samples captured".to_string());
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
macos_version_at_least(12, 3)
|
||||
}
|
||||
|
||||
fn macos_version_at_least(required_major: u64, required_minor: u64) -> bool {
|
||||
let output = match Command::new("sw_vers").arg("-productVersion").output() {
|
||||
Ok(output) if output.status.success() => output,
|
||||
_ => return false,
|
||||
};
|
||||
|
||||
let version = String::from_utf8_lossy(&output.stdout);
|
||||
let mut parts = version.trim().split('.');
|
||||
|
||||
let major = parts.next().and_then(|part| part.parse::<u64>().ok()).unwrap_or(0);
|
||||
let minor = parts.next().and_then(|part| part.parse::<u64>().ok()).unwrap_or(0);
|
||||
|
||||
major > required_major || (major == required_major && minor >= required_minor)
|
||||
}
|
||||
|
||||
fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, String> {
|
||||
// Use the crate's built-in method to get audio buffer list
|
||||
let audio_buffer_list = sample_buffer
|
||||
.audio_buffer_list()
|
||||
.ok_or_else(|| "Failed to get audio buffer list".to_string())?;
|
||||
|
||||
let buffers: Vec<_> = audio_buffer_list.iter().collect();
|
||||
let num_buffers = buffers.len();
|
||||
|
||||
if num_buffers == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
// ScreenCaptureKit on macOS provides audio in Float32 format
|
||||
// The audio can be either:
|
||||
// - Interleaved (1 buffer with L,R,L,R,... samples)
|
||||
// - Planar (2 buffers, one for L channel, one for R channel)
|
||||
|
||||
if num_buffers == 1 {
|
||||
// Interleaved stereo or mono in a single buffer
|
||||
let buffer = &buffers[0];
|
||||
let data_bytes = buffer.data();
|
||||
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
|
||||
|
||||
if num_samples > 0 {
|
||||
unsafe {
|
||||
let data_ptr = data_bytes.as_ptr() as *const f32;
|
||||
let data = std::slice::from_raw_parts(data_ptr, num_samples);
|
||||
return Ok(data.to_vec());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Planar format - separate buffer for each channel
|
||||
// We need to interleave them: L0, R0, L1, R1, ...
|
||||
let mut channel_data: Vec<Vec<f32>> = Vec::new();
|
||||
let mut max_samples = 0;
|
||||
|
||||
for buffer in &buffers {
|
||||
let data_bytes = buffer.data();
|
||||
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
|
||||
|
||||
if num_samples > 0 {
|
||||
unsafe {
|
||||
let data_ptr = data_bytes.as_ptr() as *const f32;
|
||||
let data = std::slice::from_raw_parts(data_ptr, num_samples);
|
||||
channel_data.push(data.to_vec());
|
||||
max_samples = max_samples.max(num_samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Interleave the channels
|
||||
let mut interleaved = Vec::with_capacity(max_samples * num_buffers);
|
||||
for i in 0..max_samples {
|
||||
for channel in &channel_data {
|
||||
if i < channel.len() {
|
||||
interleaved.push(channel[i]);
|
||||
} else {
|
||||
interleaved.push(0.0); // Pad with silence if needed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(interleaved);
|
||||
}
|
||||
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer = WavWriter::new(cursor, spec)
|
||||
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub use macos::*;
|
||||
#[cfg(target_os = "windows")]
|
||||
pub use windows::*;
|
||||
#[cfg(target_os = "linux")]
|
||||
pub use linux::*;
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use screencapturekit::stream::sc_stream::SCStream;
|
||||
|
||||
pub struct AudioCaptureState {
|
||||
pub samples: Arc<Mutex<Vec<f32>>>,
|
||||
pub sample_rate: Arc<Mutex<u32>>,
|
||||
pub channels: Arc<Mutex<u16>>,
|
||||
pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
|
||||
pub error: Arc<Mutex<Option<String>>>,
|
||||
#[cfg(target_os = "macos")]
|
||||
pub stream: Arc<Mutex<Option<SCStream>>>,
|
||||
}
|
||||
|
||||
impl AudioCaptureState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
samples: Arc::new(Mutex::new(Vec::new())),
|
||||
sample_rate: Arc::new(Mutex::new(44100)),
|
||||
channels: Arc::new(Mutex::new(2)),
|
||||
stop_tx: Arc::new(Mutex::new(None)),
|
||||
error: Arc::new(Mutex::new(None)),
|
||||
#[cfg(target_os = "macos")]
|
||||
stream: Arc::new(Mutex::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reset(&self) {
|
||||
*self.samples.lock().unwrap() = Vec::new();
|
||||
*self.error.lock().unwrap() = None;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
use crate::audio_capture::AudioCaptureState;
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use hound::{WavSpec, WavWriter};
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
use wasapi::*;
|
||||
use windows::Win32::System::Com::{CoInitializeEx, CoUninitialize, COINIT_MULTITHREADED};
|
||||
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
) -> Result<(), String> {
|
||||
// Reset previous samples
|
||||
state.reset();
|
||||
|
||||
let samples = state.samples.clone();
|
||||
let sample_rate_arc = state.sample_rate.clone();
|
||||
let channels_arc = state.channels.clone();
|
||||
let stop_tx = state.stop_tx.clone();
|
||||
let error_arc = state.error.clone();
|
||||
|
||||
// Use AtomicBool for stop signal (works with non-Send types)
|
||||
let stop_flag = Arc::new(AtomicBool::new(false));
|
||||
let stop_flag_clone = stop_flag.clone();
|
||||
|
||||
// Create tokio channel and spawn a task to bridge it to the AtomicBool
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(1);
|
||||
*stop_tx.lock().unwrap() = Some(tx);
|
||||
|
||||
tokio::spawn(async move {
|
||||
rx.recv().await;
|
||||
stop_flag_clone.store(true, Ordering::Relaxed);
|
||||
});
|
||||
|
||||
// Spawn capture task on a dedicated thread (WASAPI COM objects are not Send)
|
||||
// All WASAPI objects must be created and used on the same thread
|
||||
thread::spawn(move || {
|
||||
// Initialize COM for this thread
|
||||
unsafe {
|
||||
let hr = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
if hr.is_err() {
|
||||
eprintln!("Failed to initialize COM: {:?}", hr);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure COM is uninitialized when thread exits
|
||||
let _com_guard = scopeguard::guard((), |_| unsafe {
|
||||
CoUninitialize();
|
||||
});
|
||||
|
||||
// Initialize WASAPI on this thread
|
||||
let device = match DeviceEnumerator::new()
|
||||
.and_then(|enumerator| enumerator.get_default_device(&Direction::Render))
|
||||
{
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get audio device: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut audio_client = match device.get_iaudioclient() {
|
||||
Ok(client) => client,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get audio client: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mix_format = match audio_client.get_mixformat() {
|
||||
Ok(format) => format,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get mix format: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Set sample rate and channels
|
||||
let channels = mix_format.get_nchannels() as usize;
|
||||
let bytes_per_sample = (mix_format.get_bitspersample() / 8) as usize;
|
||||
*sample_rate_arc.lock().unwrap() = mix_format.get_samplespersec();
|
||||
*channels_arc.lock().unwrap() = mix_format.get_nchannels();
|
||||
|
||||
// Get device period
|
||||
let (_def_period, min_period) = match audio_client.get_device_period() {
|
||||
Ok(periods) => periods,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to get device period: {}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize audio client for loopback with StreamMode
|
||||
// For loopback mode: get Render device, initialize with Capture direction
|
||||
// This triggers AUDCLNT_STREAMFLAGS_LOOPBACK in the wasapi crate
|
||||
let stream_mode = StreamMode::EventsShared {
|
||||
autoconvert: true, // Enable automatic format conversion
|
||||
buffer_duration_hns: min_period, // Use minimum period
|
||||
};
|
||||
|
||||
if let Err(e) = audio_client.initialize_client(&mix_format, &Direction::Capture, &stream_mode) {
|
||||
let error_msg = format!("Failed to initialize audio client: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Set up event handle for EventsShared mode
|
||||
let h_event = match audio_client.set_get_eventhandle() {
|
||||
Ok(event) => event,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to set event handle: {}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let capture_client = match audio_client.get_audiocaptureclient() {
|
||||
Ok(client) => client,
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to get capture client: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = audio_client.start_stream() {
|
||||
let error_msg = format!("Failed to start stream: {}", e);
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
// Check if stop signal was received
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Try to get available data
|
||||
match capture_client.get_next_packet_size() {
|
||||
Ok(Some(frames_available)) => {
|
||||
if frames_available > 0 {
|
||||
// Calculate buffer size needed (frames * channels * bytes_per_sample)
|
||||
let buffer_size = frames_available as usize * channels * bytes_per_sample;
|
||||
|
||||
let mut buffer = vec![0u8; buffer_size];
|
||||
match capture_client.read_from_device(&mut buffer) {
|
||||
Ok((frames_read, _buffer_info)) => {
|
||||
if frames_read > 0 {
|
||||
// Convert bytes to f32 samples
|
||||
let samples_read = (frames_read as usize * channels) as usize;
|
||||
let mut samples_guard = samples.lock().unwrap();
|
||||
|
||||
// Assuming 32-bit float format
|
||||
if bytes_per_sample == 4 {
|
||||
for i in 0..samples_read {
|
||||
let byte_offset = i * 4;
|
||||
if byte_offset + 4 <= buffer.len() {
|
||||
let sample = f32::from_le_bytes([
|
||||
buffer[byte_offset],
|
||||
buffer[byte_offset + 1],
|
||||
buffer[byte_offset + 2],
|
||||
buffer[byte_offset + 3],
|
||||
]);
|
||||
samples_guard.push(sample);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error reading from device: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// Exclusive mode - handle differently if needed
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error getting next packet size: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for event signal (with timeout to allow checking stop flag)
|
||||
if h_event.wait_for_event(100).is_err() {
|
||||
// Timeout is expected - just continue to check stop flag
|
||||
}
|
||||
}
|
||||
|
||||
// Stop the stream when done
|
||||
audio_client.stop_stream().ok();
|
||||
});
|
||||
|
||||
// Spawn timeout task
|
||||
let stop_tx_clone = state.stop_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
|
||||
// Take the sender out of the mutex before awaiting
|
||||
let tx = stop_tx_clone.lock().unwrap().take();
|
||||
if let Some(tx) = tx {
|
||||
let _ = tx.send(()).await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
|
||||
// Signal stop
|
||||
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
|
||||
let _ = tx.send(());
|
||||
}
|
||||
|
||||
// Wait a bit for capture to stop
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
// Check if there was an error during capture
|
||||
if let Some(error) = state.error.lock().unwrap().as_ref() {
|
||||
return Err(error.clone());
|
||||
}
|
||||
|
||||
// Get samples
|
||||
let samples = state.samples.lock().unwrap().clone();
|
||||
let sample_rate = *state.sample_rate.lock().unwrap();
|
||||
let channels = *state.channels.lock().unwrap();
|
||||
|
||||
if samples.is_empty() {
|
||||
return Err("No audio samples captured. Make sure audio is playing on your system during recording.".to_string());
|
||||
}
|
||||
|
||||
// Convert to WAV
|
||||
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
|
||||
|
||||
// Encode to base64
|
||||
let base64_data = general_purpose::STANDARD.encode(&wav_data);
|
||||
|
||||
Ok(base64_data)
|
||||
}
|
||||
|
||||
pub fn is_supported() -> bool {
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
true
|
||||
}
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
|
||||
let mut buffer = Vec::new();
|
||||
let cursor = Cursor::new(&mut buffer);
|
||||
|
||||
let spec = WavSpec {
|
||||
channels,
|
||||
sample_rate,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
|
||||
let mut writer = WavWriter::new(cursor, spec)
|
||||
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
|
||||
|
||||
// Convert f32 samples to i16
|
||||
for sample in samples {
|
||||
let clamped = sample.clamp(-1.0, 1.0);
|
||||
let i16_sample = (clamped * 32767.0) as i16;
|
||||
writer.write_sample(i16_sample)
|
||||
.map_err(|e| format!("Failed to write sample: {}", e))?;
|
||||
}
|
||||
|
||||
writer.finalize()
|
||||
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
|
||||
|
||||
Ok(buffer)
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
use cpal::{Device, Host, SampleFormat, StreamConfig};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct AudioOutputDevice {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
pub struct AudioOutputState {
|
||||
host: Host,
|
||||
stop_flag: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl AudioOutputState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
host: cpal::default_host(),
|
||||
stop_flag: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stop_all_playback(&self) -> Result<(), String> {
|
||||
eprintln!("stop_all_playback: Setting stop flag");
|
||||
self.stop_flag.store(true, Ordering::Relaxed);
|
||||
eprintln!("stop_all_playback: Stop flag set - active streams will output silence");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_output_devices(&self) -> Result<Vec<AudioOutputDevice>, String> {
|
||||
let devices = self
|
||||
.host
|
||||
.output_devices()
|
||||
.map_err(|e| format!("Failed to enumerate output devices: {}", e))?;
|
||||
|
||||
let default_device = self.host.default_output_device();
|
||||
|
||||
let mut result = Vec::new();
|
||||
for device in devices {
|
||||
let name = device
|
||||
.name()
|
||||
.map_err(|e| format!("Failed to get device name: {}", e))?;
|
||||
|
||||
// Generate a stable ID from the device name (cpal doesn't provide stable IDs)
|
||||
let id = format!("device_{}", name.replace(' ', "_").to_lowercase());
|
||||
|
||||
let is_default = default_device
|
||||
.as_ref()
|
||||
.map(|d| d.name().unwrap_or_default() == name)
|
||||
.unwrap_or(false);
|
||||
|
||||
result.push(AudioOutputDevice {
|
||||
id,
|
||||
name,
|
||||
is_default,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn play_audio_to_devices(
|
||||
&self,
|
||||
audio_data: Vec<u8>,
|
||||
device_ids: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("play_audio_to_devices called with {} bytes, {} device IDs", audio_data.len(), device_ids.len());
|
||||
eprintln!("Requested device IDs: {:?}", device_ids);
|
||||
|
||||
// Decode audio file (assuming WAV format)
|
||||
eprintln!("Decoding audio data...");
|
||||
let (samples, sample_rate, channels) = self.decode_wav(&audio_data)?;
|
||||
eprintln!("Audio decoded: {} samples, {}Hz, {} channels", samples.len(), sample_rate, channels);
|
||||
|
||||
// Find devices by ID
|
||||
eprintln!("Enumerating output devices...");
|
||||
let devices: Vec<Device> = self
|
||||
.host
|
||||
.output_devices()
|
||||
.map_err(|e| format!("Failed to enumerate devices: {}", e))?
|
||||
.filter_map(|device| {
|
||||
let name = device.name().ok()?;
|
||||
let id = format!("device_{}", name.replace(' ', "_").to_lowercase());
|
||||
eprintln!("Found device: {} (id: {})", name, id);
|
||||
if device_ids.contains(&id) {
|
||||
eprintln!(" -> Matched! Will play to this device");
|
||||
Some(device)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
if devices.is_empty() {
|
||||
eprintln!("ERROR: No matching devices found");
|
||||
return Err("No matching devices found".to_string());
|
||||
}
|
||||
|
||||
eprintln!("Playing to {} device(s)", devices.len());
|
||||
|
||||
// Stop any existing playback first
|
||||
self.stop_all_playback().ok();
|
||||
|
||||
// Reset stop flag for new playback
|
||||
self.stop_flag.store(false, Ordering::Relaxed);
|
||||
|
||||
// Play to each device
|
||||
for (i, device) in devices.iter().enumerate() {
|
||||
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
||||
eprintln!("Playing to device {}/{}: {}", i + 1, devices.len(), device_name);
|
||||
self.play_to_device(device, samples.clone(), sample_rate, channels, self.stop_flag.clone())
|
||||
.map_err(|e| format!("Failed to play to device {}: {}", device_name, e))?;
|
||||
eprintln!("Successfully started playback on device: {}", device_name);
|
||||
}
|
||||
|
||||
eprintln!("play_audio_to_devices completed successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_wav(&self, data: &[u8]) -> Result<(Vec<f32>, u32, u16), String> {
|
||||
use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::MediaSourceStream;
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
|
||||
eprintln!("decode_wav: Creating MediaSourceStream from {} bytes", data.len());
|
||||
let mss = MediaSourceStream::new(
|
||||
Box::new(std::io::Cursor::new(data.to_vec())),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
eprintln!("decode_wav: Probing audio format...");
|
||||
let mut format = symphonia::default::get_probe()
|
||||
.format(
|
||||
&Default::default(),
|
||||
mss,
|
||||
&FormatOptions::default(),
|
||||
&MetadataOptions::default(),
|
||||
)
|
||||
.map_err(|e| {
|
||||
eprintln!("decode_wav: Failed to probe audio: {}", e);
|
||||
format!("Failed to probe audio: {}", e)
|
||||
})?
|
||||
.format;
|
||||
|
||||
eprintln!("decode_wav: Audio format probed successfully");
|
||||
|
||||
eprintln!("decode_wav: Finding audio track...");
|
||||
let track = format
|
||||
.tracks()
|
||||
.iter()
|
||||
.find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL)
|
||||
.ok_or_else(|| {
|
||||
eprintln!("decode_wav: No audio track found");
|
||||
"No audio track found".to_string()
|
||||
})?;
|
||||
|
||||
let sample_rate = track
|
||||
.codec_params
|
||||
.sample_rate
|
||||
.ok_or_else(|| {
|
||||
eprintln!("decode_wav: No sample rate found in track");
|
||||
"No sample rate found".to_string()
|
||||
})?;
|
||||
|
||||
let channels = track
|
||||
.codec_params
|
||||
.channels
|
||||
.ok_or_else(|| {
|
||||
eprintln!("decode_wav: No channels found in track");
|
||||
"No channels found".to_string()
|
||||
})?
|
||||
.count() as u16;
|
||||
|
||||
eprintln!("decode_wav: Track info - sample_rate: {}, channels: {}", sample_rate, channels);
|
||||
|
||||
eprintln!("decode_wav: Creating decoder...");
|
||||
let mut decoder = symphonia::default::get_codecs()
|
||||
.make(&track.codec_params, &Default::default())
|
||||
.map_err(|e| {
|
||||
eprintln!("decode_wav: Failed to create decoder: {}", e);
|
||||
format!("Failed to create decoder: {}", e)
|
||||
})?;
|
||||
|
||||
eprintln!("decode_wav: Decoder created successfully");
|
||||
|
||||
let mut samples = Vec::new();
|
||||
let mut packet_count = 0;
|
||||
eprintln!("decode_wav: Starting packet decoding loop...");
|
||||
loop {
|
||||
let packet = match format.next_packet() {
|
||||
Ok(packet) => packet,
|
||||
Err(e) => {
|
||||
eprintln!("decode_wav: End of stream or error: {:?}", e);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
packet_count += 1;
|
||||
let decoded = decoder
|
||||
.decode(&packet)
|
||||
.map_err(|e| {
|
||||
eprintln!("decode_wav: Decode error on packet {}: {}", packet_count, e);
|
||||
format!("Decode error: {}", e)
|
||||
})?;
|
||||
|
||||
// Convert to f32 samples by matching on the buffer type
|
||||
use symphonia::core::audio::{AudioBufferRef, Signal};
|
||||
use symphonia::core::conv::FromSample;
|
||||
|
||||
let spec = *decoded.spec();
|
||||
let num_channels = spec.channels.count();
|
||||
let num_frames = decoded.frames();
|
||||
|
||||
eprintln!("decode_wav: Packet {} - {} frames, {} channels", packet_count, num_frames, num_channels);
|
||||
|
||||
// Interleave samples from all channels
|
||||
for frame_idx in 0..num_frames {
|
||||
for ch in 0..num_channels {
|
||||
let sample_f32 = match &decoded {
|
||||
AudioBufferRef::U8(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::U16(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::U24(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::U32(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::S8(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::S16(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::S24(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::S32(buf) => f32::from_sample(buf.chan(ch)[frame_idx]),
|
||||
AudioBufferRef::F32(buf) => buf.chan(ch)[frame_idx],
|
||||
AudioBufferRef::F64(buf) => buf.chan(ch)[frame_idx] as f32,
|
||||
};
|
||||
samples.push(sample_f32);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("decode_wav: Decoded {} packets, total {} samples", packet_count, samples.len());
|
||||
eprintln!("decode_wav: Returning sample_rate={}, channels={}", sample_rate, channels);
|
||||
Ok((samples, sample_rate, channels))
|
||||
}
|
||||
|
||||
fn play_to_device(
|
||||
&self,
|
||||
device: &Device,
|
||||
samples: Vec<f32>,
|
||||
sample_rate: u32,
|
||||
channels: u16,
|
||||
stop_flag: Arc<AtomicBool>,
|
||||
) -> Result<(), String> {
|
||||
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
||||
eprintln!("play_to_device: Starting playback to device: {}", device_name);
|
||||
eprintln!("play_to_device: Input - {} samples, {}Hz, {} channels", samples.len(), sample_rate, channels);
|
||||
|
||||
let config = device
|
||||
.default_output_config()
|
||||
.map_err(|e| format!("Failed to get default config: {}", e))?;
|
||||
|
||||
// Prepare samples for the device's format
|
||||
let device_sample_rate = config.sample_rate().0;
|
||||
let device_channels = config.channels();
|
||||
let device_sample_format = config.sample_format();
|
||||
|
||||
eprintln!("play_to_device: Device config - {}Hz, {} channels, format: {:?}",
|
||||
device_sample_rate, device_channels, device_sample_format);
|
||||
|
||||
// Resample if needed (simple linear interpolation for now)
|
||||
let resampled = if device_sample_rate != sample_rate {
|
||||
eprintln!("play_to_device: Resampling from {}Hz to {}Hz", sample_rate, device_sample_rate);
|
||||
let result = self.resample(&samples, sample_rate, device_sample_rate);
|
||||
eprintln!("play_to_device: Resampled {} samples to {} samples", samples.len(), result.len());
|
||||
result
|
||||
} else {
|
||||
eprintln!("play_to_device: No resampling needed");
|
||||
samples
|
||||
};
|
||||
|
||||
// Interleave/convert channels if needed
|
||||
eprintln!("play_to_device: Interleaving channels from {} to {} channels", channels, device_channels);
|
||||
let interleaved = self.interleave_channels(&resampled, channels, device_channels);
|
||||
eprintln!("play_to_device: Interleaved to {} samples", interleaved.len());
|
||||
|
||||
// Create shared buffer for playback
|
||||
let buffer: Arc<Mutex<Vec<f32>>> = Arc::new(Mutex::new(interleaved));
|
||||
let position = Arc::new(AtomicUsize::new(0));
|
||||
let buffer_clone = buffer.clone();
|
||||
let position_clone = position.clone();
|
||||
|
||||
let err_fn = |err| eprintln!("Playback error: {}", err);
|
||||
|
||||
let stream_config = StreamConfig {
|
||||
channels: device_channels,
|
||||
sample_rate: cpal::SampleRate(device_sample_rate),
|
||||
buffer_size: cpal::BufferSize::Default,
|
||||
};
|
||||
|
||||
let stop_flag_clone = stop_flag.clone();
|
||||
let stream = match config.sample_format() {
|
||||
SampleFormat::F32 => {
|
||||
let buffer = buffer_clone.clone();
|
||||
let pos = position_clone.clone();
|
||||
device
|
||||
.build_output_stream(
|
||||
&stream_config,
|
||||
move |data: &mut [f32], _: &cpal::OutputCallbackInfo| {
|
||||
// Check stop flag - if set, output silence
|
||||
if stop_flag_clone.load(Ordering::Relaxed) {
|
||||
for sample in data.iter_mut() {
|
||||
*sample = 0.0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mut idx = pos.load(Ordering::Relaxed);
|
||||
let buf = buffer.lock().unwrap();
|
||||
for sample in data.iter_mut() {
|
||||
if idx < buf.len() {
|
||||
*sample = buf[idx];
|
||||
idx += 1;
|
||||
} else {
|
||||
*sample = 0.0;
|
||||
}
|
||||
}
|
||||
pos.store(idx, Ordering::Relaxed);
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
.map_err(|e| format!("Failed to build stream: {}", e))?
|
||||
}
|
||||
SampleFormat::I16 => {
|
||||
let buffer = buffer_clone.clone();
|
||||
let pos = position_clone.clone();
|
||||
device
|
||||
.build_output_stream(
|
||||
&stream_config,
|
||||
move |data: &mut [i16], _: &cpal::OutputCallbackInfo| {
|
||||
// Check stop flag - if set, output silence
|
||||
if stop_flag_clone.load(Ordering::Relaxed) {
|
||||
for sample in data.iter_mut() {
|
||||
*sample = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mut idx = pos.load(Ordering::Relaxed);
|
||||
let buf = buffer.lock().unwrap();
|
||||
for sample in data.iter_mut() {
|
||||
if idx < buf.len() {
|
||||
*sample = (buf[idx] * 32767.0) as i16;
|
||||
idx += 1;
|
||||
} else {
|
||||
*sample = 0;
|
||||
}
|
||||
}
|
||||
pos.store(idx, Ordering::Relaxed);
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
.map_err(|e| format!("Failed to build stream: {}", e))?
|
||||
}
|
||||
SampleFormat::U16 => {
|
||||
let buffer = buffer_clone.clone();
|
||||
let pos = position_clone.clone();
|
||||
device
|
||||
.build_output_stream(
|
||||
&stream_config,
|
||||
move |data: &mut [u16], _: &cpal::OutputCallbackInfo| {
|
||||
// Check stop flag - if set, output silence
|
||||
if stop_flag_clone.load(Ordering::Relaxed) {
|
||||
for sample in data.iter_mut() {
|
||||
*sample = 32768;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mut idx = pos.load(Ordering::Relaxed);
|
||||
let buf = buffer.lock().unwrap();
|
||||
for sample in data.iter_mut() {
|
||||
if idx < buf.len() {
|
||||
*sample = ((buf[idx] + 1.0) * 32767.5) as u16;
|
||||
idx += 1;
|
||||
} else {
|
||||
*sample = 32768;
|
||||
}
|
||||
}
|
||||
pos.store(idx, Ordering::Relaxed);
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)
|
||||
.map_err(|e| format!("Failed to build stream: {}", e))?
|
||||
}
|
||||
_ => return Err("Unsupported sample format".to_string()),
|
||||
};
|
||||
|
||||
eprintln!("play_to_device: Starting stream playback...");
|
||||
stream.play().map_err(|e| {
|
||||
eprintln!("play_to_device: Failed to play stream: {}", e);
|
||||
format!("Failed to play stream: {}", e)
|
||||
})?;
|
||||
|
||||
eprintln!("play_to_device: Stream started successfully");
|
||||
|
||||
// Keep the stream alive until playback finishes.
|
||||
// Previously the stream was dropped immediately on function return,
|
||||
// causing silent playback (cpal stops output when its Stream is dropped).
|
||||
let total_samples = {
|
||||
buffer.lock().unwrap().len()
|
||||
};
|
||||
loop {
|
||||
let pos = position.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if pos >= total_samples || stop_flag.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
|
||||
// stream is dropped here, after audio has finished playing
|
||||
drop(stream);
|
||||
eprintln!("play_to_device: Function completed successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resample(&self, samples: &[f32], from_rate: u32, to_rate: u32) -> Vec<f32> {
|
||||
if from_rate == to_rate {
|
||||
return samples.to_vec();
|
||||
}
|
||||
|
||||
let ratio = to_rate as f64 / from_rate as f64;
|
||||
let new_len = (samples.len() as f64 * ratio) as usize;
|
||||
let mut resampled = Vec::with_capacity(new_len);
|
||||
|
||||
for i in 0..new_len {
|
||||
let src_idx = (i as f64 / ratio) as usize;
|
||||
if src_idx < samples.len() {
|
||||
resampled.push(samples[src_idx]);
|
||||
} else {
|
||||
resampled.push(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
resampled
|
||||
}
|
||||
|
||||
fn interleave_channels(
|
||||
&self,
|
||||
samples: &[f32],
|
||||
src_channels: u16,
|
||||
dst_channels: u16,
|
||||
) -> Vec<f32> {
|
||||
if src_channels == dst_channels {
|
||||
return samples.to_vec();
|
||||
}
|
||||
|
||||
let mut interleaved = Vec::new();
|
||||
let samples_per_channel = samples.len() / src_channels as usize;
|
||||
|
||||
for i in 0..samples_per_channel {
|
||||
for ch in 0..dst_channels {
|
||||
let src_ch = if ch < src_channels { ch } else { src_channels - 1 };
|
||||
let idx = (i * src_channels as usize) + src_ch as usize;
|
||||
if idx < samples.len() {
|
||||
interleaved.push(samples[idx]);
|
||||
} else {
|
||||
interleaved.push(0.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interleaved
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AudioOutputState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
//! Snapshot / write / restore helpers around the system clipboard.
|
||||
//!
|
||||
//! Used by the auto-paste flow: before synthesising the paste accelerator
|
||||
//! into a foreign app we need to (1) remember what the user had on the
|
||||
//! clipboard, (2) stage our transcribed text, (3) paste, (4) put the
|
||||
//! original contents back. Missing step 4 turns every dictation into a
|
||||
//! silent clipboard-stomp.
|
||||
//!
|
||||
//! On **macOS** the snapshot walks `NSPasteboard.pasteboardItems` and
|
||||
//! copies every `(UTI, data)` pair into an owned `Vec<u8>`, so restore
|
||||
//! rebuilds the full multi-type payload — not just the plain-text
|
||||
//! fallback. Images, styled text, file-reference lists all survive the
|
||||
//! round-trip.
|
||||
//!
|
||||
//! On **Windows** the snapshot walks `EnumClipboardFormats` and copies the
|
||||
//! HGLOBAL payload for every advertised format. GDI-handle formats (DIB
|
||||
//! bitmap, metafile, enhanced metafile, palette), owner-display variants,
|
||||
//! and the private-/GDI-object format ranges are skipped — those can't be
|
||||
//! round-tripped across processes without synthesising the underlying
|
||||
//! kernel/GDI objects, which isn't worth the complexity for a dictation
|
||||
//! clipboard guard. CF_UNICODETEXT, CF_HDROP, CF_DIB (bitmap data in
|
||||
//! memory, not a handle), CF_DIBV5, and every registered format (HTML
|
||||
//! Format, Rich Text Format, FileGroupDescriptor, etc.) all survive.
|
||||
//!
|
||||
//! On **macOS** every entry point manages its own `NSAutoreleasePool`
|
||||
//! because the Tauri command runtime threads don't have one by default —
|
||||
//! without it, every autoreleased `NSString` / `NSData` we touch would
|
||||
//! leak for the life of the process. On Windows, HGLOBAL ownership
|
||||
//! transfers to the clipboard on `SetClipboardData` success, so we only
|
||||
//! free handles we allocated but didn't hand off.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::runtime::Object;
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::{class, msg_send, sel, sel_impl};
|
||||
|
||||
/// One full-fidelity snapshot of the general pasteboard. Hold on to the value
|
||||
/// until the paste has landed, then pass it to [`restore_clipboard`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClipboardSnapshot {
|
||||
/// Outer vec: pasteboard items. Inner: `(uti, raw bytes)` per type. We
|
||||
/// store the raw UTI string and the raw `NSData` payload so we can rebuild
|
||||
/// the item with `setData:forType:` without interpreting the contents.
|
||||
items: Vec<Vec<(String, Vec<u8>)>>,
|
||||
/// `NSPasteboard.changeCount` at the moment of capture. Incremented by AppKit
|
||||
/// on every mutation from any process, so a caller can decide whether a
|
||||
/// restore is still safe (change_count == expected) or whether someone
|
||||
/// else wrote to the clipboard in the interim and we should back off.
|
||||
change_count: i64,
|
||||
}
|
||||
|
||||
impl ClipboardSnapshot {
|
||||
pub fn change_count(&self) -> i64 {
|
||||
self.change_count
|
||||
}
|
||||
|
||||
pub fn item_count(&self) -> usize {
|
||||
self.items.len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
type Id = *mut Object;
|
||||
|
||||
/// RAII wrapper so the pool drains even on early return / `?` propagation.
|
||||
#[cfg(target_os = "macos")]
|
||||
struct AutoreleasePool {
|
||||
pool: Id,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl AutoreleasePool {
|
||||
unsafe fn new() -> Self {
|
||||
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
|
||||
let pool: Id = msg_send![pool, init];
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl Drop for AutoreleasePool {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _: () = msg_send![self.pool, drain];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an autoreleased `NSString` from a Rust `&str` without scanning for
|
||||
/// interior nulls (which is what `initWithUTF8String:` would require).
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string(s: &str) -> Id {
|
||||
// NSUTF8StringEncoding = 4.
|
||||
let obj: Id = msg_send![class!(NSString), alloc];
|
||||
let obj: Id = msg_send![
|
||||
obj,
|
||||
initWithBytes: s.as_ptr()
|
||||
length: s.len()
|
||||
encoding: 4u64
|
||||
];
|
||||
let _: () = msg_send![obj, autorelease];
|
||||
obj
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bytes: *const i8 = msg_send![s, UTF8String];
|
||||
if bytes.is_null() {
|
||||
return None;
|
||||
}
|
||||
std::ffi::CStr::from_ptr(bytes)
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn general_pasteboard() -> Result<Id, String> {
|
||||
let pb: Id = msg_send![class!(NSPasteboard), generalPasteboard];
|
||||
if pb.is_null() {
|
||||
return Err("NSPasteboard generalPasteboard returned nil".into());
|
||||
}
|
||||
Ok(pb)
|
||||
}
|
||||
|
||||
/// Read the pasteboard's current change count without snapshotting contents.
|
||||
///
|
||||
/// AppKit increments this every time any process writes to the general
|
||||
/// pasteboard, so it's a cheap way to detect "did someone clobber my staged
|
||||
/// text before the paste landed?".
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let c: i64 = msg_send![pb, changeCount];
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture every item on the general pasteboard into an owned snapshot.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let change_count: i64 = msg_send![pb, changeCount];
|
||||
|
||||
let items: Id = msg_send![pb, pasteboardItems];
|
||||
if items.is_null() {
|
||||
return Ok(ClipboardSnapshot {
|
||||
items: Vec::new(),
|
||||
change_count,
|
||||
});
|
||||
}
|
||||
|
||||
let count: usize = msg_send![items, count];
|
||||
let mut saved: Vec<Vec<(String, Vec<u8>)>> = Vec::with_capacity(count);
|
||||
|
||||
for i in 0..count {
|
||||
let item: Id = msg_send![items, objectAtIndex: i];
|
||||
if item.is_null() {
|
||||
continue;
|
||||
}
|
||||
let types: Id = msg_send![item, types];
|
||||
if types.is_null() {
|
||||
continue;
|
||||
}
|
||||
let type_count: usize = msg_send![types, count];
|
||||
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(type_count);
|
||||
for j in 0..type_count {
|
||||
let t: Id = msg_send![types, objectAtIndex: j];
|
||||
let Some(type_str) = ns_string_to_rust(t) else {
|
||||
continue;
|
||||
};
|
||||
let data: Id = msg_send![item, dataForType: t];
|
||||
if data.is_null() {
|
||||
// Type advertised but no concrete data (lazy provider).
|
||||
// Skipping is safer than trying to force it to materialise.
|
||||
continue;
|
||||
}
|
||||
let length: usize = msg_send![data, length];
|
||||
let bytes_ptr: *const u8 = msg_send![data, bytes];
|
||||
let bytes = if bytes_ptr.is_null() || length == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
std::slice::from_raw_parts(bytes_ptr, length).to_vec()
|
||||
};
|
||||
pairs.push((type_str, bytes));
|
||||
}
|
||||
saved.push(pairs);
|
||||
}
|
||||
|
||||
Ok(ClipboardSnapshot {
|
||||
items: saved,
|
||||
change_count,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the pasteboard contents with a single plain-text string. Returns
|
||||
/// the post-write change count so a later restore can verify nothing else
|
||||
/// touched the clipboard in between.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn write_text(text: &str) -> Result<i64, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let _new_count: i64 = msg_send![pb, clearContents];
|
||||
|
||||
let ns_text = ns_string(text);
|
||||
// `public.utf8-plain-text` is the raw UTI behind `NSPasteboardTypeString`
|
||||
// and works for every text-aware paste target we care about.
|
||||
let ns_type = ns_string("public.utf8-plain-text");
|
||||
let ok: bool = msg_send![pb, setString: ns_text forType: ns_type];
|
||||
if !ok {
|
||||
return Err("NSPasteboard setString:forType: returned NO".into());
|
||||
}
|
||||
|
||||
let after: i64 = msg_send![pb, changeCount];
|
||||
Ok(after)
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild the pasteboard from a snapshot, replacing whatever is on it now.
|
||||
///
|
||||
/// Does not consult the change count — callers that want safe restore should
|
||||
/// compare [`current_change_count`] against the value returned by
|
||||
/// [`write_text`] first.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let _: i64 = msg_send![pb, clearContents];
|
||||
|
||||
if snapshot.items.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let array: Id = msg_send![class!(NSMutableArray), array];
|
||||
|
||||
for pairs in &snapshot.items {
|
||||
let item: Id = msg_send![class!(NSPasteboardItem), alloc];
|
||||
let item: Id = msg_send![item, init];
|
||||
let _: () = msg_send![item, autorelease];
|
||||
|
||||
for (uti, bytes) in pairs {
|
||||
let ns_type = ns_string(uti);
|
||||
let data: Id = msg_send![
|
||||
class!(NSData),
|
||||
dataWithBytes: bytes.as_ptr()
|
||||
length: bytes.len()
|
||||
];
|
||||
let _ok: bool = msg_send![item, setData: data forType: ns_type];
|
||||
}
|
||||
|
||||
let _: () = msg_send![array, addObject: item];
|
||||
}
|
||||
|
||||
let ok: bool = msg_send![pb, writeObjects: array];
|
||||
if !ok {
|
||||
return Err("NSPasteboard writeObjects: returned NO".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
//! Windows clipboard implementation.
|
||||
//!
|
||||
//! The snapshot is structured so it mirrors the macOS `Vec<Vec<_>>`
|
||||
//! shape: a single outer "item" holding one `(format-name, bytes)`
|
||||
//! pair per enumerated format. Windows has no notion of multiple
|
||||
//! pasteboard items, so there's always exactly one or zero outer
|
||||
//! entries — enough to keep `item_count()` meaningful without
|
||||
//! fan-out.
|
||||
//!
|
||||
//! Format IDs are serialised as strings so the snapshot type can stay
|
||||
//! platform-neutral. Predefined formats use their canonical
|
||||
//! identifier (`"CF_UNICODETEXT"`, `"CF_HDROP"`, `"CF_DIB"`, …);
|
||||
//! registered formats use their string name from
|
||||
//! `GetClipboardFormatNameW` (`"HTML Format"`, `"Rich Text
|
||||
//! Format"`, …). Restore reverses the mapping with a lookup table
|
||||
//! for the predefined IDs and `RegisterClipboardFormatW` for the
|
||||
//! rest.
|
||||
//!
|
||||
//! Skipped format classes:
|
||||
//! - CF_BITMAP (2), CF_METAFILEPICT (3), CF_PALETTE (9),
|
||||
//! CF_ENHMETAFILE (14) — HGLOBAL's actually an HBITMAP /
|
||||
//! HENHMETAFILE, not raw memory. Rebuilding them across processes
|
||||
//! is possible but not worth it for clipboard stashing.
|
||||
//! - CF_OWNERDISPLAY (0x80) and the CF_DSPxxx variants (0x81–0x8E) —
|
||||
//! the owner draws these on demand. No data to snapshot.
|
||||
//! - CF_PRIVATEFIRST..CF_PRIVATELAST (0x200–0x2FF) — app-private,
|
||||
//! meaningless to restore from a different process.
|
||||
//! - CF_GDIOBJFIRST..CF_GDIOBJLAST (0x300–0x3FF) — GDI handles.
|
||||
//!
|
||||
//! Text formats that Windows auto-synthesises (CF_TEXT, CF_OEMTEXT,
|
||||
//! CF_LOCALE) are also skipped during save: `SetClipboardData` on
|
||||
//! CF_UNICODETEXT regenerates them lazily on restore.
|
||||
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use windows::core::PCWSTR;
|
||||
use windows::Win32::Foundation::{GlobalFree, HANDLE, HGLOBAL, HWND};
|
||||
use windows::Win32::System::DataExchange::{
|
||||
CloseClipboard, EmptyClipboard, EnumClipboardFormats, GetClipboardData,
|
||||
GetClipboardFormatNameW, GetClipboardSequenceNumber, OpenClipboard,
|
||||
RegisterClipboardFormatW, SetClipboardData,
|
||||
};
|
||||
use windows::Win32::System::Memory::{
|
||||
GlobalAlloc, GlobalLock, GlobalSize, GlobalUnlock, GLOBAL_ALLOC_FLAGS,
|
||||
};
|
||||
|
||||
// `windows` 0.62 doesn't re-export every predefined clipboard format
|
||||
// under a stable feature flag, so the values are pinned inline.
|
||||
// These numbers are ABI-stable back to Windows 3.1 — verified against
|
||||
// winuser.h.
|
||||
pub const CF_TEXT: u32 = 1;
|
||||
pub const CF_BITMAP: u32 = 2;
|
||||
pub const CF_METAFILEPICT: u32 = 3;
|
||||
pub const CF_SYLK: u32 = 4;
|
||||
pub const CF_DIF: u32 = 5;
|
||||
pub const CF_TIFF: u32 = 6;
|
||||
pub const CF_OEMTEXT: u32 = 7;
|
||||
pub const CF_DIB: u32 = 8;
|
||||
pub const CF_PALETTE: u32 = 9;
|
||||
pub const CF_PENDATA: u32 = 10;
|
||||
pub const CF_RIFF: u32 = 11;
|
||||
pub const CF_WAVE: u32 = 12;
|
||||
pub const CF_UNICODETEXT: u32 = 13;
|
||||
pub const CF_ENHMETAFILE: u32 = 14;
|
||||
pub const CF_HDROP: u32 = 15;
|
||||
pub const CF_LOCALE: u32 = 16;
|
||||
pub const CF_DIBV5: u32 = 17;
|
||||
pub const CF_OWNERDISPLAY: u32 = 0x0080;
|
||||
pub const CF_DSPTEXT: u32 = 0x0081;
|
||||
pub const CF_DSPBITMAP: u32 = 0x0082;
|
||||
pub const CF_DSPMETAFILEPICT: u32 = 0x0083;
|
||||
pub const CF_DSPENHMETAFILE: u32 = 0x008E;
|
||||
pub const CF_PRIVATEFIRST: u32 = 0x0200;
|
||||
pub const CF_PRIVATELAST: u32 = 0x02FF;
|
||||
pub const CF_GDIOBJFIRST: u32 = 0x0300;
|
||||
pub const CF_GDIOBJLAST: u32 = 0x03FF;
|
||||
|
||||
/// `GlobalAlloc` movable-memory flag — `GMEM_MOVEABLE` (0x0002).
|
||||
/// Required for HGLOBAL handles destined for `SetClipboardData`; fixed
|
||||
/// allocations are rejected.
|
||||
const GMEM_MOVEABLE: GLOBAL_ALLOC_FLAGS = GLOBAL_ALLOC_FLAGS(0x0002);
|
||||
|
||||
/// Map a predefined clipboard format ID to its canonical identifier
|
||||
/// string. Registered formats (IDs >= 0xC000) aren't handled here —
|
||||
/// the caller resolves those via `GetClipboardFormatNameW`.
|
||||
pub fn predefined_name(id: u32) -> Option<&'static str> {
|
||||
Some(match id {
|
||||
CF_TEXT => "CF_TEXT",
|
||||
CF_BITMAP => "CF_BITMAP",
|
||||
CF_METAFILEPICT => "CF_METAFILEPICT",
|
||||
CF_SYLK => "CF_SYLK",
|
||||
CF_DIF => "CF_DIF",
|
||||
CF_TIFF => "CF_TIFF",
|
||||
CF_OEMTEXT => "CF_OEMTEXT",
|
||||
CF_DIB => "CF_DIB",
|
||||
CF_PALETTE => "CF_PALETTE",
|
||||
CF_PENDATA => "CF_PENDATA",
|
||||
CF_RIFF => "CF_RIFF",
|
||||
CF_WAVE => "CF_WAVE",
|
||||
CF_UNICODETEXT => "CF_UNICODETEXT",
|
||||
CF_ENHMETAFILE => "CF_ENHMETAFILE",
|
||||
CF_HDROP => "CF_HDROP",
|
||||
CF_LOCALE => "CF_LOCALE",
|
||||
CF_DIBV5 => "CF_DIBV5",
|
||||
CF_OWNERDISPLAY => "CF_OWNERDISPLAY",
|
||||
CF_DSPTEXT => "CF_DSPTEXT",
|
||||
CF_DSPBITMAP => "CF_DSPBITMAP",
|
||||
CF_DSPMETAFILEPICT => "CF_DSPMETAFILEPICT",
|
||||
CF_DSPENHMETAFILE => "CF_DSPENHMETAFILE",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reverse of [`predefined_name`].
|
||||
pub fn predefined_id(name: &str) -> Option<u32> {
|
||||
Some(match name {
|
||||
"CF_TEXT" => CF_TEXT,
|
||||
"CF_BITMAP" => CF_BITMAP,
|
||||
"CF_METAFILEPICT" => CF_METAFILEPICT,
|
||||
"CF_SYLK" => CF_SYLK,
|
||||
"CF_DIF" => CF_DIF,
|
||||
"CF_TIFF" => CF_TIFF,
|
||||
"CF_OEMTEXT" => CF_OEMTEXT,
|
||||
"CF_DIB" => CF_DIB,
|
||||
"CF_PALETTE" => CF_PALETTE,
|
||||
"CF_PENDATA" => CF_PENDATA,
|
||||
"CF_RIFF" => CF_RIFF,
|
||||
"CF_WAVE" => CF_WAVE,
|
||||
"CF_UNICODETEXT" => CF_UNICODETEXT,
|
||||
"CF_ENHMETAFILE" => CF_ENHMETAFILE,
|
||||
"CF_HDROP" => CF_HDROP,
|
||||
"CF_LOCALE" => CF_LOCALE,
|
||||
"CF_DIBV5" => CF_DIBV5,
|
||||
"CF_OWNERDISPLAY" => CF_OWNERDISPLAY,
|
||||
"CF_DSPTEXT" => CF_DSPTEXT,
|
||||
"CF_DSPBITMAP" => CF_DSPBITMAP,
|
||||
"CF_DSPMETAFILEPICT" => CF_DSPMETAFILEPICT,
|
||||
"CF_DSPENHMETAFILE" => CF_DSPENHMETAFILE,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns true for predefined formats whose payload is a GDI handle
|
||||
/// or owner-display sentinel rather than plain memory — callers must
|
||||
/// skip these during snapshot because GlobalSize/GlobalLock wouldn't
|
||||
/// return usable bytes.
|
||||
pub fn is_skipped_format(id: u32) -> bool {
|
||||
matches!(
|
||||
id,
|
||||
CF_BITMAP
|
||||
| CF_METAFILEPICT
|
||||
| CF_PALETTE
|
||||
| CF_ENHMETAFILE
|
||||
| CF_OWNERDISPLAY
|
||||
| CF_DSPTEXT
|
||||
| CF_DSPBITMAP
|
||||
| CF_DSPMETAFILEPICT
|
||||
| CF_DSPENHMETAFILE
|
||||
) || (CF_PRIVATEFIRST..=CF_PRIVATELAST).contains(&id)
|
||||
|| (CF_GDIOBJFIRST..=CF_GDIOBJLAST).contains(&id)
|
||||
}
|
||||
|
||||
/// Auto-synthesised formats that Windows regenerates from
|
||||
/// CF_UNICODETEXT on demand. Safe to skip during save; restore
|
||||
/// lets `SetClipboardData(CF_UNICODETEXT)` re-derive them.
|
||||
pub fn is_auto_synthesised(id: u32) -> bool {
|
||||
matches!(id, CF_TEXT | CF_OEMTEXT | CF_LOCALE)
|
||||
}
|
||||
|
||||
/// RAII wrapper around `OpenClipboard` / `CloseClipboard`.
|
||||
///
|
||||
/// The clipboard is a global exclusive resource — only one process at
|
||||
/// a time holds the handle. `OpenClipboard` fails with
|
||||
/// ERROR_ACCESS_DENIED when another process is mid-paste; the retry
|
||||
/// loop here absorbs the common transient case without bubbling a
|
||||
/// user-visible error.
|
||||
pub struct ClipboardGuard;
|
||||
|
||||
impl ClipboardGuard {
|
||||
pub fn open() -> Result<Self, String> {
|
||||
const MAX_ATTEMPTS: usize = 10;
|
||||
const RETRY_DELAY: Duration = Duration::from_millis(10);
|
||||
let mut last_err: Option<windows::core::Error> = None;
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
let result = unsafe { OpenClipboard(Some(HWND(std::ptr::null_mut()))) };
|
||||
match result {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
thread::sleep(RETRY_DELAY);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!(
|
||||
"OpenClipboard failed after {} retries ({:?}). Another process likely holds the clipboard open.",
|
||||
MAX_ATTEMPTS, last_err
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ClipboardGuard {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _ = CloseClipboard();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the full payload for `format` from the currently open
|
||||
/// clipboard into an owned `Vec<u8>`. Returns `Ok(None)` when the
|
||||
/// clipboard advertises the format but provides no concrete data
|
||||
/// (delay-rendered format that's never been realised).
|
||||
pub fn read_format_bytes(format: u32) -> Result<Option<Vec<u8>>, String> {
|
||||
unsafe {
|
||||
let handle = GetClipboardData(format)
|
||||
.map_err(|e| format!("GetClipboardData({format}) failed: {e}"))?;
|
||||
if handle.is_invalid() {
|
||||
return Ok(None);
|
||||
}
|
||||
let hglobal = HGLOBAL(handle.0);
|
||||
let size = GlobalSize(hglobal);
|
||||
if size == 0 {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
let ptr = GlobalLock(hglobal);
|
||||
if ptr.is_null() {
|
||||
return Err(format!(
|
||||
"GlobalLock returned null for format {format} (size {size})"
|
||||
));
|
||||
}
|
||||
let bytes = std::slice::from_raw_parts(ptr as *const u8, size).to_vec();
|
||||
let _ = GlobalUnlock(hglobal);
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up the name for a registered format ID (>= 0xC000). Returns
|
||||
/// `None` for unnamed predefined IDs — the caller should have used
|
||||
/// [`predefined_name`] first.
|
||||
pub fn registered_name(id: u32) -> Option<String> {
|
||||
let mut buf = [0u16; 256];
|
||||
let len = unsafe { GetClipboardFormatNameW(id, &mut buf) };
|
||||
if len <= 0 {
|
||||
return None;
|
||||
}
|
||||
String::from_utf16(&buf[..len as usize]).ok()
|
||||
}
|
||||
|
||||
/// Allocate a movable HGLOBAL, copy `bytes` in, return the handle
|
||||
/// ready for `SetClipboardData`. On success ownership transfers to
|
||||
/// the clipboard; on failure the caller must `GlobalFree`.
|
||||
pub fn allocate_global(bytes: &[u8]) -> Result<HGLOBAL, String> {
|
||||
if bytes.is_empty() {
|
||||
// `GlobalAlloc(_, 0)` returns NULL, which `SetClipboardData`
|
||||
// would then reject as an invalid handle. Pad to one byte so
|
||||
// the format still round-trips (the receiving app already
|
||||
// has to handle zero-content payloads via GlobalSize).
|
||||
return allocate_global(&[0u8]);
|
||||
}
|
||||
unsafe {
|
||||
let hglobal = GlobalAlloc(GMEM_MOVEABLE, bytes.len())
|
||||
.map_err(|e| format!("GlobalAlloc({}) failed: {e}", bytes.len()))?;
|
||||
let ptr = GlobalLock(hglobal);
|
||||
if ptr.is_null() {
|
||||
let _ = GlobalFree(Some(hglobal));
|
||||
return Err("GlobalLock returned null after GlobalAlloc".into());
|
||||
}
|
||||
std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr as *mut u8, bytes.len());
|
||||
let _ = GlobalUnlock(hglobal);
|
||||
Ok(hglobal)
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one format's payload onto the currently open clipboard.
|
||||
/// On `SetClipboardData` success the HGLOBAL becomes the clipboard's
|
||||
/// responsibility — do not free. On failure, free it ourselves.
|
||||
pub fn put_format(format: u32, bytes: &[u8]) -> Result<(), String> {
|
||||
let hglobal = allocate_global(bytes)?;
|
||||
let handle = HANDLE(hglobal.0);
|
||||
unsafe {
|
||||
match SetClipboardData(format, Some(handle)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
let _ = GlobalFree(Some(hglobal));
|
||||
Err(format!("SetClipboardData({format}) failed: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// UTF-16 encode `s` with a trailing null code unit and push it as
|
||||
/// CF_UNICODETEXT.
|
||||
pub fn put_unicode_text(s: &str) -> Result<(), String> {
|
||||
let mut utf16: Vec<u16> = s.encode_utf16().collect();
|
||||
utf16.push(0);
|
||||
let bytes: &[u8] = unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
utf16.as_ptr() as *const u8,
|
||||
utf16.len() * std::mem::size_of::<u16>(),
|
||||
)
|
||||
};
|
||||
put_format(CF_UNICODETEXT, bytes)
|
||||
}
|
||||
|
||||
/// Walk every format currently on the clipboard. `EnumClipboardFormats(0)`
|
||||
/// returns the first; each subsequent call with the previous format
|
||||
/// returns the next, until it returns 0 (or an error).
|
||||
pub fn enumerate_formats() -> Vec<u32> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = 0u32;
|
||||
loop {
|
||||
let next = unsafe { EnumClipboardFormats(current) };
|
||||
if next == 0 {
|
||||
break;
|
||||
}
|
||||
out.push(next);
|
||||
current = next;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Resolve a snapshot's format name back to the u32 format ID.
|
||||
/// Registered names (anything not predefined) go through
|
||||
/// `RegisterClipboardFormatW`, which is idempotent — the same name
|
||||
/// yields the same ID within a Windows session.
|
||||
pub fn resolve_format_id(name: &str) -> Result<u32, String> {
|
||||
if let Some(id) = predefined_id(name) {
|
||||
return Ok(id);
|
||||
}
|
||||
let wide: Vec<u16> = name.encode_utf16().chain(std::iter::once(0)).collect();
|
||||
let id = unsafe { RegisterClipboardFormatW(PCWSTR(wide.as_ptr())) };
|
||||
if id == 0 {
|
||||
return Err(format!("RegisterClipboardFormatW failed for {name:?}"));
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn sequence_number() -> u32 {
|
||||
unsafe { GetClipboardSequenceNumber() }
|
||||
}
|
||||
|
||||
pub fn empty() -> Result<(), String> {
|
||||
unsafe { EmptyClipboard().map_err(|e| format!("EmptyClipboard failed: {e}")) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
Ok(win::sequence_number() as i64)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
let change_count = win::sequence_number() as i64;
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
|
||||
let formats = win::enumerate_formats();
|
||||
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(formats.len());
|
||||
for id in formats {
|
||||
if win::is_skipped_format(id) || win::is_auto_synthesised(id) {
|
||||
continue;
|
||||
}
|
||||
let name = match win::predefined_name(id) {
|
||||
Some(n) => n.to_string(),
|
||||
None => match win::registered_name(id) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
},
|
||||
};
|
||||
match win::read_format_bytes(id) {
|
||||
Ok(Some(bytes)) => pairs.push((name, bytes)),
|
||||
Ok(None) => {}
|
||||
Err(_) => {
|
||||
// Single-format read failure (delay-render that never
|
||||
// materialises, ACL-restricted format, etc.) shouldn't
|
||||
// abort the whole snapshot — drop this format and keep
|
||||
// going so the user's other clipboard contents still
|
||||
// survive the round-trip.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let items = if pairs.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![pairs]
|
||||
};
|
||||
|
||||
Ok(ClipboardSnapshot {
|
||||
items,
|
||||
change_count,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn write_text(text: &str) -> Result<i64, String> {
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
win::empty()?;
|
||||
win::put_unicode_text(text)?;
|
||||
// `GetClipboardSequenceNumber` reflects the post-write value as soon
|
||||
// as `SetClipboardData` returns.
|
||||
Ok(win::sequence_number() as i64)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
win::empty()?;
|
||||
|
||||
for pairs in &snapshot.items {
|
||||
for (name, bytes) in pairs {
|
||||
let id = match win::resolve_format_id(name) {
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
// Per-format failures here also don't abort the whole
|
||||
// restore — better to get the user's text content back even
|
||||
// if a weird custom format can't be rehydrated.
|
||||
let _ = win::put_format(id, bytes);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn write_text(_text: &str) -> Result<i64, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn restore_clipboard(_snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
//! Captures the focused-UI snapshot at chord-start so auto-paste can land
|
||||
//! in the user's original text field even after focus drifts during
|
||||
//! transcription / refinement.
|
||||
//!
|
||||
//! We don't try to re-focus a specific sub-element on restore — many apps
|
||||
//! expose complex focus hierarchies that don't respond consistently to
|
||||
//! programmatic focus pokes. Bringing the owning *window* to the
|
||||
//! foreground is enough: the window's own focus manager restores its
|
||||
//! last-focused field, which is what every well-behaved paste-buffer tool
|
||||
//! does and what users expect.
|
||||
//!
|
||||
//! - **macOS** — `AXUIElementCopyAttributeValue(kAXFocusedUIElement)` +
|
||||
//! `AXUIElementGetPid` + NSRunningApplication activation. Activation
|
||||
//! uses the cooperative-activation pattern on macOS 14+ (the caller
|
||||
//! `yieldActivationToApplication:`s, then the target `activate`s) and
|
||||
//! falls back to the pre-Sonoma `activateWithOptions:` on 11–13. See
|
||||
//! `activate_pid` for the rationale.
|
||||
//! - **Windows** — `GetForegroundWindow` + `GetWindowThreadProcessId` for
|
||||
//! the top-level HWND and PID; UIAutomation's `IUIAutomation::GetFocusedElement`
|
||||
//! for best-effort control-class (skipped silently if COM isn't usable).
|
||||
//! Activation walks top-level windows for the saved PID and calls
|
||||
//! `SetForegroundWindow`, bracketed by the `AttachThreadInput` dance
|
||||
//! so Windows' foreground-lock rules don't silently swallow the
|
||||
//! activation into a taskbar flash.
|
||||
//!
|
||||
//! PID + bundle id + role are all captured for diagnostics — the bundle
|
||||
//! id lets step 6 (internal direct injection) detect "focus was inside
|
||||
//! Voicebox itself" and short-circuit the synthetic-paste path. On
|
||||
//! Windows, `bundle_id` holds the lowercased exe basename (`"voicebox.exe"`)
|
||||
//! since there's no equivalent of macOS' reverse-DNS bundle identifier.
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct FocusSnapshot {
|
||||
pub pid: i32,
|
||||
pub bundle_id: Option<String>,
|
||||
pub role: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use core_foundation_sys::base::{kCFAllocatorDefault, CFRelease};
|
||||
#[cfg(target_os = "macos")]
|
||||
use core_foundation_sys::string::{
|
||||
kCFStringEncodingUTF8, CFStringCreateWithCString, CFStringGetCString, CFStringGetLength,
|
||||
CFStringRef,
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::runtime::Object;
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::{class, msg_send, sel, sel_impl};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
type Id = *mut Object;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use core_foundation_sys::base::CFTypeRef;
|
||||
use core_foundation_sys::string::CFStringRef;
|
||||
|
||||
pub type AXError = i32;
|
||||
pub const AX_ERROR_SUCCESS: AXError = 0;
|
||||
pub type AXUIElementRef = *const std::ffi::c_void;
|
||||
pub type Pid = i32;
|
||||
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn AXUIElementCreateSystemWide() -> AXUIElementRef;
|
||||
pub fn AXUIElementCopyAttributeValue(
|
||||
element: AXUIElementRef,
|
||||
attribute: CFStringRef,
|
||||
value: *mut CFTypeRef,
|
||||
) -> AXError;
|
||||
pub fn AXUIElementGetPid(element: AXUIElementRef, pid: *mut Pid) -> AXError;
|
||||
}
|
||||
// AX attribute keys are exposed as C macros that expand to CFSTR(...)
|
||||
// literals, not as linkable symbols — build the CFStrings at runtime
|
||||
// instead (see `cf_string_const` in focus_capture.rs).
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
struct AutoreleasePool {
|
||||
pool: Id,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl AutoreleasePool {
|
||||
unsafe fn new() -> Self {
|
||||
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
|
||||
let pool: Id = msg_send![pool, init];
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl Drop for AutoreleasePool {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _: () = msg_send![self.pool, drain];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bytes: *const i8 = msg_send![s, UTF8String];
|
||||
if bytes.is_null() {
|
||||
return None;
|
||||
}
|
||||
std::ffi::CStr::from_ptr(bytes)
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
/// Build a `+1` retained CFString from an ASCII constant. Caller owns the
|
||||
/// returned reference and must `CFRelease` it. Used for AX attribute keys
|
||||
/// (`"AXFocusedUIElement"`, `"AXRole"`) because those aren't exported as
|
||||
/// linker symbols — Apple ships them as `CFSTR(...)` macros.
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn cf_string_const(s: &str) -> Option<CFStringRef> {
|
||||
let cstr = std::ffi::CString::new(s).ok()?;
|
||||
let result = CFStringCreateWithCString(kCFAllocatorDefault, cstr.as_ptr(), kCFStringEncodingUTF8);
|
||||
if result.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn cfstring_to_rust(s: CFStringRef) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let len = CFStringGetLength(s);
|
||||
if len == 0 {
|
||||
return Some(String::new());
|
||||
}
|
||||
// CFStringGetLength is in UTF-16 code units; UTF-8 can need up to 4
|
||||
// bytes per unit plus the trailing NUL.
|
||||
let max_bytes = (len * 4 + 1) as usize;
|
||||
let mut buf = vec![0u8; max_bytes];
|
||||
let ok = CFStringGetCString(
|
||||
s,
|
||||
buf.as_mut_ptr() as *mut i8,
|
||||
max_bytes as isize,
|
||||
kCFStringEncodingUTF8,
|
||||
);
|
||||
if ok == 0 {
|
||||
return None;
|
||||
}
|
||||
let cstr = std::ffi::CStr::from_ptr(buf.as_ptr() as *const i8);
|
||||
cstr.to_str().ok().map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn bundle_id_for_pid(pid: i32) -> Option<String> {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let app: Id = msg_send![
|
||||
class!(NSRunningApplication),
|
||||
runningApplicationWithProcessIdentifier: pid
|
||||
];
|
||||
if app.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bundle: Id = msg_send![app, bundleIdentifier];
|
||||
ns_string_to_rust(bundle)
|
||||
}
|
||||
|
||||
/// Read the system-wide focused UI element's PID, bundle id, and AX role.
|
||||
///
|
||||
/// Returns an error when no element is focused (e.g. Dock has focus) or
|
||||
/// when Accessibility permission is missing — `AXUIElementCopyAttributeValue`
|
||||
/// returns `-25204 kAXErrorAPIDisabled` in that case.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
use ffi::*;
|
||||
unsafe {
|
||||
let system_wide = AXUIElementCreateSystemWide();
|
||||
if system_wide.is_null() {
|
||||
return Err("AXUIElementCreateSystemWide returned null".into());
|
||||
}
|
||||
let _sys_guard = scopeguard::guard(system_wide, |e| {
|
||||
CFRelease(e as *const std::ffi::c_void)
|
||||
});
|
||||
|
||||
let focused_attr = cf_string_const("AXFocusedUIElement")
|
||||
.ok_or("Failed to build AXFocusedUIElement CFString")?;
|
||||
let _focused_attr_guard =
|
||||
scopeguard::guard(focused_attr, |s| CFRelease(s as *const std::ffi::c_void));
|
||||
|
||||
let mut focused: *const std::ffi::c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(
|
||||
system_wide,
|
||||
focused_attr,
|
||||
&mut focused as *mut _,
|
||||
);
|
||||
if err != AX_ERROR_SUCCESS || focused.is_null() {
|
||||
return Err(format!(
|
||||
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
|
||||
err
|
||||
));
|
||||
}
|
||||
let _focus_guard = scopeguard::guard(focused, |e| CFRelease(e));
|
||||
|
||||
let focused_elem = focused as AXUIElementRef;
|
||||
|
||||
let mut pid: Pid = 0;
|
||||
let err = AXUIElementGetPid(focused_elem, &mut pid);
|
||||
if err != AX_ERROR_SUCCESS {
|
||||
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
|
||||
}
|
||||
|
||||
let role = {
|
||||
let role_attr = cf_string_const("AXRole");
|
||||
match role_attr {
|
||||
Some(role_attr) => {
|
||||
let _role_attr_guard = scopeguard::guard(role_attr, |s| {
|
||||
CFRelease(s as *const std::ffi::c_void)
|
||||
});
|
||||
let mut role_value: *const std::ffi::c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(
|
||||
focused_elem,
|
||||
role_attr,
|
||||
&mut role_value as *mut _,
|
||||
);
|
||||
if err == AX_ERROR_SUCCESS && !role_value.is_null() {
|
||||
let _role_guard = scopeguard::guard(role_value, |e| CFRelease(e));
|
||||
cfstring_to_rust(role_value as CFStringRef)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
};
|
||||
|
||||
let bundle_id = bundle_id_for_pid(pid);
|
||||
|
||||
Ok(FocusSnapshot {
|
||||
pid,
|
||||
bundle_id,
|
||||
role,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Bring the app owning `pid` to the foreground, re-activating its
|
||||
/// last-focused window. Paired with [`capture_focus`] at chord-start so a
|
||||
/// post-transcription synthetic ⌘V lands where the user started, not
|
||||
/// wherever focus drifted to during the transcribe / refine window.
|
||||
///
|
||||
/// macOS 14 (Sonoma) deprecated `activateWithOptions:` in favour of a
|
||||
/// cooperative-activation pattern: the caller first invokes
|
||||
/// `yieldActivationToApplication:` on its own `NSRunningApplication` to
|
||||
/// grant the target activation rights, then the target's `activate`
|
||||
/// succeeds against the tightened Sonoma foreground rules. Without the
|
||||
/// yield, `activate` on 14+ sometimes silently fails or only bounces the
|
||||
/// dock icon — exactly the "paste lands in the wrong app" symptom we're
|
||||
/// trying to prevent. The yield is discovered at runtime via
|
||||
/// `respondsToSelector:` so we don't need an operatingSystemVersion probe
|
||||
/// and the pre-Sonoma path stays identical.
|
||||
///
|
||||
/// The BOOL return of both `activate` and `activateWithOptions:` is now
|
||||
/// propagated — if the system refuses activation (target quit mid-
|
||||
/// transcription, trust revoked, cooperative-activation refused) the
|
||||
/// caller aborts before clobbering the clipboard.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn activate_pid(pid: i32) -> Result<(), String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let target: Id = msg_send![
|
||||
class!(NSRunningApplication),
|
||||
runningApplicationWithProcessIdentifier: pid
|
||||
];
|
||||
if target.is_null() {
|
||||
return Err(format!("No running application for PID {}", pid));
|
||||
}
|
||||
|
||||
let activated: bool = if can_yield_activation() {
|
||||
let current: Id =
|
||||
msg_send![class!(NSRunningApplication), currentApplication];
|
||||
if !current.is_null() {
|
||||
let _: () = msg_send![current, yieldActivationToApplication: target];
|
||||
}
|
||||
msg_send![target, activate]
|
||||
} else {
|
||||
// NSApplicationActivateIgnoringOtherApps = 1 << 1 = 2.
|
||||
msg_send![target, activateWithOptions: 2u64]
|
||||
};
|
||||
|
||||
if !activated {
|
||||
return Err(format!(
|
||||
"NSRunningApplication activate returned false for PID {} — the target may have quit mid-transcription, Accessibility is no longer trusted, or the system refused cooperative activation.",
|
||||
pid
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// `true` when `NSRunningApplication` responds to
|
||||
/// `yieldActivationToApplication:` — the macOS 14+ discriminator for the
|
||||
/// cooperative-activation APIs. Cached since the answer doesn't change
|
||||
/// over a process's lifetime and the objc_msgSend probe is otherwise
|
||||
/// repeated on every paste.
|
||||
#[cfg(target_os = "macos")]
|
||||
fn can_yield_activation() -> bool {
|
||||
use std::sync::OnceLock;
|
||||
static CACHED: OnceLock<bool> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| unsafe {
|
||||
let current: Id = msg_send![class!(NSRunningApplication), currentApplication];
|
||||
if current.is_null() {
|
||||
return false;
|
||||
}
|
||||
let responds: bool = msg_send![
|
||||
current,
|
||||
respondsToSelector: sel!(yieldActivationToApplication:)
|
||||
];
|
||||
responds
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
use std::path::Path;
|
||||
|
||||
use windows::core::{IUnknown, BOOL, BSTR, PWSTR};
|
||||
use windows::Win32::Foundation::{CloseHandle, HWND, LPARAM};
|
||||
use windows::Win32::System::Com::{
|
||||
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
|
||||
};
|
||||
use windows::Win32::System::Threading::{
|
||||
AttachThreadInput, GetCurrentThreadId, OpenProcess, QueryFullProcessImageNameW,
|
||||
PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION,
|
||||
};
|
||||
use windows::Win32::UI::Accessibility::{CUIAutomation, IUIAutomation, IUIAutomationElement};
|
||||
use windows::Win32::UI::WindowsAndMessaging::{
|
||||
EnumWindows, GetForegroundWindow, GetWindow, GetWindowThreadProcessId, IsWindowVisible,
|
||||
SetForegroundWindow, GW_OWNER,
|
||||
};
|
||||
|
||||
/// Read the PID that owns `hwnd`. Returns 0 on failure.
|
||||
pub unsafe fn hwnd_pid(hwnd: HWND) -> u32 {
|
||||
let mut pid: u32 = 0;
|
||||
let _ = GetWindowThreadProcessId(hwnd, Some(&mut pid as *mut _));
|
||||
pid
|
||||
}
|
||||
|
||||
/// Query a PID's executable path and return its lowercased basename
|
||||
/// (e.g. `"voicebox.exe"`). This is the Windows analogue of macOS'
|
||||
/// `bundleIdentifier`, just less globally unique — two apps with the
|
||||
/// same exe name can collide, but that's rare enough to accept for
|
||||
/// the self-paste short-circuit.
|
||||
pub fn exe_basename(pid: u32) -> Option<String> {
|
||||
unsafe {
|
||||
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
|
||||
let mut buf = [0u16; 1024];
|
||||
let mut size = buf.len() as u32;
|
||||
let ok = QueryFullProcessImageNameW(
|
||||
handle,
|
||||
PROCESS_NAME_FORMAT(0),
|
||||
PWSTR(buf.as_mut_ptr()),
|
||||
&mut size,
|
||||
);
|
||||
let _ = CloseHandle(handle);
|
||||
if ok.is_err() || size == 0 {
|
||||
return None;
|
||||
}
|
||||
let full = String::from_utf16(&buf[..size as usize]).ok()?;
|
||||
let basename = Path::new(&full)
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())?;
|
||||
Some(basename)
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort `UIAutomation::GetFocusedElement().CurrentClassName()`.
|
||||
/// Returns `None` when COM init, CoCreateInstance, or any UIA call
|
||||
/// fails — role info is nice-to-have, not load-bearing for paste.
|
||||
pub fn focused_control_class() -> Option<String> {
|
||||
unsafe {
|
||||
// MTA per-thread init. Ignore HRESULT: S_OK / S_FALSE /
|
||||
// RPC_E_CHANGED_MODE are all benign for our uses here, and
|
||||
// we deliberately never call CoUninitialize (the Tauri
|
||||
// runtime thread lives for the life of the process, so
|
||||
// leaving COM init in place is fine).
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
|
||||
let automation: IUIAutomation =
|
||||
CoCreateInstance(&CUIAutomation, None::<&IUnknown>, CLSCTX_INPROC_SERVER).ok()?;
|
||||
let element: IUIAutomationElement = automation.GetFocusedElement().ok()?;
|
||||
// UIAutomationElement's CurrentClassName allocates a BSTR
|
||||
// the caller has to drop. `BSTR` in `windows` crate is a
|
||||
// Drop-wrapped owned string, so just returning `.to_string()`
|
||||
// is safe.
|
||||
let class: BSTR = element.CurrentClassName().ok()?;
|
||||
let s = class.to_string();
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a visible top-level window owned by `pid`. Returns the first
|
||||
/// match via `EnumWindows`. Top-level ≡ no owner window.
|
||||
pub fn find_top_level_window(pid: u32) -> Option<HWND> {
|
||||
struct Ctx {
|
||||
target_pid: u32,
|
||||
found: Option<HWND>,
|
||||
}
|
||||
let mut ctx = Ctx {
|
||||
target_pid: pid,
|
||||
found: None,
|
||||
};
|
||||
unsafe extern "system" fn callback(hwnd: HWND, lparam: LPARAM) -> BOOL {
|
||||
let ctx = &mut *(lparam.0 as *mut Ctx);
|
||||
if hwnd_pid(hwnd) != ctx.target_pid {
|
||||
return BOOL(1);
|
||||
}
|
||||
// Skip tool windows / invisible shells. `GetWindow(GW_OWNER)`
|
||||
// is non-null for modal dialogs and other secondary windows;
|
||||
// we want the real app frame, which has no owner.
|
||||
if !IsWindowVisible(hwnd).as_bool() {
|
||||
return BOOL(1);
|
||||
}
|
||||
if !GetWindow(hwnd, GW_OWNER).unwrap_or(HWND(std::ptr::null_mut())).is_invalid() {
|
||||
return BOOL(1);
|
||||
}
|
||||
ctx.found = Some(hwnd);
|
||||
BOOL(0)
|
||||
}
|
||||
unsafe {
|
||||
let _ = EnumWindows(
|
||||
Some(callback),
|
||||
LPARAM(&mut ctx as *mut _ as isize),
|
||||
);
|
||||
}
|
||||
ctx.found
|
||||
}
|
||||
|
||||
/// Bring `hwnd` to the foreground reliably.
|
||||
///
|
||||
/// Plain `SetForegroundWindow` loses to Windows' foreground-lock
|
||||
/// rules — when our process isn't already foreground it can't hand
|
||||
/// focus to another app. The documented workaround is to attach the
|
||||
/// current thread's input queue to the current foreground window's
|
||||
/// thread for the duration of the call, which temporarily lets us
|
||||
/// share that thread's "last user activity" stamp.
|
||||
pub fn activate_hwnd(hwnd: HWND) -> Result<(), String> {
|
||||
unsafe {
|
||||
let fg = GetForegroundWindow();
|
||||
if fg == hwnd {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let our_thread = GetCurrentThreadId();
|
||||
let fg_thread = if fg.is_invalid() {
|
||||
0
|
||||
} else {
|
||||
let mut _pid: u32 = 0;
|
||||
GetWindowThreadProcessId(fg, Some(&mut _pid as *mut _))
|
||||
};
|
||||
|
||||
let attached = fg_thread != 0
|
||||
&& fg_thread != our_thread
|
||||
&& AttachThreadInput(our_thread, fg_thread, true).as_bool();
|
||||
|
||||
let ok = SetForegroundWindow(hwnd).as_bool();
|
||||
|
||||
if attached {
|
||||
let _ = AttachThreadInput(our_thread, fg_thread, false);
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return Err(format!(
|
||||
"SetForegroundWindow failed for HWND {:?} — Windows foreground-lock may have denied the activation.",
|
||||
hwnd.0
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
|
||||
|
||||
unsafe {
|
||||
let hwnd = GetForegroundWindow();
|
||||
if hwnd.is_invalid() {
|
||||
return Err(
|
||||
"GetForegroundWindow returned null — the desktop has no focused window (secure attention sequence, lock screen, or no user session)."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let pid = win::hwnd_pid(hwnd);
|
||||
if pid == 0 {
|
||||
return Err("GetWindowThreadProcessId returned PID 0 for the foreground window".into());
|
||||
}
|
||||
let bundle_id = win::exe_basename(pid);
|
||||
let role = win::focused_control_class();
|
||||
Ok(FocusSnapshot {
|
||||
pid: pid as i32,
|
||||
bundle_id,
|
||||
role,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn activate_pid(pid: i32) -> Result<(), String> {
|
||||
if pid <= 0 {
|
||||
return Err(format!("Cannot activate invalid PID {pid}"));
|
||||
}
|
||||
let hwnd = win::find_top_level_window(pid as u32)
|
||||
.ok_or_else(|| format!("No visible top-level window for PID {pid}"))?;
|
||||
win::activate_hwnd(hwnd)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
Err("focus capture is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn activate_pid(_pid: i32) -> Result<(), String> {
|
||||
Err("app activation is not yet implemented on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//! Global hotkey → dictation effect bridge.
|
||||
//!
|
||||
//! Thin adapter from `keytap::chord::ChordMatcher` to Tauri events. keytap
|
||||
//! owns the OS event tap + the chord state machine (Momentary vs Toggle,
|
||||
//! longest-match resolution, sticky-toggle semantics); this module's only
|
||||
//! job is:
|
||||
//!
|
||||
//! 1. Build a `ChordMatcher` from the user's saved PTT + Toggle chords.
|
||||
//! 2. Translate `ChordEvent` → voicebox's [`Effect`] on a dispatcher
|
||||
//! thread.
|
||||
//! 3. Fan [`Effect`]s out into Tauri events + dictate-window show/hide.
|
||||
//!
|
||||
//! The [`Effect::RestartRecording`] signal is emitted when keytap fires
|
||||
//! `End(PTT)` and `Start(Toggle)` with the *same* [`Instant`] — which
|
||||
//! happens when the held set upgrades from a shorter chord to a longer
|
||||
//! superset in a single event (the classic PTT→hands-free transition).
|
||||
//! We detect the pair with a 5 ms peek on the matcher's receiver and
|
||||
//! coalesce into one `Restart` so hosts can discard the transition-
|
||||
//! moment audio rather than treat it as an unrelated Stop+Start pair.
|
||||
//!
|
||||
//! Left- and right-hand modifier variants are kept distinct all the way
|
||||
//! down to the OS event tap (keytap's core promise). Defaults bind to
|
||||
//! right-hand Cmd + right-hand Option on macOS / right-hand Ctrl +
|
||||
//! right-hand Shift on Windows so the usual left-hand shortcuts stay
|
||||
//! with the OS / app.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::Duration;
|
||||
|
||||
use keytap::chord::{Chord, ChordEvent, ChordMatcher};
|
||||
use keytap::{Key, RecvTimeoutError};
|
||||
use tauri::{AppHandle, Emitter, Manager};
|
||||
|
||||
use crate::focus_capture;
|
||||
use crate::DICTATE_WINDOW_LABEL;
|
||||
|
||||
// ========================================================================
|
||||
// Public types
|
||||
// ========================================================================
|
||||
|
||||
/// Semantic action a chord can be bound to. `PushToTalk` = hold chord to
|
||||
/// record, release to stop. `ToggleToTalk` = press chord to start recording,
|
||||
/// press again to stop.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ChordAction {
|
||||
PushToTalk,
|
||||
ToggleToTalk,
|
||||
}
|
||||
|
||||
/// Effect produced after the chord matcher resolves an event. Hosts
|
||||
/// translate these into UI / recorder calls.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Effect {
|
||||
StartRecording(ChordAction),
|
||||
StopRecording(ChordAction),
|
||||
/// Emitted when a push-to-talk chord is "upgraded" into the toggle
|
||||
/// chord mid-hold — hosts may want to discard the captured audio and
|
||||
/// restart so the transition moment isn't in the recording.
|
||||
RestartRecording(ChordAction),
|
||||
}
|
||||
|
||||
/// Chord key sets from capture settings. Both actions use the same
|
||||
/// `HashSet<Key>` shape so callers don't need to know about keytap's
|
||||
/// `Chord` type.
|
||||
pub type Bindings = HashMap<ChordAction, HashSet<Key>>;
|
||||
|
||||
// ========================================================================
|
||||
// Monitor
|
||||
// ========================================================================
|
||||
|
||||
pub struct HotkeyMonitor {
|
||||
app: AppHandle,
|
||||
active: Option<Active>,
|
||||
}
|
||||
|
||||
struct Active {
|
||||
dispatcher: JoinHandle<()>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl HotkeyMonitor {
|
||||
/// Build the monitor with initial bindings. Equivalent to constructing
|
||||
/// an empty monitor and calling [`Self::update_bindings`] once.
|
||||
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
||||
let mut m = Self { app, active: None };
|
||||
m.apply(bindings);
|
||||
m
|
||||
}
|
||||
|
||||
/// Swap in a fresh set of chord bindings. Tears down the existing
|
||||
/// `ChordMatcher` (which stops keytap's chord worker thread and
|
||||
/// closes the OS tap) and spawns a new one. No-op for the "all
|
||||
/// empty" case so "disable hotkey" doesn't keep a tap running for
|
||||
/// no reason.
|
||||
pub fn update_bindings(&mut self, bindings: Bindings) {
|
||||
self.apply(bindings);
|
||||
}
|
||||
|
||||
fn apply(&mut self, bindings: Bindings) {
|
||||
// Tear down any existing matcher + dispatcher first. The
|
||||
// dispatcher sees the shutdown flag on its next recv_timeout
|
||||
// (≤100ms) and returns; joining waits for that. Dropping the
|
||||
// ChordMatcher stops keytap's chord-worker thread and the
|
||||
// underlying Tap.
|
||||
if let Some(active) = self.active.take() {
|
||||
active.shutdown.store(true, Ordering::Relaxed);
|
||||
let _ = active.dispatcher.join();
|
||||
}
|
||||
|
||||
if bindings.values().all(|set| set.is_empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
let matcher = match build_matcher(&bindings) {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
eprintln!(
|
||||
"HotkeyMonitor: ChordMatcher build failed ({err}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch."
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let shutdown = Arc::new(AtomicBool::new(false));
|
||||
let shutdown_for_thread = shutdown.clone();
|
||||
let app = self.app.clone();
|
||||
let dispatcher = thread::Builder::new()
|
||||
.name("voicebox-hotkey-dispatcher".into())
|
||||
.spawn(move || dispatcher_loop(app, matcher, shutdown_for_thread))
|
||||
.expect("spawn hotkey dispatcher thread");
|
||||
|
||||
self.active = Some(Active { dispatcher, shutdown });
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for HotkeyMonitor {
|
||||
fn drop(&mut self) {
|
||||
if let Some(active) = self.active.take() {
|
||||
active.shutdown.store(true, Ordering::Relaxed);
|
||||
let _ = active.dispatcher.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Matcher construction + dispatch
|
||||
// ========================================================================
|
||||
|
||||
fn build_matcher(bindings: &Bindings) -> Result<ChordMatcher<ChordAction>, keytap::Error> {
|
||||
let mut builder = ChordMatcher::builder();
|
||||
if let Some(keys) = bindings.get(&ChordAction::PushToTalk) {
|
||||
if !keys.is_empty() {
|
||||
builder = builder.add(
|
||||
ChordAction::PushToTalk,
|
||||
Chord::of(keys.iter().copied()),
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Some(keys) = bindings.get(&ChordAction::ToggleToTalk) {
|
||||
if !keys.is_empty() {
|
||||
builder = builder.add_toggle(
|
||||
ChordAction::ToggleToTalk,
|
||||
Chord::of(keys.iter().copied()),
|
||||
);
|
||||
}
|
||||
}
|
||||
builder.build()
|
||||
}
|
||||
|
||||
fn dispatcher_loop(
|
||||
app: AppHandle,
|
||||
matcher: ChordMatcher<ChordAction>,
|
||||
shutdown: Arc<AtomicBool>,
|
||||
) {
|
||||
while !shutdown.load(Ordering::Relaxed) {
|
||||
match matcher.recv_timeout(Duration::from_millis(100)) {
|
||||
Ok(event) => process_event(&app, &matcher, event),
|
||||
Err(RecvTimeoutError::Timeout) => continue,
|
||||
Err(RecvTimeoutError::Disconnected) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn a single [`ChordEvent`] into zero or one [`Effect`]s, peeking at
|
||||
/// the matcher once for a same-Instant follow-up so upgrade transitions
|
||||
/// coalesce into [`Effect::RestartRecording`] instead of a Stop+Start
|
||||
/// pair.
|
||||
fn process_event(
|
||||
app: &AppHandle,
|
||||
matcher: &ChordMatcher<ChordAction>,
|
||||
event: ChordEvent<ChordAction>,
|
||||
) {
|
||||
match event {
|
||||
ChordEvent::Start { id, .. } => {
|
||||
apply_effect(app, Effect::StartRecording(id));
|
||||
}
|
||||
ChordEvent::End { id: end_id, time: end_time } => {
|
||||
// Peek for an immediately-following Start. keytap emits
|
||||
// End+Start atomically (same Instant) when the held set
|
||||
// transitions between registered chords — our 5 ms window
|
||||
// is well under perceptible latency but far longer than the
|
||||
// channel hop between keytap's chord worker and our
|
||||
// dispatcher.
|
||||
match matcher.recv_timeout(Duration::from_millis(5)) {
|
||||
Ok(ChordEvent::Start { id: start_id, time: start_time })
|
||||
if start_time == end_time =>
|
||||
{
|
||||
apply_effect(app, Effect::RestartRecording(start_id));
|
||||
}
|
||||
Ok(other) => {
|
||||
apply_effect(app, Effect::StopRecording(end_id));
|
||||
// The peeked event wasn't a transition partner;
|
||||
// process it in its own right. Recursion depth is
|
||||
// bounded by the number of back-to-back chord
|
||||
// events, in practice 1–2.
|
||||
process_event(app, matcher, other);
|
||||
}
|
||||
Err(_) => {
|
||||
apply_effect(app, Effect::StopRecording(end_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Effect → Tauri
|
||||
// ========================================================================
|
||||
|
||||
fn apply_effect(app: &AppHandle, effect: Effect) {
|
||||
match effect {
|
||||
Effect::StartRecording(_) => {
|
||||
// Snapshot focus BEFORE we touch the window — any AppKit
|
||||
// reshuffle triggered by set_position / show could in principle
|
||||
// steal key focus and poison the reading. In practice those
|
||||
// calls leave keyWindow alone, but capturing first is free.
|
||||
let focus = focus_capture::capture_focus().ok();
|
||||
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
// The previous hide-cycle parked the window off-screen and
|
||||
// made it click-through — undo both before showing, so the
|
||||
// pill lands at top-center and the user can actually click
|
||||
// the error pill / stop button.
|
||||
//
|
||||
// `current_monitor()` returns None when the window is off
|
||||
// any display (our hide handler parks it at -10_000, -10_000
|
||||
// precisely so it never intercepts clicks), so fall back to
|
||||
// the primary monitor for the reposition.
|
||||
let monitor = window
|
||||
.current_monitor()
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| window.primary_monitor().ok().flatten());
|
||||
if let Some(monitor) = monitor {
|
||||
let monitor_pos = monitor.position();
|
||||
let monitor_size = monitor.size();
|
||||
if let Ok(win_size) = window.outer_size() {
|
||||
let x = monitor_pos.x
|
||||
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
|
||||
}
|
||||
}
|
||||
// Skip on Linux: aborts if the window was never realized
|
||||
// (see show_dictate_window in main.rs).
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = window.set_ignore_cursor_events(false);
|
||||
// Deliberately no set_focus() — taking key focus would yank
|
||||
// it out of whatever app the user was typing in, which is
|
||||
// the opposite of what a dictation overlay should do.
|
||||
let _ = window.show();
|
||||
let payload = serde_json::json!({ "focus": focus });
|
||||
let _ = window.emit("dictate:start", payload);
|
||||
}
|
||||
}
|
||||
Effect::StopRecording(_) => {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
let _ = window.emit("dictate:stop", ());
|
||||
}
|
||||
}
|
||||
Effect::RestartRecording(_) => {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
let _ = window.emit("dictate:restart", ());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Platform permission gate for the global keyboard tap.
|
||||
//!
|
||||
//! On macOS 10.15+, creating a CGEventTap that observes keyboard events
|
||||
//! requires the host process to be listed under System Settings → Privacy &
|
||||
//! Security → Input Monitoring. Without that trust, keytap's `Tap` returns
|
||||
//! a permission error and no key events ever flow through the chord engine.
|
||||
//!
|
||||
//! The relevant TCC pair lives in IOKit, mirroring `AXIsProcessTrusted` /
|
||||
//! `AXIsProcessTrustedWithOptions` on the Accessibility side:
|
||||
//!
|
||||
//! - `IOHIDCheckAccess(kIOHIDRequestTypeListenEvent)` — read the current
|
||||
//! grant without prompting. We call this from the Captures settings UI
|
||||
//! so the row can show "granted" / "missing" without surprising the user.
|
||||
//! - `IOHIDRequestAccess(kIOHIDRequestTypeListenEvent)` — fire the
|
||||
//! "Voicebox would like to receive keystrokes from any application"
|
||||
//! dialog and add Voicebox to the Input Monitoring pane (toggle off).
|
||||
//! Returns true when access is already granted; otherwise returns false
|
||||
//! and queues the prompt. The user still has to flip the toggle on; this
|
||||
//! just gets us into the list.
|
||||
//!
|
||||
//! `enable_hotkey` calls `request` on first invocation so the prompt fires
|
||||
//! from a deterministic, user-initiated point (the Captures toggle) instead
|
||||
//! of as a side-effect of keytap's `Tap` creating its CGEventTap.
|
||||
//!
|
||||
//! Windows / Linux don't gate keyboard taps behind a TCC-style permission,
|
||||
//! so those branches return `true`.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use std::os::raw::c_uint;
|
||||
|
||||
/// `kIOHIDRequestTypeListenEvent` from `<IOKit/hidsystem/IOHIDLib.h>` —
|
||||
/// the request-type discriminator for "I want to read keyboard / mouse
|
||||
/// events created by other processes."
|
||||
pub const REQUEST_TYPE_LISTEN_EVENT: c_uint = 1;
|
||||
|
||||
/// `kIOHIDAccessTypeGranted` from `IOHIDLib.h`. The other values are
|
||||
/// `Denied = 1` and `Unknown = 2`; we only ever care about the granted
|
||||
/// case so they don't get their own constants.
|
||||
pub const ACCESS_TYPE_GRANTED: c_uint = 0;
|
||||
|
||||
#[link(name = "IOKit", kind = "framework")]
|
||||
extern "C" {
|
||||
/// Returns the current access state as an `IOHIDAccessType` enum
|
||||
/// (Granted=0, Denied=1, Unknown=2). No prompt side-effect.
|
||||
///
|
||||
/// Declared as `c_uint` rather than `bool`: the C signature returns
|
||||
/// the full enum, and reading a 3-valued enum into Rust's 1-bit
|
||||
/// `bool` is undefined behaviour that silently inverts our gate.
|
||||
pub fn IOHIDCheckAccess(request_type: c_uint) -> c_uint;
|
||||
|
||||
/// Returns true when access is already granted; otherwise queues
|
||||
/// the system prompt and returns false synchronously. Safe to call
|
||||
/// repeatedly — once the entry exists in the Input Monitoring pane
|
||||
/// macOS won't re-prompt. Real `Boolean` (UInt8) return on the C
|
||||
/// side, so `bool` here is correct.
|
||||
pub fn IOHIDRequestAccess(request_type: c_uint) -> bool;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn is_trusted() -> bool {
|
||||
unsafe { ffi::IOHIDCheckAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) == ffi::ACCESS_TYPE_GRANTED }
|
||||
}
|
||||
|
||||
/// Fire the Input Monitoring prompt if not already granted. Returns the
|
||||
/// current grant state; a `false` here means the prompt was queued and the
|
||||
/// user needs to flip the toggle in System Settings before key events flow.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn request() -> bool {
|
||||
unsafe { ffi::IOHIDRequestAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) }
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn is_trusted() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn request() -> bool {
|
||||
true
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Stable string ↔ `keytap::Key` mapping for chord persistence.
|
||||
//!
|
||||
//! The frontend captures keypresses through the browser keyboard API (which
|
||||
//! exposes `event.code` like `"MetaRight"`, `"AltRight"`, `"Space"`, `"KeyA"`)
|
||||
//! and stores chords in capture_settings as JSON arrays of canonical names.
|
||||
//! On the way back the same names need to round-trip into `keytap::Key`
|
||||
//! variants the chord engine actually matches against.
|
||||
//!
|
||||
//! Input strings follow the W3C `KeyboardEvent.code` identifiers exactly —
|
||||
//! `"MetaRight"`, `"AltRight"`, `"KeyA"`, `"Digit0"`, `"ArrowUp"`, … —
|
||||
//! which is also what the browser emits natively, so on-disk chords
|
||||
//! round-trip without translation on the frontend side. Legacy aliases
|
||||
//! (`"Alt"` / `"AltGr"` / `"Num0"` / `"UpArrow"` / …) are accepted too so
|
||||
//! older capture_settings rows written before the keytap swap keep working.
|
||||
|
||||
use keytap::Key;
|
||||
|
||||
/// Resolve a canonical key name to its `keytap::Key`. Returns `None` for
|
||||
/// names that don't have a corresponding variant — the command surface
|
||||
/// rejects those so we never silently drop keys from a chord.
|
||||
pub fn key_from_str(name: &str) -> Option<Key> {
|
||||
Some(match name {
|
||||
// Modifiers — left/right distinction matters for chord defaults.
|
||||
"AltLeft" | "Alt" => Key::AltLeft,
|
||||
"AltRight" | "AltGr" => Key::AltRight,
|
||||
"ControlLeft" => Key::ControlLeft,
|
||||
"ControlRight" => Key::ControlRight,
|
||||
"MetaLeft" => Key::MetaLeft,
|
||||
"MetaRight" => Key::MetaRight,
|
||||
"ShiftLeft" => Key::ShiftLeft,
|
||||
"ShiftRight" => Key::ShiftRight,
|
||||
"CapsLock" => Key::CapsLock,
|
||||
"Function" => Key::Function,
|
||||
|
||||
// Whitespace / navigation
|
||||
"Space" => Key::Space,
|
||||
"Tab" => Key::Tab,
|
||||
"Enter" | "Return" => Key::Enter,
|
||||
"Backspace" => Key::Backspace,
|
||||
"Delete" => Key::Delete,
|
||||
"Escape" => Key::Escape,
|
||||
"Insert" => Key::Insert,
|
||||
"Home" => Key::Home,
|
||||
"End" => Key::End,
|
||||
"PageUp" => Key::PageUp,
|
||||
"PageDown" => Key::PageDown,
|
||||
"ArrowUp" | "UpArrow" => Key::ArrowUp,
|
||||
"ArrowDown" | "DownArrow" => Key::ArrowDown,
|
||||
"ArrowLeft" | "LeftArrow" => Key::ArrowLeft,
|
||||
"ArrowRight" | "RightArrow" => Key::ArrowRight,
|
||||
|
||||
// Function row
|
||||
"F1" => Key::F1, "F2" => Key::F2, "F3" => Key::F3, "F4" => Key::F4,
|
||||
"F5" => Key::F5, "F6" => Key::F6, "F7" => Key::F7, "F8" => Key::F8,
|
||||
"F9" => Key::F9, "F10" => Key::F10, "F11" => Key::F11, "F12" => Key::F12,
|
||||
|
||||
// Digits
|
||||
"Digit0" | "Num0" => Key::Digit0,
|
||||
"Digit1" | "Num1" => Key::Digit1,
|
||||
"Digit2" | "Num2" => Key::Digit2,
|
||||
"Digit3" | "Num3" => Key::Digit3,
|
||||
"Digit4" | "Num4" => Key::Digit4,
|
||||
"Digit5" | "Num5" => Key::Digit5,
|
||||
"Digit6" | "Num6" => Key::Digit6,
|
||||
"Digit7" | "Num7" => Key::Digit7,
|
||||
"Digit8" | "Num8" => Key::Digit8,
|
||||
"Digit9" | "Num9" => Key::Digit9,
|
||||
|
||||
// Letters — browser emits "KeyA"; keytap uses the bare letter.
|
||||
"KeyA" => Key::A, "KeyB" => Key::B, "KeyC" => Key::C,
|
||||
"KeyD" => Key::D, "KeyE" => Key::E, "KeyF" => Key::F,
|
||||
"KeyG" => Key::G, "KeyH" => Key::H, "KeyI" => Key::I,
|
||||
"KeyJ" => Key::J, "KeyK" => Key::K, "KeyL" => Key::L,
|
||||
"KeyM" => Key::M, "KeyN" => Key::N, "KeyO" => Key::O,
|
||||
"KeyP" => Key::P, "KeyQ" => Key::Q, "KeyR" => Key::R,
|
||||
"KeyS" => Key::S, "KeyT" => Key::T, "KeyU" => Key::U,
|
||||
"KeyV" => Key::V, "KeyW" => Key::W, "KeyX" => Key::X,
|
||||
"KeyY" => Key::Y, "KeyZ" => Key::Z,
|
||||
|
||||
// Punctuation / symbols
|
||||
"Backquote" | "BackQuote" => Key::Backtick,
|
||||
"Minus" => Key::Minus,
|
||||
"Equal" => Key::Equal,
|
||||
"BracketLeft" | "LeftBracket" => Key::BracketLeft,
|
||||
"BracketRight" | "RightBracket" => Key::BracketRight,
|
||||
"Semicolon" | "SemiColon" => Key::Semicolon,
|
||||
"Quote" => Key::Quote,
|
||||
"Backslash" | "BackSlash" => Key::Backslash,
|
||||
"Comma" => Key::Comma,
|
||||
"Period" | "Dot" => Key::Period,
|
||||
"Slash" => Key::Slash,
|
||||
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
//! Layout-aware resolution of the keycode whose current-layout translation
|
||||
//! is `'v'`. Drives [`crate::synthetic_keys::send_paste`] so the synthetic
|
||||
//! Cmd+V it posts is interpreted as Paste by the focused app regardless of
|
||||
//! the user's active keyboard layout (Dvorak, Colemak, AZERTY, …).
|
||||
//!
|
||||
//! macOS apps process Cmd+V via NSMenu key equivalents, which match against
|
||||
//! `[NSEvent charactersIgnoringModifiers]` — i.e. the layout-translated
|
||||
//! character, not the raw keycode. Posting `kVK_ANSI_V` (= 9, the QWERTY V
|
||||
//! position) on Dvorak therefore produces Cmd+. and never triggers Paste.
|
||||
//!
|
||||
//! All TIS calls happen on the main thread: once at startup via [`init`]
|
||||
//! from Tauri's setup hook, and again from the
|
||||
//! `kTISNotifySelectedKeyboardInputSourceChanged` distributed notification
|
||||
//! (delivered to the main runloop). The hot path ([`paste_keycode_v`])
|
||||
//! only reads an [`AtomicU16`], so paste latency is unchanged.
|
||||
//!
|
||||
//! Windows is intentionally not covered here. `SendInput` with
|
||||
//! `wVk = VK_V` delivers `WM_KEYDOWN` to the target with `wParam = VK_V`
|
||||
//! regardless of the active layout — most Windows apps treat that as
|
||||
//! Ctrl+V. AutoHotkey relies on the same behaviour.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::sync::atomic::{AtomicU16, Ordering};
|
||||
|
||||
/// `kVK_ANSI_V` — the keycode for the physical V key on a US QWERTY
|
||||
/// layout. Used as the fallback whenever live resolution can't produce a
|
||||
/// better answer (no Unicode key layout data, lookup failure, non-macOS).
|
||||
#[cfg(target_os = "macos")]
|
||||
const FALLBACK_V_KEYCODE: u16 = 9;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
static V_KEYCODE: AtomicU16 = AtomicU16::new(FALLBACK_V_KEYCODE);
|
||||
|
||||
/// Returns the keycode whose current-layout translation is `'v'`. Falls
|
||||
/// back to `kVK_ANSI_V` when resolution hasn't run, the active input
|
||||
/// source carries no Unicode key layout data, or no keycode in the layout
|
||||
/// produces `v`.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn paste_keycode_v() -> u16 {
|
||||
V_KEYCODE.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn init() {
|
||||
macos::init();
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn init() {}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos {
|
||||
use super::{FALLBACK_V_KEYCODE, V_KEYCODE};
|
||||
use core_foundation_sys::base::CFRelease;
|
||||
use core_foundation_sys::data::{CFDataGetBytePtr, CFDataRef};
|
||||
use core_foundation_sys::dictionary::CFDictionaryRef;
|
||||
use core_foundation_sys::notification_center::{
|
||||
CFNotificationCenterAddObserver, CFNotificationCenterGetDistributedCenter,
|
||||
CFNotificationCenterRef, CFNotificationName,
|
||||
CFNotificationSuspensionBehaviorDeliverImmediately,
|
||||
};
|
||||
use core_foundation_sys::string::CFStringRef;
|
||||
use std::ffi::c_void;
|
||||
use std::ptr;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
type TISInputSourceRef = *mut c_void;
|
||||
|
||||
/// `kUCKeyActionDown`.
|
||||
const K_UC_KEY_ACTION_DOWN: u16 = 0;
|
||||
/// `kUCKeyTranslateNoDeadKeysMask` — collapse dead-key state machine so
|
||||
/// a single call gives us the bare character. V is never a dead key on
|
||||
/// any layout we care about, but the flag costs nothing and removes
|
||||
/// any chance of ambiguous output.
|
||||
const K_UC_KEY_TRANSLATE_NO_DEAD_KEYS_MASK: u32 = 1;
|
||||
/// Standard US-style virtual keycodes occupy 0..0x7F. We iterate the
|
||||
/// full range so non-US-extended layouts (ISO, JIS) can still be
|
||||
/// resolved if their `v` lives outside the ANSI range.
|
||||
const MAX_KEYCODE: u16 = 127;
|
||||
const TARGET_CHAR: u16 = b'v' as u16;
|
||||
|
||||
#[link(name = "Carbon", kind = "framework")]
|
||||
extern "C" {
|
||||
fn TISCopyCurrentKeyboardLayoutInputSource() -> TISInputSourceRef;
|
||||
fn TISGetInputSourceProperty(
|
||||
source: TISInputSourceRef,
|
||||
key: CFStringRef,
|
||||
) -> *mut c_void;
|
||||
fn LMGetKbdType() -> u8;
|
||||
fn UCKeyTranslate(
|
||||
keyboard_layout: *const u8,
|
||||
virtual_key_code: u16,
|
||||
key_action: u16,
|
||||
modifier_key_state: u32,
|
||||
keyboard_type: u32,
|
||||
key_translate_options: u32,
|
||||
dead_key_state: *mut u32,
|
||||
max_string_length: usize,
|
||||
actual_string_length: *mut usize,
|
||||
unicode_string: *mut u16,
|
||||
) -> i32;
|
||||
|
||||
static kTISPropertyUnicodeKeyLayoutData: CFStringRef;
|
||||
static kTISNotifySelectedKeyboardInputSourceChanged: CFStringRef;
|
||||
}
|
||||
|
||||
pub fn init() {
|
||||
resolve_into_cache();
|
||||
register_layout_change_observer();
|
||||
}
|
||||
|
||||
fn resolve_into_cache() {
|
||||
let kc = resolve_v_keycode().unwrap_or(FALLBACK_V_KEYCODE);
|
||||
V_KEYCODE.store(kc, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn resolve_v_keycode() -> Option<u16> {
|
||||
unsafe {
|
||||
let source = TISCopyCurrentKeyboardLayoutInputSource();
|
||||
if source.is_null() {
|
||||
return None;
|
||||
}
|
||||
let _src_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
||||
|
||||
let layout_data_ptr =
|
||||
TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
|
||||
if layout_data_ptr.is_null() {
|
||||
return None;
|
||||
}
|
||||
let layout_bytes = CFDataGetBytePtr(layout_data_ptr as CFDataRef);
|
||||
if layout_bytes.is_null() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let kbd_type = LMGetKbdType() as u32;
|
||||
|
||||
for keycode in 0..=MAX_KEYCODE {
|
||||
let mut dead_key_state: u32 = 0;
|
||||
let mut chars: [u16; 4] = [0; 4];
|
||||
let mut actual_len: usize = 0;
|
||||
let status = UCKeyTranslate(
|
||||
layout_bytes,
|
||||
keycode,
|
||||
K_UC_KEY_ACTION_DOWN,
|
||||
0, // no modifiers
|
||||
kbd_type,
|
||||
K_UC_KEY_TRANSLATE_NO_DEAD_KEYS_MASK,
|
||||
&mut dead_key_state,
|
||||
chars.len(),
|
||||
&mut actual_len,
|
||||
chars.as_mut_ptr(),
|
||||
);
|
||||
if status == 0 && actual_len == 1 && chars[0] == TARGET_CHAR {
|
||||
return Some(keycode);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" fn layout_changed(
|
||||
_center: CFNotificationCenterRef,
|
||||
_observer: *mut c_void,
|
||||
_name: CFNotificationName,
|
||||
_object: *const c_void,
|
||||
_user_info: CFDictionaryRef,
|
||||
) {
|
||||
resolve_into_cache();
|
||||
}
|
||||
|
||||
fn register_layout_change_observer() {
|
||||
unsafe {
|
||||
let center = CFNotificationCenterGetDistributedCenter();
|
||||
if center.is_null() {
|
||||
return;
|
||||
}
|
||||
CFNotificationCenterAddObserver(
|
||||
center,
|
||||
ptr::null(),
|
||||
layout_changed,
|
||||
kTISNotifySelectedKeyboardInputSourceChanged,
|
||||
ptr::null(),
|
||||
CFNotificationSuspensionBehaviorDeliverImmediately,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod audio_capture;
|
||||
@@ -0,0 +1,180 @@
|
||||
//! Rust-side subscriber for the backend `/events/speak` SSE stream.
|
||||
//!
|
||||
//! Owns the pill-window lifecycle for agent-initiated speech. The dictate
|
||||
//! webview used to do this itself via `EventSource`, but hidden WebKit
|
||||
//! windows on macOS throttle long-lived network connections, so speak events
|
||||
//! never reached the pill. Tauri's event bus, on the other hand, reliably
|
||||
//! delivers events to hidden webviews (the chord path proves it), so we
|
||||
//! subscribe here and fan out via `emit`.
|
||||
//!
|
||||
//! Flow:
|
||||
//! backend speak-start → show dictate window + emit("dictate:speak-start")
|
||||
//! backend speak-end → emit("dictate:speak-end")
|
||||
//! The pill webview handles the rest (audio playback, then emits
|
||||
//! `dictate:hide` back to Rust when the audio element's `ended` fires).
|
||||
//!
|
||||
//! Reconnect policy: idle-timeout + escalating backoff. The stream is
|
||||
//! infinite by design, so a successful round means "we were receiving
|
||||
//! frames and then the backend closed the connection" (typically a
|
||||
//! server restart) — reset backoff and reconnect quickly. A failure or
|
||||
//! a round that produced no frames escalates backoff up to a 30 s cap so
|
||||
//! long-term outages stop filling stderr with reconnect log lines.
|
||||
//!
|
||||
//! The idle timeout guards against the worst silent-failure mode: a
|
||||
//! backend that accepts the TCP connection but stops producing frames
|
||||
//! (deadlocked SSE endpoint, zombie process). Without a timeout the
|
||||
//! `chunk().await` blocks forever and the task never notices. The
|
||||
//! backend emits a `:ping` comment every 15 s, so 45 s without any data
|
||||
//! is a reliable signal the stream is dead.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use tauri::{AppHandle, Emitter};
|
||||
|
||||
use crate::{ensure_dictate_window, SERVER_PORT};
|
||||
|
||||
const INITIAL_BACKOFF: Duration = Duration::from_millis(500);
|
||||
const MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
/// Backend emits a `:ping` heartbeat every 15 s. Giving the stream 45 s
|
||||
/// of idle budget absorbs one missed heartbeat (slow GC pause, brief
|
||||
/// backend stall) without being so long that a truly dead stream blocks
|
||||
/// the pill from surfacing for minutes.
|
||||
const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
|
||||
pub fn spawn_speak_monitor(app: AppHandle) {
|
||||
tauri::async_runtime::spawn(async move {
|
||||
run(app).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn run(app: AppHandle) {
|
||||
let url = format!("http://127.0.0.1:{}/events/speak", SERVER_PORT);
|
||||
let client = match reqwest::Client::builder().build() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("speak_monitor: failed to build HTTP client: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut backoff = INITIAL_BACKOFF;
|
||||
let mut attempt: u32 = 0;
|
||||
|
||||
loop {
|
||||
let stream_result = stream_once(&client, &url, &app).await;
|
||||
let had_success = matches!(stream_result, Ok(true));
|
||||
|
||||
if had_success {
|
||||
backoff = INITIAL_BACKOFF;
|
||||
attempt = 0;
|
||||
} else {
|
||||
attempt += 1;
|
||||
let reason = match stream_result {
|
||||
Ok(_) => "stream closed without data".to_string(),
|
||||
Err(e) => format!("stream err: {e}"),
|
||||
};
|
||||
eprintln!(
|
||||
"speak_monitor: {reason} (attempt {attempt}, retry in {:?})",
|
||||
backoff
|
||||
);
|
||||
}
|
||||
|
||||
tokio::time::sleep(backoff).await;
|
||||
if !had_success {
|
||||
backoff = (backoff * 2).min(MAX_BACKOFF);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume the SSE stream until it closes or errors. Returns `Ok(true)`
|
||||
/// if at least one frame was received (the connection was genuinely
|
||||
/// productive), `Ok(false)` on a clean but empty close, and `Err` for
|
||||
/// any connection or parse failure.
|
||||
async fn stream_once(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
app: &AppHandle,
|
||||
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut resp = client
|
||||
.get(url)
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("speak_monitor: backend returned {}", resp.status()).into());
|
||||
}
|
||||
let mut buf = String::new();
|
||||
let mut saw_data = false;
|
||||
loop {
|
||||
let chunk = match tokio::time::timeout(STREAM_IDLE_TIMEOUT, resp.chunk()).await {
|
||||
Ok(Ok(Some(chunk))) => chunk,
|
||||
Ok(Ok(None)) => return Ok(saw_data),
|
||||
Ok(Err(e)) => return Err(Box::new(e)),
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"no data for {:?} (heartbeat should arrive every 15 s)",
|
||||
STREAM_IDLE_TIMEOUT
|
||||
)
|
||||
.into())
|
||||
}
|
||||
};
|
||||
saw_data = true;
|
||||
buf.push_str(std::str::from_utf8(&chunk)?);
|
||||
// sse-starlette emits CRLF framing; the spec also permits LF, so
|
||||
// handle either. Drain whichever separator appears first.
|
||||
loop {
|
||||
let crlf = buf.find("\r\n\r\n");
|
||||
let lf = buf.find("\n\n");
|
||||
let (end, sep_len) = match (crlf, lf) {
|
||||
(Some(c), Some(l)) if c <= l => (c, 4),
|
||||
(Some(c), None) => (c, 4),
|
||||
(_, Some(l)) => (l, 2),
|
||||
(None, None) => break,
|
||||
};
|
||||
let frame: String = buf.drain(..end + sep_len).collect();
|
||||
if let Some((event, data)) = parse_frame(&frame) {
|
||||
dispatch(app, &event, &data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a single SSE frame into (event_name, data_json).
|
||||
///
|
||||
/// Returns None for comment-only frames (lines starting with `:`) and
|
||||
/// for frames without a recognizable `event:` or `data:` line.
|
||||
fn parse_frame(frame: &str) -> Option<(String, String)> {
|
||||
let mut event: Option<String> = None;
|
||||
let mut data_lines: Vec<&str> = Vec::new();
|
||||
for line in frame.lines() {
|
||||
if line.is_empty() || line.starts_with(':') {
|
||||
continue;
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix("event:") {
|
||||
event = Some(rest.trim().to_string());
|
||||
} else if let Some(rest) = line.strip_prefix("data:") {
|
||||
data_lines.push(rest.trim_start());
|
||||
}
|
||||
}
|
||||
let event = event?;
|
||||
let data = data_lines.join("\n");
|
||||
Some((event, data))
|
||||
}
|
||||
|
||||
fn dispatch(app: &AppHandle, event: &str, data: &str) {
|
||||
match event {
|
||||
"speak-start" => {
|
||||
// Defensive for dev/restart paths where the setup-created pill
|
||||
// is not present — but don't *show* it here. The pill
|
||||
// surfaces itself from `audio.onplaying` via `dictate:show`, so
|
||||
// users never see the empty-silent generation window.
|
||||
ensure_dictate_window(app);
|
||||
let _ = app.emit("dictate:speak-start", data.to_string());
|
||||
}
|
||||
"speak-end" => {
|
||||
let _ = app.emit("dictate:speak-end", data.to_string());
|
||||
}
|
||||
// `ready` and `ping` are heartbeats; ignore.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! Synthetic keyboard event posting for the auto-paste pipeline.
|
||||
//!
|
||||
//! `send_paste` fires the four-event paste sequence onto the OS input
|
||||
//! pipeline so the focused app performs its native paste action against
|
||||
//! whatever the clipboard module has just staged.
|
||||
//!
|
||||
//! - **macOS** — Cmd down with Cmd flag, V down with Cmd flag, V up with
|
||||
//! Cmd flag, Cmd up via `CGEventPost` at `kCGHIDEventTap`. The Cmd-down
|
||||
//! event carries the Command flag so its `flagsChanged` representation
|
||||
//! matches hardware — Electron/Chromium tracks modifier state from that
|
||||
//! flag and drops the paste otherwise (see the note on the event table).
|
||||
//! Accessibility permission is load-bearing: without it the system
|
||||
//! swallows the events silently, so callers must gate on
|
||||
//! [`crate::accessibility::is_trusted`].
|
||||
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
|
||||
//! permission gate, but UAC/UIPI blocks delivery into elevated target
|
||||
//! windows when we run non-elevated — nothing we can do short of also
|
||||
//! running elevated.
|
||||
//!
|
||||
//! On macOS the V keycode is resolved per-layout by
|
||||
//! [`crate::keyboard_layout`] — Cmd+V is matched against the layout-
|
||||
//! translated character via NSMenu key equivalents, so hardcoding
|
||||
//! `kVK_ANSI_V` (the QWERTY V position) would fire Cmd+. on Dvorak. The
|
||||
//! resolved keycode is read once per paste from an atomic; the cache is
|
||||
//! primed at startup and refreshed on layout change.
|
||||
//!
|
||||
//! Windows hardcodes `VK_V`. `SendInput` with `wVk = VK_V` makes the
|
||||
//! target receive `WM_KEYDOWN` with `wParam = VK_V` regardless of the
|
||||
//! active layout, and most Windows apps treat that as Ctrl+V (the same
|
||||
//! reason `Send "^v"` works in AutoHotkey on Dvorak Windows).
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[repr(C)]
|
||||
pub struct CGEvent {
|
||||
_opaque: [u8; 0],
|
||||
}
|
||||
pub type CGEventRef = *mut CGEvent;
|
||||
|
||||
#[repr(C)]
|
||||
pub struct CGEventSource {
|
||||
_opaque: [u8; 0],
|
||||
}
|
||||
pub type CGEventSourceRef = *mut CGEventSource;
|
||||
|
||||
pub type CGEventTapLocation = u32;
|
||||
pub type CGKeyCode = u16;
|
||||
pub type CGEventFlags = u64;
|
||||
pub type CGEventSourceStateID = i32;
|
||||
|
||||
/// `kCGHIDEventTap` — posted events enter at the HID level so every
|
||||
/// downstream tap (including the target app) sees them exactly as if the
|
||||
/// hardware had produced them.
|
||||
pub const K_CG_HID_EVENT_TAP: CGEventTapLocation = 0;
|
||||
|
||||
/// `kCGEventSourceStateHIDSystemState` — mimics hardware, which is what
|
||||
/// we want: modifier bookkeeping inside target apps stays consistent.
|
||||
pub const K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE: CGEventSourceStateID = 1;
|
||||
|
||||
/// `kCGEventFlagMaskCommand` — the Cmd modifier bit inside `CGEventFlags`.
|
||||
pub const K_CG_EVENT_FLAG_MASK_COMMAND: CGEventFlags = 0x00100000;
|
||||
|
||||
/// `kVK_Command` (left Cmd).
|
||||
pub const KEYCODE_LEFT_CMD: CGKeyCode = 0x37;
|
||||
|
||||
#[link(name = "CoreGraphics", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn CGEventSourceCreate(state_id: CGEventSourceStateID) -> CGEventSourceRef;
|
||||
pub fn CGEventCreateKeyboardEvent(
|
||||
source: CGEventSourceRef,
|
||||
virtual_key: CGKeyCode,
|
||||
key_down: bool,
|
||||
) -> CGEventRef;
|
||||
pub fn CGEventSetFlags(event: CGEventRef, flags: CGEventFlags);
|
||||
pub fn CGEventPost(tap: CGEventTapLocation, event: CGEventRef);
|
||||
}
|
||||
|
||||
#[link(name = "CoreFoundation", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn CFRelease(cf: *const c_void);
|
||||
}
|
||||
}
|
||||
|
||||
/// Post the four-event Cmd+V sequence to the HID event tap.
|
||||
///
|
||||
/// Returns after the events are queued — there's no completion callback,
|
||||
/// so callers should sleep briefly afterwards to let the target app
|
||||
/// process the paste before any follow-up (e.g. clipboard restore).
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
use ffi::*;
|
||||
|
||||
let v_keycode = crate::keyboard_layout::paste_keycode_v();
|
||||
|
||||
unsafe {
|
||||
let source = CGEventSourceCreate(K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE);
|
||||
if source.is_null() {
|
||||
return Err("CGEventSourceCreate returned null".into());
|
||||
}
|
||||
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
||||
|
||||
let events = [
|
||||
// The Cmd-down event must carry the Command flag itself. On real
|
||||
// hardware the Cmd keyDown is a flagsChanged event whose flags
|
||||
// already include Command; Chromium/Electron builds its tracked
|
||||
// modifier state from that flag. Posting Cmd-down with flags = 0
|
||||
// leaves that tracker showing "Command up", so the following V —
|
||||
// even though its own flags carry Command — matches neither the
|
||||
// Cmd+V accelerator (tracker says no modifier) nor plain-text
|
||||
// insertion (event flags say Command), and Electron drops it
|
||||
// silently. AppKit reads the V event's own flags and pastes
|
||||
// regardless, which is why native apps worked but Electron
|
||||
// targets (Slack, VS Code) silently no-op'd.
|
||||
(KEYCODE_LEFT_CMD, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(KEYCODE_LEFT_CMD, false, 0),
|
||||
];
|
||||
|
||||
// Build the four events up front so CFRelease happens after all posts.
|
||||
// Posting in a loop that interleaved create → post → release would
|
||||
// work, but keeping the events alive for the full sequence matches
|
||||
// the pattern CGEventPost's docs show and is easier to reason about.
|
||||
let mut guards = Vec::with_capacity(events.len());
|
||||
let mut created = Vec::with_capacity(events.len());
|
||||
|
||||
for (key, down, flags) in events {
|
||||
let event = CGEventCreateKeyboardEvent(source, key, down);
|
||||
if event.is_null() {
|
||||
return Err(format!(
|
||||
"CGEventCreateKeyboardEvent(key={}, down={}) returned null",
|
||||
key, down
|
||||
));
|
||||
}
|
||||
let guard = scopeguard::guard(event, |e| CFRelease(e as *const c_void));
|
||||
if flags != 0 {
|
||||
CGEventSetFlags(event, flags);
|
||||
}
|
||||
created.push(event);
|
||||
guards.push(guard);
|
||||
}
|
||||
|
||||
for event in created {
|
||||
CGEventPost(K_CG_HID_EVENT_TAP, event);
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
||||
INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_KEYUP,
|
||||
VIRTUAL_KEY,
|
||||
};
|
||||
|
||||
pub fn make_key(vk: VIRTUAL_KEY, up: bool) -> INPUT {
|
||||
let flags = if up {
|
||||
KEYEVENTF_KEYUP
|
||||
} else {
|
||||
KEYBD_EVENT_FLAGS(0)
|
||||
};
|
||||
INPUT {
|
||||
r#type: INPUT_KEYBOARD,
|
||||
Anonymous: INPUT_0 {
|
||||
ki: KEYBDINPUT {
|
||||
wVk: vk,
|
||||
wScan: 0,
|
||||
dwFlags: flags,
|
||||
time: 0,
|
||||
dwExtraInfo: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
||||
SendInput, INPUT, VK_CONTROL, VK_V,
|
||||
};
|
||||
|
||||
// Four-event Ctrl+V sequence. Matches the macOS CGEvent pattern: the
|
||||
// modifier brackets the letter so the target app sees a fully formed
|
||||
// accelerator rather than a lone V. `dwExtraInfo` is zero — we're not
|
||||
// tagging these as "ours" because no consumer in the paste path needs
|
||||
// to distinguish synthetic events from hardware ones.
|
||||
let events = [
|
||||
win::make_key(VK_CONTROL, false),
|
||||
win::make_key(VK_V, false),
|
||||
win::make_key(VK_V, true),
|
||||
win::make_key(VK_CONTROL, true),
|
||||
];
|
||||
|
||||
unsafe {
|
||||
let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);
|
||||
if sent as usize != events.len() {
|
||||
return Err(format!(
|
||||
"SendInput delivered {} of {} events — the input desktop may be locked (secure attention sequence) or a higher-integrity window is intercepting.",
|
||||
sent,
|
||||
events.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
Err("synthetic paste is not yet implemented on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "Voicebox",
|
||||
"version": "0.5.0",
|
||||
"identifier": "sh.voicebox.app",
|
||||
"build": {
|
||||
"beforeDevCommand": "bun run dev",
|
||||
"beforeBuildCommand": "bun run build",
|
||||
"frontendDist": "../dist",
|
||||
"devUrl": "http://localhost:5173"
|
||||
},
|
||||
"bundle": {
|
||||
"active": true,
|
||||
"targets": "all",
|
||||
"createUpdaterArtifacts": "v1Compatible",
|
||||
"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"],
|
||||
"icon": [
|
||||
"icons/32x32.png",
|
||||
"icons/128x128.png",
|
||||
"icons/[email protected]",
|
||||
"icons/icon.icns",
|
||||
"icons/icon.ico"
|
||||
],
|
||||
"macOS": {
|
||||
"frameworks": [],
|
||||
"minimumSystemVersion": "11.0",
|
||||
"infoPlist": "Info.plist",
|
||||
"entitlements": "Entitlements.plist"
|
||||
},
|
||||
"resources": {
|
||||
"gen/Assets.car": "./",
|
||||
"gen/voicebox.icns": "./",
|
||||
"gen/partial.plist": "./"
|
||||
}
|
||||
},
|
||||
"app": {
|
||||
"macOSPrivateApi": true,
|
||||
"security": {
|
||||
"csp": null,
|
||||
"capabilities": ["default"]
|
||||
},
|
||||
"windows": [
|
||||
{
|
||||
"title": "",
|
||||
"width": 1200,
|
||||
"height": 800,
|
||||
"minWidth": 800,
|
||||
"minHeight": 600,
|
||||
"resizable": true,
|
||||
"fullscreen": false,
|
||||
"devtools": true,
|
||||
"userAgent": null,
|
||||
"titleBarStyle": "Overlay"
|
||||
}
|
||||
],
|
||||
"withGlobalTauri": true
|
||||
},
|
||||
"plugins": {
|
||||
"shell": {
|
||||
"open": ".*"
|
||||
},
|
||||
"updater": {
|
||||
"pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IEUxRENBQkRBQjdBNTM1OTIKUldTU05hVzMycXZjNGJGcUxmcVVocll2QjdSaTJNdlFxR2M3VDJsMnVvbDdyZGRPMmRlOW9aWTcK",
|
||||
"endpoints": ["https://github.com/jamiepine/voicebox/releases/latest/download/latest.json"]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// NOTE: This test requires system audio to be playing during execution.
|
||||
// To run this test successfully:
|
||||
// 1. Start playing audio (music, video, etc.)
|
||||
// 2. Run: cargo test --test audio_capture_test -- --nocapture
|
||||
// 3. The test will capture audio for 5 seconds and verify the output
|
||||
|
||||
use voicebox::audio_capture::{AudioCaptureState, start_capture, stop_capture};
|
||||
use base64::Engine;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_system_audio_capture() {
|
||||
// Create AudioCaptureState
|
||||
let state = AudioCaptureState::new();
|
||||
|
||||
println!("Starting system audio capture with 5 second max duration...");
|
||||
|
||||
// Start capture with 5 second max duration
|
||||
let result = start_capture(&state, 5).await;
|
||||
|
||||
if let Err(e) = result {
|
||||
panic!("Failed to start capture: {}", e);
|
||||
}
|
||||
|
||||
println!("Capture started, waiting 5 seconds...");
|
||||
|
||||
// Wait 5 seconds for capture to complete
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
|
||||
|
||||
println!("Stopping capture...");
|
||||
|
||||
// Stop capture and get the result
|
||||
let audio_data = stop_capture(&state).await;
|
||||
|
||||
match audio_data {
|
||||
Ok(base64_wav) => {
|
||||
println!("Capture stopped successfully");
|
||||
|
||||
// Validate the returned base64 WAV data
|
||||
println!("Validating base64 WAV data...");
|
||||
|
||||
// Decode base64 to bytes
|
||||
let decoded_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&base64_wav)
|
||||
.expect("Failed to decode base64 data");
|
||||
|
||||
// Verify bytes array is not empty
|
||||
assert!(!decoded_bytes.is_empty(), "Decoded bytes array is empty");
|
||||
|
||||
// Confirm data has content (length > 0)
|
||||
println!("WAV data length: {} bytes", decoded_bytes.len());
|
||||
assert!(decoded_bytes.len() > 0, "WAV data has no content");
|
||||
|
||||
println!("✓ Test passed: Audio capture produced valid WAV data");
|
||||
}
|
||||
Err(e) => {
|
||||
panic!("Failed to stop capture or get audio data: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
|
||||
// import { ReactQueryDevtools } from '@tanstack/react-query-devtools';
|
||||
import App from '@/App';
|
||||
// Import CSS from app directory using alias so Tailwind can scan the source files
|
||||
import '@/index.css';
|
||||
import { PlatformProvider } from '@/platform/PlatformContext';
|
||||
import { tauriPlatform } from './platform';
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
staleTime: 1000 * 60 * 5, // 5 minutes
|
||||
gcTime: 1000 * 60 * 10, // 10 minutes
|
||||
retry: 1,
|
||||
refetchOnWindowFocus: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<PlatformProvider platform={tauriPlatform}>
|
||||
<App />
|
||||
{/* <ReactQueryDevtools initialIsOpen={false} /> */}
|
||||
</PlatformProvider>
|
||||
</QueryClientProvider>
|
||||
</React.StrictMode>,
|
||||
);
|
||||
@@ -0,0 +1,44 @@
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import type { PlatformAudio, AudioDevice } from '@/platform/types';
|
||||
|
||||
export const tauriAudio: PlatformAudio = {
|
||||
async isSystemAudioSupported(): Promise<boolean> {
|
||||
return await invoke<boolean>('is_system_audio_supported');
|
||||
},
|
||||
|
||||
async startSystemAudioCapture(maxDurationSecs: number): Promise<void> {
|
||||
await invoke('start_system_audio_capture', {
|
||||
maxDurationSecs,
|
||||
});
|
||||
},
|
||||
|
||||
async stopSystemAudioCapture(): Promise<Blob> {
|
||||
const base64Data = await invoke<string>('stop_system_audio_capture');
|
||||
|
||||
// Convert base64 to Blob
|
||||
const binaryString = atob(base64Data);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
return new Blob([bytes], { type: 'audio/wav' });
|
||||
},
|
||||
|
||||
async listOutputDevices(): Promise<AudioDevice[]> {
|
||||
return await invoke<AudioDevice[]>('list_audio_output_devices');
|
||||
},
|
||||
|
||||
async playToDevices(audioData: Uint8Array, deviceIds: string[]): Promise<void> {
|
||||
await invoke('play_audio_to_devices', {
|
||||
audioData: Array.from(audioData),
|
||||
deviceIds,
|
||||
});
|
||||
},
|
||||
|
||||
stopPlayback(): void {
|
||||
invoke('stop_audio_playback').catch((error) => {
|
||||
console.error('Failed to stop audio playback:', error);
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { FileFilter, PlatformFilesystem } from '@/platform/types';
|
||||
|
||||
export const tauriFilesystem: PlatformFilesystem = {
|
||||
async saveFile(filename: string, blob: Blob, filters?: FileFilter[]) {
|
||||
const { save } = await import('@tauri-apps/plugin-dialog');
|
||||
const { writeFile } = await import('@tauri-apps/plugin-fs');
|
||||
|
||||
const filePath = await save({
|
||||
defaultPath: filename,
|
||||
filters: filters || [],
|
||||
});
|
||||
|
||||
if (!filePath) return; // User cancelled the dialog
|
||||
|
||||
const resolvedPath =
|
||||
typeof filePath === 'string' ? filePath : (filePath as { path: string }).path;
|
||||
|
||||
if (!resolvedPath) {
|
||||
throw new Error('Failed to resolve save path from dialog');
|
||||
}
|
||||
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
await writeFile(resolvedPath, new Uint8Array(arrayBuffer));
|
||||
},
|
||||
|
||||
async openPath(path: string) {
|
||||
const { open } = await import('@tauri-apps/plugin-shell');
|
||||
await open(path);
|
||||
},
|
||||
|
||||
async pickDirectory(title: string) {
|
||||
const { open } = await import('@tauri-apps/plugin-dialog');
|
||||
const selected = await open({ directory: true, title });
|
||||
if (!selected) return null;
|
||||
const dir = typeof selected === 'string' ? selected : (selected as { path: string }).path;
|
||||
return dir || null;
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { Platform } from '@/platform/types';
|
||||
import { tauriFilesystem } from './filesystem';
|
||||
import { tauriUpdater } from './updater';
|
||||
import { tauriAudio } from './audio';
|
||||
import { tauriLifecycle } from './lifecycle';
|
||||
import { tauriMetadata } from './metadata';
|
||||
|
||||
export const tauriPlatform: Platform = {
|
||||
filesystem: tauriFilesystem,
|
||||
updater: tauriUpdater,
|
||||
audio: tauriAudio,
|
||||
lifecycle: tauriLifecycle,
|
||||
metadata: tauriMetadata,
|
||||
};
|
||||
@@ -0,0 +1,125 @@
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import { emit, listen } from '@tauri-apps/api/event';
|
||||
import type { PlatformLifecycle, ServerLogEntry } from '@/platform/types';
|
||||
|
||||
class TauriLifecycle implements PlatformLifecycle {
|
||||
onServerReady?: () => void;
|
||||
|
||||
async startServer(remote = false, modelsDir?: string | null): Promise<string> {
|
||||
try {
|
||||
const result = await invoke<string>('start_server', {
|
||||
remote,
|
||||
modelsDir: modelsDir ?? undefined,
|
||||
});
|
||||
console.log('Server started:', result);
|
||||
this.onServerReady?.();
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('Failed to start server:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async stopServer(): Promise<void> {
|
||||
try {
|
||||
await invoke('stop_server');
|
||||
console.log('Server stopped');
|
||||
} catch (error) {
|
||||
console.error('Failed to stop server:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async restartServer(modelsDir?: string | null): Promise<string> {
|
||||
try {
|
||||
const result = await invoke<string>('restart_server', {
|
||||
modelsDir: modelsDir ?? undefined,
|
||||
});
|
||||
console.log('Server restarted:', result);
|
||||
this.onServerReady?.();
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('Failed to restart server:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async setKeepServerRunning(keepRunning: boolean): Promise<void> {
|
||||
try {
|
||||
await invoke('set_keep_server_running', { keepRunning });
|
||||
} catch (error) {
|
||||
console.error('Failed to set keep server running setting:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async setBackendOverride(backend?: string | null): Promise<void> {
|
||||
try {
|
||||
await invoke('set_backend_override', { backend: backend ?? undefined });
|
||||
} catch (error) {
|
||||
console.error('Failed to set backend override:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async setupWindowCloseHandler(): Promise<void> {
|
||||
try {
|
||||
// Listen for window close request from Rust
|
||||
await listen<null>('window-close-requested', async () => {
|
||||
// Import store here to avoid circular dependency
|
||||
const { useServerStore } = await import('@/stores/serverStore');
|
||||
const keepRunning = useServerStore.getState().keepServerRunningOnClose;
|
||||
|
||||
// Check if server was started by this app instance
|
||||
// @ts-expect-error - accessing module-level variable from another module
|
||||
const serverStartedByApp = window.__voiceboxServerStartedByApp ?? false;
|
||||
|
||||
console.log(
|
||||
'[lifecycle] window-close-requested: keepRunning=%s, serverStartedByApp=%s',
|
||||
keepRunning,
|
||||
serverStartedByApp,
|
||||
);
|
||||
|
||||
if (!keepRunning && serverStartedByApp) {
|
||||
// Stop server before closing (only if we started it)
|
||||
try {
|
||||
await this.stopServer();
|
||||
} catch (error) {
|
||||
console.error('Failed to stop server on close:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Emit event back to Rust to allow close
|
||||
await emit('window-close-allowed');
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to setup window close handler:', error);
|
||||
}
|
||||
}
|
||||
|
||||
subscribeToServerLogs(callback: (entry: ServerLogEntry) => void): () => void {
|
||||
let disposed = false;
|
||||
let unlisten: (() => void) | null = null;
|
||||
|
||||
void listen<ServerLogEntry>('server-log', (event) => {
|
||||
callback(event.payload);
|
||||
})
|
||||
.then((fn) => {
|
||||
if (disposed) {
|
||||
fn();
|
||||
return;
|
||||
}
|
||||
unlisten = fn;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Failed to subscribe to server logs:', error);
|
||||
});
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
unlisten?.();
|
||||
unlisten = null;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const tauriLifecycle = new TauriLifecycle();
|
||||
@@ -0,0 +1,14 @@
|
||||
import { getVersion } from '@tauri-apps/api/app';
|
||||
import type { PlatformMetadata } from '@/platform/types';
|
||||
|
||||
export const tauriMetadata: PlatformMetadata = {
|
||||
async getVersion(): Promise<string> {
|
||||
try {
|
||||
return await getVersion();
|
||||
} catch (error) {
|
||||
console.error('Failed to get version:', error);
|
||||
return '0.1.0';
|
||||
}
|
||||
},
|
||||
isTauri: true,
|
||||
};
|
||||