mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 14:20:42 -07:00
feat(capture): dictation, personalities, 0.5.0
Ships the Capture release end to end. Global-hotkey dictation with synthetic paste into the focused app on macOS and Windows, an on-screen pill across recording / transcribing / refining, customizable push-to- talk and toggle chords, and an accessibility-permission prompt scoped to Settings → Captures with inline re-check feedback. Voice profiles gain optional personalities that power compose / rewrite / respond actions via a local Qwen3 LLM — shared with refinement, so there is one local LLM in the app, not two. Refinement hardened with deterministic Whisper-loop collapse before the LLM sees the transcript, per-capture flag snapshots for re-runs, and a ten-transcript evaluation harness across every bundled refinement size. Version bump 0.4.5 → 0.5.0. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
ed2eec591a
commit
87c582ad54
@@ -0,0 +1,42 @@
|
||||
//! Platform permission gate for the auto-paste pipeline.
|
||||
//!
|
||||
//! On macOS, posting synthetic keyboard events and reading focused-UI state
|
||||
//! via the AX API both require the host process to be listed under System
|
||||
//! Settings → Privacy & Security → Accessibility. Without that trust,
|
||||
//! `CGEventPost` silently drops events and `AXUIElementCopyAttributeValue`
|
||||
//! returns an error. We surface a boolean check up front so the paste
|
||||
//! pipeline can short-circuit with a clear "grant permission" message
|
||||
//! instead of running through the full save → write → post → restore dance
|
||||
//! with nothing to show for it.
|
||||
//!
|
||||
//! Windows has no equivalent user-facing permission — `SendInput` and
|
||||
//! UIAutomation work for any non-elevated target out of the box. (UAC /
|
||||
//! UIPI still blocks sending input *into* an elevated target window from a
|
||||
//! non-elevated process, but that's per-target, not a global switch, and
|
||||
//! there's no Settings pane to send users to.) So the Windows branch just
|
||||
//! returns `true`.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
/// Returns true when the current process is listed in Accessibility.
|
||||
/// No prompt side-effect.
|
||||
pub fn AXIsProcessTrusted() -> bool;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn is_trusted() -> bool {
|
||||
unsafe { ffi::AXIsProcessTrusted() }
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn is_trusted() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn is_trusted() -> bool {
|
||||
false
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
//! Snapshot / write / restore helpers around the system clipboard.
|
||||
//!
|
||||
//! Used by the auto-paste flow: before synthesising the paste accelerator
|
||||
//! into a foreign app we need to (1) remember what the user had on the
|
||||
//! clipboard, (2) stage our transcribed text, (3) paste, (4) put the
|
||||
//! original contents back. Missing step 4 turns every dictation into a
|
||||
//! silent clipboard-stomp.
|
||||
//!
|
||||
//! On **macOS** the snapshot walks `NSPasteboard.pasteboardItems` and
|
||||
//! copies every `(UTI, data)` pair into an owned `Vec<u8>`, so restore
|
||||
//! rebuilds the full multi-type payload — not just the plain-text
|
||||
//! fallback. Images, styled text, file-reference lists all survive the
|
||||
//! round-trip.
|
||||
//!
|
||||
//! On **Windows** the snapshot walks `EnumClipboardFormats` and copies the
|
||||
//! HGLOBAL payload for every advertised format. GDI-handle formats (DIB
|
||||
//! bitmap, metafile, enhanced metafile, palette), owner-display variants,
|
||||
//! and the private-/GDI-object format ranges are skipped — those can't be
|
||||
//! round-tripped across processes without synthesising the underlying
|
||||
//! kernel/GDI objects, which isn't worth the complexity for a dictation
|
||||
//! clipboard guard. CF_UNICODETEXT, CF_HDROP, CF_DIB (bitmap data in
|
||||
//! memory, not a handle), CF_DIBV5, and every registered format (HTML
|
||||
//! Format, Rich Text Format, FileGroupDescriptor, etc.) all survive.
|
||||
//!
|
||||
//! On **macOS** every entry point manages its own `NSAutoreleasePool`
|
||||
//! because the Tauri command runtime threads don't have one by default —
|
||||
//! without it, every autoreleased `NSString` / `NSData` we touch would
|
||||
//! leak for the life of the process. On Windows, HGLOBAL ownership
|
||||
//! transfers to the clipboard on `SetClipboardData` success, so we only
|
||||
//! free handles we allocated but didn't hand off.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::runtime::Object;
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::{class, msg_send, sel, sel_impl};
|
||||
|
||||
/// One full-fidelity snapshot of the general pasteboard. Hold on to the value
|
||||
/// until the paste has landed, then pass it to [`restore_clipboard`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClipboardSnapshot {
|
||||
/// Outer vec: pasteboard items. Inner: `(uti, raw bytes)` per type. We
|
||||
/// store the raw UTI string and the raw `NSData` payload so we can rebuild
|
||||
/// the item with `setData:forType:` without interpreting the contents.
|
||||
items: Vec<Vec<(String, Vec<u8>)>>,
|
||||
/// `NSPasteboard.changeCount` at the moment of capture. Incremented by AppKit
|
||||
/// on every mutation from any process, so a caller can decide whether a
|
||||
/// restore is still safe (change_count == expected) or whether someone
|
||||
/// else wrote to the clipboard in the interim and we should back off.
|
||||
change_count: i64,
|
||||
}
|
||||
|
||||
impl ClipboardSnapshot {
|
||||
pub fn change_count(&self) -> i64 {
|
||||
self.change_count
|
||||
}
|
||||
|
||||
pub fn item_count(&self) -> usize {
|
||||
self.items.len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
type Id = *mut Object;
|
||||
|
||||
/// RAII wrapper so the pool drains even on early return / `?` propagation.
|
||||
#[cfg(target_os = "macos")]
|
||||
struct AutoreleasePool {
|
||||
pool: Id,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl AutoreleasePool {
|
||||
unsafe fn new() -> Self {
|
||||
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
|
||||
let pool: Id = msg_send![pool, init];
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl Drop for AutoreleasePool {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _: () = msg_send![self.pool, drain];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an autoreleased `NSString` from a Rust `&str` without scanning for
|
||||
/// interior nulls (which is what `initWithUTF8String:` would require).
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string(s: &str) -> Id {
|
||||
// NSUTF8StringEncoding = 4.
|
||||
let obj: Id = msg_send![class!(NSString), alloc];
|
||||
let obj: Id = msg_send![
|
||||
obj,
|
||||
initWithBytes: s.as_ptr()
|
||||
length: s.len()
|
||||
encoding: 4u64
|
||||
];
|
||||
let _: () = msg_send![obj, autorelease];
|
||||
obj
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bytes: *const i8 = msg_send![s, UTF8String];
|
||||
if bytes.is_null() {
|
||||
return None;
|
||||
}
|
||||
std::ffi::CStr::from_ptr(bytes)
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn general_pasteboard() -> Result<Id, String> {
|
||||
let pb: Id = msg_send![class!(NSPasteboard), generalPasteboard];
|
||||
if pb.is_null() {
|
||||
return Err("NSPasteboard generalPasteboard returned nil".into());
|
||||
}
|
||||
Ok(pb)
|
||||
}
|
||||
|
||||
/// Read the pasteboard's current change count without snapshotting contents.
|
||||
///
|
||||
/// AppKit increments this every time any process writes to the general
|
||||
/// pasteboard, so it's a cheap way to detect "did someone clobber my staged
|
||||
/// text before the paste landed?".
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let c: i64 = msg_send![pb, changeCount];
|
||||
Ok(c)
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture every item on the general pasteboard into an owned snapshot.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let change_count: i64 = msg_send![pb, changeCount];
|
||||
|
||||
let items: Id = msg_send![pb, pasteboardItems];
|
||||
if items.is_null() {
|
||||
return Ok(ClipboardSnapshot {
|
||||
items: Vec::new(),
|
||||
change_count,
|
||||
});
|
||||
}
|
||||
|
||||
let count: usize = msg_send![items, count];
|
||||
let mut saved: Vec<Vec<(String, Vec<u8>)>> = Vec::with_capacity(count);
|
||||
|
||||
for i in 0..count {
|
||||
let item: Id = msg_send![items, objectAtIndex: i];
|
||||
if item.is_null() {
|
||||
continue;
|
||||
}
|
||||
let types: Id = msg_send![item, types];
|
||||
if types.is_null() {
|
||||
continue;
|
||||
}
|
||||
let type_count: usize = msg_send![types, count];
|
||||
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(type_count);
|
||||
for j in 0..type_count {
|
||||
let t: Id = msg_send![types, objectAtIndex: j];
|
||||
let Some(type_str) = ns_string_to_rust(t) else {
|
||||
continue;
|
||||
};
|
||||
let data: Id = msg_send![item, dataForType: t];
|
||||
if data.is_null() {
|
||||
// Type advertised but no concrete data (lazy provider).
|
||||
// Skipping is safer than trying to force it to materialise.
|
||||
continue;
|
||||
}
|
||||
let length: usize = msg_send![data, length];
|
||||
let bytes_ptr: *const u8 = msg_send![data, bytes];
|
||||
let bytes = if bytes_ptr.is_null() || length == 0 {
|
||||
Vec::new()
|
||||
} else {
|
||||
std::slice::from_raw_parts(bytes_ptr, length).to_vec()
|
||||
};
|
||||
pairs.push((type_str, bytes));
|
||||
}
|
||||
saved.push(pairs);
|
||||
}
|
||||
|
||||
Ok(ClipboardSnapshot {
|
||||
items: saved,
|
||||
change_count,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the pasteboard contents with a single plain-text string. Returns
|
||||
/// the post-write change count so a later restore can verify nothing else
|
||||
/// touched the clipboard in between.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn write_text(text: &str) -> Result<i64, String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let _new_count: i64 = msg_send![pb, clearContents];
|
||||
|
||||
let ns_text = ns_string(text);
|
||||
// `public.utf8-plain-text` is the raw UTI behind `NSPasteboardTypeString`
|
||||
// and works for every text-aware paste target we care about.
|
||||
let ns_type = ns_string("public.utf8-plain-text");
|
||||
let ok: bool = msg_send![pb, setString: ns_text forType: ns_type];
|
||||
if !ok {
|
||||
return Err("NSPasteboard setString:forType: returned NO".into());
|
||||
}
|
||||
|
||||
let after: i64 = msg_send![pb, changeCount];
|
||||
Ok(after)
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild the pasteboard from a snapshot, replacing whatever is on it now.
|
||||
///
|
||||
/// Does not consult the change count — callers that want safe restore should
|
||||
/// compare [`current_change_count`] against the value returned by
|
||||
/// [`write_text`] first.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let pb = general_pasteboard()?;
|
||||
let _: i64 = msg_send![pb, clearContents];
|
||||
|
||||
if snapshot.items.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let array: Id = msg_send![class!(NSMutableArray), array];
|
||||
|
||||
for pairs in &snapshot.items {
|
||||
let item: Id = msg_send![class!(NSPasteboardItem), alloc];
|
||||
let item: Id = msg_send![item, init];
|
||||
let _: () = msg_send![item, autorelease];
|
||||
|
||||
for (uti, bytes) in pairs {
|
||||
let ns_type = ns_string(uti);
|
||||
let data: Id = msg_send![
|
||||
class!(NSData),
|
||||
dataWithBytes: bytes.as_ptr()
|
||||
length: bytes.len()
|
||||
];
|
||||
let _ok: bool = msg_send![item, setData: data forType: ns_type];
|
||||
}
|
||||
|
||||
let _: () = msg_send![array, addObject: item];
|
||||
}
|
||||
|
||||
let ok: bool = msg_send![pb, writeObjects: array];
|
||||
if !ok {
|
||||
return Err("NSPasteboard writeObjects: returned NO".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
//! Windows clipboard implementation.
|
||||
//!
|
||||
//! The snapshot is structured so it mirrors the macOS `Vec<Vec<_>>`
|
||||
//! shape: a single outer "item" holding one `(format-name, bytes)`
|
||||
//! pair per enumerated format. Windows has no notion of multiple
|
||||
//! pasteboard items, so there's always exactly one or zero outer
|
||||
//! entries — enough to keep `item_count()` meaningful without
|
||||
//! fan-out.
|
||||
//!
|
||||
//! Format IDs are serialised as strings so the snapshot type can stay
|
||||
//! platform-neutral. Predefined formats use their canonical
|
||||
//! identifier (`"CF_UNICODETEXT"`, `"CF_HDROP"`, `"CF_DIB"`, …);
|
||||
//! registered formats use their string name from
|
||||
//! `GetClipboardFormatNameW` (`"HTML Format"`, `"Rich Text
|
||||
//! Format"`, …). Restore reverses the mapping with a lookup table
|
||||
//! for the predefined IDs and `RegisterClipboardFormatW` for the
|
||||
//! rest.
|
||||
//!
|
||||
//! Skipped format classes:
|
||||
//! - CF_BITMAP (2), CF_METAFILEPICT (3), CF_PALETTE (9),
|
||||
//! CF_ENHMETAFILE (14) — HGLOBAL's actually an HBITMAP /
|
||||
//! HENHMETAFILE, not raw memory. Rebuilding them across processes
|
||||
//! is possible but not worth it for clipboard stashing.
|
||||
//! - CF_OWNERDISPLAY (0x80) and the CF_DSPxxx variants (0x81–0x8E) —
|
||||
//! the owner draws these on demand. No data to snapshot.
|
||||
//! - CF_PRIVATEFIRST..CF_PRIVATELAST (0x200–0x2FF) — app-private,
|
||||
//! meaningless to restore from a different process.
|
||||
//! - CF_GDIOBJFIRST..CF_GDIOBJLAST (0x300–0x3FF) — GDI handles.
|
||||
//!
|
||||
//! Text formats that Windows auto-synthesises (CF_TEXT, CF_OEMTEXT,
|
||||
//! CF_LOCALE) are also skipped during save: `SetClipboardData` on
|
||||
//! CF_UNICODETEXT regenerates them lazily on restore.
|
||||
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use windows::core::PCWSTR;
|
||||
use windows::Win32::Foundation::{GlobalFree, HANDLE, HGLOBAL, HWND};
|
||||
use windows::Win32::System::DataExchange::{
|
||||
CloseClipboard, EmptyClipboard, EnumClipboardFormats, GetClipboardData,
|
||||
GetClipboardFormatNameW, GetClipboardSequenceNumber, OpenClipboard,
|
||||
RegisterClipboardFormatW, SetClipboardData,
|
||||
};
|
||||
use windows::Win32::System::Memory::{
|
||||
GlobalAlloc, GlobalLock, GlobalSize, GlobalUnlock, GLOBAL_ALLOC_FLAGS,
|
||||
};
|
||||
|
||||
// `windows` 0.62 doesn't re-export every predefined clipboard format
|
||||
// under a stable feature flag, so the values are pinned inline.
|
||||
// These numbers are ABI-stable back to Windows 3.1 — verified against
|
||||
// winuser.h.
|
||||
pub const CF_TEXT: u32 = 1;
|
||||
pub const CF_BITMAP: u32 = 2;
|
||||
pub const CF_METAFILEPICT: u32 = 3;
|
||||
pub const CF_SYLK: u32 = 4;
|
||||
pub const CF_DIF: u32 = 5;
|
||||
pub const CF_TIFF: u32 = 6;
|
||||
pub const CF_OEMTEXT: u32 = 7;
|
||||
pub const CF_DIB: u32 = 8;
|
||||
pub const CF_PALETTE: u32 = 9;
|
||||
pub const CF_PENDATA: u32 = 10;
|
||||
pub const CF_RIFF: u32 = 11;
|
||||
pub const CF_WAVE: u32 = 12;
|
||||
pub const CF_UNICODETEXT: u32 = 13;
|
||||
pub const CF_ENHMETAFILE: u32 = 14;
|
||||
pub const CF_HDROP: u32 = 15;
|
||||
pub const CF_LOCALE: u32 = 16;
|
||||
pub const CF_DIBV5: u32 = 17;
|
||||
pub const CF_OWNERDISPLAY: u32 = 0x0080;
|
||||
pub const CF_DSPTEXT: u32 = 0x0081;
|
||||
pub const CF_DSPBITMAP: u32 = 0x0082;
|
||||
pub const CF_DSPMETAFILEPICT: u32 = 0x0083;
|
||||
pub const CF_DSPENHMETAFILE: u32 = 0x008E;
|
||||
pub const CF_PRIVATEFIRST: u32 = 0x0200;
|
||||
pub const CF_PRIVATELAST: u32 = 0x02FF;
|
||||
pub const CF_GDIOBJFIRST: u32 = 0x0300;
|
||||
pub const CF_GDIOBJLAST: u32 = 0x03FF;
|
||||
|
||||
/// `GlobalAlloc` movable-memory flag — `GMEM_MOVEABLE` (0x0002).
|
||||
/// Required for HGLOBAL handles destined for `SetClipboardData`; fixed
|
||||
/// allocations are rejected.
|
||||
const GMEM_MOVEABLE: GLOBAL_ALLOC_FLAGS = GLOBAL_ALLOC_FLAGS(0x0002);
|
||||
|
||||
/// Map a predefined clipboard format ID to its canonical identifier
|
||||
/// string. Registered formats (IDs >= 0xC000) aren't handled here —
|
||||
/// the caller resolves those via `GetClipboardFormatNameW`.
|
||||
pub fn predefined_name(id: u32) -> Option<&'static str> {
|
||||
Some(match id {
|
||||
CF_TEXT => "CF_TEXT",
|
||||
CF_BITMAP => "CF_BITMAP",
|
||||
CF_METAFILEPICT => "CF_METAFILEPICT",
|
||||
CF_SYLK => "CF_SYLK",
|
||||
CF_DIF => "CF_DIF",
|
||||
CF_TIFF => "CF_TIFF",
|
||||
CF_OEMTEXT => "CF_OEMTEXT",
|
||||
CF_DIB => "CF_DIB",
|
||||
CF_PALETTE => "CF_PALETTE",
|
||||
CF_PENDATA => "CF_PENDATA",
|
||||
CF_RIFF => "CF_RIFF",
|
||||
CF_WAVE => "CF_WAVE",
|
||||
CF_UNICODETEXT => "CF_UNICODETEXT",
|
||||
CF_ENHMETAFILE => "CF_ENHMETAFILE",
|
||||
CF_HDROP => "CF_HDROP",
|
||||
CF_LOCALE => "CF_LOCALE",
|
||||
CF_DIBV5 => "CF_DIBV5",
|
||||
CF_OWNERDISPLAY => "CF_OWNERDISPLAY",
|
||||
CF_DSPTEXT => "CF_DSPTEXT",
|
||||
CF_DSPBITMAP => "CF_DSPBITMAP",
|
||||
CF_DSPMETAFILEPICT => "CF_DSPMETAFILEPICT",
|
||||
CF_DSPENHMETAFILE => "CF_DSPENHMETAFILE",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Reverse of [`predefined_name`].
|
||||
pub fn predefined_id(name: &str) -> Option<u32> {
|
||||
Some(match name {
|
||||
"CF_TEXT" => CF_TEXT,
|
||||
"CF_BITMAP" => CF_BITMAP,
|
||||
"CF_METAFILEPICT" => CF_METAFILEPICT,
|
||||
"CF_SYLK" => CF_SYLK,
|
||||
"CF_DIF" => CF_DIF,
|
||||
"CF_TIFF" => CF_TIFF,
|
||||
"CF_OEMTEXT" => CF_OEMTEXT,
|
||||
"CF_DIB" => CF_DIB,
|
||||
"CF_PALETTE" => CF_PALETTE,
|
||||
"CF_PENDATA" => CF_PENDATA,
|
||||
"CF_RIFF" => CF_RIFF,
|
||||
"CF_WAVE" => CF_WAVE,
|
||||
"CF_UNICODETEXT" => CF_UNICODETEXT,
|
||||
"CF_ENHMETAFILE" => CF_ENHMETAFILE,
|
||||
"CF_HDROP" => CF_HDROP,
|
||||
"CF_LOCALE" => CF_LOCALE,
|
||||
"CF_DIBV5" => CF_DIBV5,
|
||||
"CF_OWNERDISPLAY" => CF_OWNERDISPLAY,
|
||||
"CF_DSPTEXT" => CF_DSPTEXT,
|
||||
"CF_DSPBITMAP" => CF_DSPBITMAP,
|
||||
"CF_DSPMETAFILEPICT" => CF_DSPMETAFILEPICT,
|
||||
"CF_DSPENHMETAFILE" => CF_DSPENHMETAFILE,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns true for predefined formats whose payload is a GDI handle
|
||||
/// or owner-display sentinel rather than plain memory — callers must
|
||||
/// skip these during snapshot because GlobalSize/GlobalLock wouldn't
|
||||
/// return usable bytes.
|
||||
pub fn is_skipped_format(id: u32) -> bool {
|
||||
matches!(
|
||||
id,
|
||||
CF_BITMAP
|
||||
| CF_METAFILEPICT
|
||||
| CF_PALETTE
|
||||
| CF_ENHMETAFILE
|
||||
| CF_OWNERDISPLAY
|
||||
| CF_DSPTEXT
|
||||
| CF_DSPBITMAP
|
||||
| CF_DSPMETAFILEPICT
|
||||
| CF_DSPENHMETAFILE
|
||||
) || (CF_PRIVATEFIRST..=CF_PRIVATELAST).contains(&id)
|
||||
|| (CF_GDIOBJFIRST..=CF_GDIOBJLAST).contains(&id)
|
||||
}
|
||||
|
||||
/// Auto-synthesised formats that Windows regenerates from
|
||||
/// CF_UNICODETEXT on demand. Safe to skip during save; restore
|
||||
/// lets `SetClipboardData(CF_UNICODETEXT)` re-derive them.
|
||||
pub fn is_auto_synthesised(id: u32) -> bool {
|
||||
matches!(id, CF_TEXT | CF_OEMTEXT | CF_LOCALE)
|
||||
}
|
||||
|
||||
/// RAII wrapper around `OpenClipboard` / `CloseClipboard`.
|
||||
///
|
||||
/// The clipboard is a global exclusive resource — only one process at
|
||||
/// a time holds the handle. `OpenClipboard` fails with
|
||||
/// ERROR_ACCESS_DENIED when another process is mid-paste; the retry
|
||||
/// loop here absorbs the common transient case without bubbling a
|
||||
/// user-visible error.
|
||||
pub struct ClipboardGuard;
|
||||
|
||||
impl ClipboardGuard {
|
||||
pub fn open() -> Result<Self, String> {
|
||||
const MAX_ATTEMPTS: usize = 10;
|
||||
const RETRY_DELAY: Duration = Duration::from_millis(10);
|
||||
let mut last_err: Option<windows::core::Error> = None;
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
let result = unsafe { OpenClipboard(Some(HWND(std::ptr::null_mut()))) };
|
||||
match result {
|
||||
Ok(()) => return Ok(Self),
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
thread::sleep(RETRY_DELAY);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!(
|
||||
"OpenClipboard failed after {} retries ({:?}). Another process likely holds the clipboard open.",
|
||||
MAX_ATTEMPTS, last_err
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ClipboardGuard {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _ = CloseClipboard();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the full payload for `format` from the currently open
|
||||
/// clipboard into an owned `Vec<u8>`. Returns `Ok(None)` when the
|
||||
/// clipboard advertises the format but provides no concrete data
|
||||
/// (delay-rendered format that's never been realised).
|
||||
pub fn read_format_bytes(format: u32) -> Result<Option<Vec<u8>>, String> {
|
||||
unsafe {
|
||||
let handle = GetClipboardData(format)
|
||||
.map_err(|e| format!("GetClipboardData({format}) failed: {e}"))?;
|
||||
if handle.is_invalid() {
|
||||
return Ok(None);
|
||||
}
|
||||
let hglobal = HGLOBAL(handle.0);
|
||||
let size = GlobalSize(hglobal);
|
||||
if size == 0 {
|
||||
return Ok(Some(Vec::new()));
|
||||
}
|
||||
let ptr = GlobalLock(hglobal);
|
||||
if ptr.is_null() {
|
||||
return Err(format!(
|
||||
"GlobalLock returned null for format {format} (size {size})"
|
||||
));
|
||||
}
|
||||
let bytes = std::slice::from_raw_parts(ptr as *const u8, size).to_vec();
|
||||
let _ = GlobalUnlock(hglobal);
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up the name for a registered format ID (>= 0xC000). Returns
|
||||
/// `None` for unnamed predefined IDs — the caller should have used
|
||||
/// [`predefined_name`] first.
|
||||
pub fn registered_name(id: u32) -> Option<String> {
|
||||
let mut buf = [0u16; 256];
|
||||
let len = unsafe { GetClipboardFormatNameW(id, &mut buf) };
|
||||
if len <= 0 {
|
||||
return None;
|
||||
}
|
||||
String::from_utf16(&buf[..len as usize]).ok()
|
||||
}
|
||||
|
||||
/// Allocate a movable HGLOBAL, copy `bytes` in, return the handle
|
||||
/// ready for `SetClipboardData`. On success ownership transfers to
|
||||
/// the clipboard; on failure the caller must `GlobalFree`.
|
||||
pub fn allocate_global(bytes: &[u8]) -> Result<HGLOBAL, String> {
|
||||
if bytes.is_empty() {
|
||||
// `GlobalAlloc(_, 0)` returns NULL, which `SetClipboardData`
|
||||
// would then reject as an invalid handle. Pad to one byte so
|
||||
// the format still round-trips (the receiving app already
|
||||
// has to handle zero-content payloads via GlobalSize).
|
||||
return allocate_global(&[0u8]);
|
||||
}
|
||||
unsafe {
|
||||
let hglobal = GlobalAlloc(GMEM_MOVEABLE, bytes.len())
|
||||
.map_err(|e| format!("GlobalAlloc({}) failed: {e}", bytes.len()))?;
|
||||
let ptr = GlobalLock(hglobal);
|
||||
if ptr.is_null() {
|
||||
let _ = GlobalFree(Some(hglobal));
|
||||
return Err("GlobalLock returned null after GlobalAlloc".into());
|
||||
}
|
||||
std::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr as *mut u8, bytes.len());
|
||||
let _ = GlobalUnlock(hglobal);
|
||||
Ok(hglobal)
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one format's payload onto the currently open clipboard.
|
||||
/// On `SetClipboardData` success the HGLOBAL becomes the clipboard's
|
||||
/// responsibility — do not free. On failure, free it ourselves.
|
||||
pub fn put_format(format: u32, bytes: &[u8]) -> Result<(), String> {
|
||||
let hglobal = allocate_global(bytes)?;
|
||||
let handle = HANDLE(hglobal.0);
|
||||
unsafe {
|
||||
match SetClipboardData(format, Some(handle)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
let _ = GlobalFree(Some(hglobal));
|
||||
Err(format!("SetClipboardData({format}) failed: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// UTF-16 encode `s` with a trailing null code unit and push it as
|
||||
/// CF_UNICODETEXT.
|
||||
pub fn put_unicode_text(s: &str) -> Result<(), String> {
|
||||
let mut utf16: Vec<u16> = s.encode_utf16().collect();
|
||||
utf16.push(0);
|
||||
let bytes: &[u8] = unsafe {
|
||||
std::slice::from_raw_parts(
|
||||
utf16.as_ptr() as *const u8,
|
||||
utf16.len() * std::mem::size_of::<u16>(),
|
||||
)
|
||||
};
|
||||
put_format(CF_UNICODETEXT, bytes)
|
||||
}
|
||||
|
||||
/// Walk every format currently on the clipboard. `EnumClipboardFormats(0)`
|
||||
/// returns the first; each subsequent call with the previous format
|
||||
/// returns the next, until it returns 0 (or an error).
|
||||
pub fn enumerate_formats() -> Vec<u32> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = 0u32;
|
||||
loop {
|
||||
let next = unsafe { EnumClipboardFormats(current) };
|
||||
if next == 0 {
|
||||
break;
|
||||
}
|
||||
out.push(next);
|
||||
current = next;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Resolve a snapshot's format name back to the u32 format ID.
|
||||
/// Registered names (anything not predefined) go through
|
||||
/// `RegisterClipboardFormatW`, which is idempotent — the same name
|
||||
/// yields the same ID within a Windows session.
|
||||
pub fn resolve_format_id(name: &str) -> Result<u32, String> {
|
||||
if let Some(id) = predefined_id(name) {
|
||||
return Ok(id);
|
||||
}
|
||||
let wide: Vec<u16> = name.encode_utf16().chain(std::iter::once(0)).collect();
|
||||
let id = unsafe { RegisterClipboardFormatW(PCWSTR(wide.as_ptr())) };
|
||||
if id == 0 {
|
||||
return Err(format!("RegisterClipboardFormatW failed for {name:?}"));
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn sequence_number() -> u32 {
|
||||
unsafe { GetClipboardSequenceNumber() }
|
||||
}
|
||||
|
||||
pub fn empty() -> Result<(), String> {
|
||||
unsafe { EmptyClipboard().map_err(|e| format!("EmptyClipboard failed: {e}")) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
Ok(win::sequence_number() as i64)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
let change_count = win::sequence_number() as i64;
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
|
||||
let formats = win::enumerate_formats();
|
||||
let mut pairs: Vec<(String, Vec<u8>)> = Vec::with_capacity(formats.len());
|
||||
for id in formats {
|
||||
if win::is_skipped_format(id) || win::is_auto_synthesised(id) {
|
||||
continue;
|
||||
}
|
||||
let name = match win::predefined_name(id) {
|
||||
Some(n) => n.to_string(),
|
||||
None => match win::registered_name(id) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
},
|
||||
};
|
||||
match win::read_format_bytes(id) {
|
||||
Ok(Some(bytes)) => pairs.push((name, bytes)),
|
||||
Ok(None) => {}
|
||||
Err(_) => {
|
||||
// Single-format read failure (delay-render that never
|
||||
// materialises, ACL-restricted format, etc.) shouldn't
|
||||
// abort the whole snapshot — drop this format and keep
|
||||
// going so the user's other clipboard contents still
|
||||
// survive the round-trip.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let items = if pairs.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![pairs]
|
||||
};
|
||||
|
||||
Ok(ClipboardSnapshot {
|
||||
items,
|
||||
change_count,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn write_text(text: &str) -> Result<i64, String> {
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
win::empty()?;
|
||||
win::put_unicode_text(text)?;
|
||||
// `GetClipboardSequenceNumber` reflects the post-write value as soon
|
||||
// as `SetClipboardData` returns.
|
||||
Ok(win::sequence_number() as i64)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn restore_clipboard(snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
let _guard = win::ClipboardGuard::open()?;
|
||||
win::empty()?;
|
||||
|
||||
for pairs in &snapshot.items {
|
||||
for (name, bytes) in pairs {
|
||||
let id = match win::resolve_format_id(name) {
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
// Per-format failures here also don't abort the whole
|
||||
// restore — better to get the user's text content back even
|
||||
// if a weird custom format can't be rehydrated.
|
||||
let _ = win::put_format(id, bytes);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn current_change_count() -> Result<i64, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn save_clipboard() -> Result<ClipboardSnapshot, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn write_text(_text: &str) -> Result<i64, String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn restore_clipboard(_snapshot: &ClipboardSnapshot) -> Result<(), String> {
|
||||
Err("clipboard snapshot is not yet implemented on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
//! Captures the focused-UI snapshot at chord-start so auto-paste can land
|
||||
//! in the user's original text field even after focus drifts during
|
||||
//! transcription / refinement.
|
||||
//!
|
||||
//! We don't try to re-focus a specific sub-element on restore — many apps
|
||||
//! expose complex focus hierarchies that don't respond consistently to
|
||||
//! programmatic focus pokes. Bringing the owning *window* to the
|
||||
//! foreground is enough: the window's own focus manager restores its
|
||||
//! last-focused field, which is what every well-behaved paste-buffer tool
|
||||
//! does and what users expect.
|
||||
//!
|
||||
//! - **macOS** — `AXUIElementCopyAttributeValue(kAXFocusedUIElement)` +
|
||||
//! `AXUIElementGetPid` + `NSRunningApplication.activateWithOptions:`.
|
||||
//! - **Windows** — `GetForegroundWindow` + `GetWindowThreadProcessId` for
|
||||
//! the top-level HWND and PID; UIAutomation's `IUIAutomation::GetFocusedElement`
|
||||
//! for best-effort control-class (skipped silently if COM isn't usable).
|
||||
//! Activation walks top-level windows for the saved PID and calls
|
||||
//! `SetForegroundWindow`, bracketed by the `AttachThreadInput` dance
|
||||
//! so Windows' foreground-lock rules don't silently swallow the
|
||||
//! activation into a taskbar flash.
|
||||
//!
|
||||
//! PID + bundle id + role are all captured for diagnostics — the bundle
|
||||
//! id lets step 6 (internal direct injection) detect "focus was inside
|
||||
//! Voicebox itself" and short-circuit the synthetic-paste path. On
|
||||
//! Windows, `bundle_id` holds the lowercased exe basename (`"voicebox.exe"`)
|
||||
//! since there's no equivalent of macOS' reverse-DNS bundle identifier.
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct FocusSnapshot {
|
||||
pub pid: i32,
|
||||
pub bundle_id: Option<String>,
|
||||
pub role: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use core_foundation_sys::base::{kCFAllocatorDefault, CFRelease};
|
||||
#[cfg(target_os = "macos")]
|
||||
use core_foundation_sys::string::{
|
||||
kCFStringEncodingUTF8, CFStringCreateWithCString, CFStringGetCString, CFStringGetLength,
|
||||
CFStringRef,
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::runtime::Object;
|
||||
#[cfg(target_os = "macos")]
|
||||
use objc::{class, msg_send, sel, sel_impl};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
type Id = *mut Object;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use core_foundation_sys::base::CFTypeRef;
|
||||
use core_foundation_sys::string::CFStringRef;
|
||||
|
||||
pub type AXError = i32;
|
||||
pub const AX_ERROR_SUCCESS: AXError = 0;
|
||||
pub type AXUIElementRef = *const std::ffi::c_void;
|
||||
pub type Pid = i32;
|
||||
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn AXUIElementCreateSystemWide() -> AXUIElementRef;
|
||||
pub fn AXUIElementCopyAttributeValue(
|
||||
element: AXUIElementRef,
|
||||
attribute: CFStringRef,
|
||||
value: *mut CFTypeRef,
|
||||
) -> AXError;
|
||||
pub fn AXUIElementGetPid(element: AXUIElementRef, pid: *mut Pid) -> AXError;
|
||||
}
|
||||
// AX attribute keys are exposed as C macros that expand to CFSTR(...)
|
||||
// literals, not as linkable symbols — build the CFStrings at runtime
|
||||
// instead (see `cf_string_const` in focus_capture.rs).
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
struct AutoreleasePool {
|
||||
pool: Id,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl AutoreleasePool {
|
||||
unsafe fn new() -> Self {
|
||||
let pool: Id = msg_send![class!(NSAutoreleasePool), alloc];
|
||||
let pool: Id = msg_send![pool, init];
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl Drop for AutoreleasePool {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
let _: () = msg_send![self.pool, drain];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn ns_string_to_rust(s: Id) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bytes: *const i8 = msg_send![s, UTF8String];
|
||||
if bytes.is_null() {
|
||||
return None;
|
||||
}
|
||||
std::ffi::CStr::from_ptr(bytes)
|
||||
.to_str()
|
||||
.ok()
|
||||
.map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
/// Build a `+1` retained CFString from an ASCII constant. Caller owns the
|
||||
/// returned reference and must `CFRelease` it. Used for AX attribute keys
|
||||
/// (`"AXFocusedUIElement"`, `"AXRole"`) because those aren't exported as
|
||||
/// linker symbols — Apple ships them as `CFSTR(...)` macros.
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn cf_string_const(s: &str) -> Option<CFStringRef> {
|
||||
let cstr = std::ffi::CString::new(s).ok()?;
|
||||
let result = CFStringCreateWithCString(kCFAllocatorDefault, cstr.as_ptr(), kCFStringEncodingUTF8);
|
||||
if result.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn cfstring_to_rust(s: CFStringRef) -> Option<String> {
|
||||
if s.is_null() {
|
||||
return None;
|
||||
}
|
||||
let len = CFStringGetLength(s);
|
||||
if len == 0 {
|
||||
return Some(String::new());
|
||||
}
|
||||
// CFStringGetLength is in UTF-16 code units; UTF-8 can need up to 4
|
||||
// bytes per unit plus the trailing NUL.
|
||||
let max_bytes = (len * 4 + 1) as usize;
|
||||
let mut buf = vec![0u8; max_bytes];
|
||||
let ok = CFStringGetCString(
|
||||
s,
|
||||
buf.as_mut_ptr() as *mut i8,
|
||||
max_bytes as isize,
|
||||
kCFStringEncodingUTF8,
|
||||
);
|
||||
if ok == 0 {
|
||||
return None;
|
||||
}
|
||||
let cstr = std::ffi::CStr::from_ptr(buf.as_ptr() as *const i8);
|
||||
cstr.to_str().ok().map(|x| x.to_owned())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
unsafe fn bundle_id_for_pid(pid: i32) -> Option<String> {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let app: Id = msg_send![
|
||||
class!(NSRunningApplication),
|
||||
runningApplicationWithProcessIdentifier: pid
|
||||
];
|
||||
if app.is_null() {
|
||||
return None;
|
||||
}
|
||||
let bundle: Id = msg_send![app, bundleIdentifier];
|
||||
ns_string_to_rust(bundle)
|
||||
}
|
||||
|
||||
/// Read the system-wide focused UI element's PID, bundle id, and AX role.
|
||||
///
|
||||
/// Returns an error when no element is focused (e.g. Dock has focus) or
|
||||
/// when Accessibility permission is missing — `AXUIElementCopyAttributeValue`
|
||||
/// returns `-25204 kAXErrorAPIDisabled` in that case.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
use ffi::*;
|
||||
unsafe {
|
||||
let system_wide = AXUIElementCreateSystemWide();
|
||||
if system_wide.is_null() {
|
||||
return Err("AXUIElementCreateSystemWide returned null".into());
|
||||
}
|
||||
let _sys_guard = scopeguard::guard(system_wide, |e| {
|
||||
CFRelease(e as *const std::ffi::c_void)
|
||||
});
|
||||
|
||||
let focused_attr = cf_string_const("AXFocusedUIElement")
|
||||
.ok_or("Failed to build AXFocusedUIElement CFString")?;
|
||||
let _focused_attr_guard =
|
||||
scopeguard::guard(focused_attr, |s| CFRelease(s as *const std::ffi::c_void));
|
||||
|
||||
let mut focused: *const std::ffi::c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(
|
||||
system_wide,
|
||||
focused_attr,
|
||||
&mut focused as *mut _,
|
||||
);
|
||||
if err != AX_ERROR_SUCCESS || focused.is_null() {
|
||||
return Err(format!(
|
||||
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
|
||||
err
|
||||
));
|
||||
}
|
||||
let _focus_guard = scopeguard::guard(focused, |e| CFRelease(e));
|
||||
|
||||
let focused_elem = focused as AXUIElementRef;
|
||||
|
||||
let mut pid: Pid = 0;
|
||||
let err = AXUIElementGetPid(focused_elem, &mut pid);
|
||||
if err != AX_ERROR_SUCCESS {
|
||||
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
|
||||
}
|
||||
|
||||
let role = {
|
||||
let role_attr = cf_string_const("AXRole");
|
||||
match role_attr {
|
||||
Some(role_attr) => {
|
||||
let _role_attr_guard = scopeguard::guard(role_attr, |s| {
|
||||
CFRelease(s as *const std::ffi::c_void)
|
||||
});
|
||||
let mut role_value: *const std::ffi::c_void = std::ptr::null();
|
||||
let err = AXUIElementCopyAttributeValue(
|
||||
focused_elem,
|
||||
role_attr,
|
||||
&mut role_value as *mut _,
|
||||
);
|
||||
if err == AX_ERROR_SUCCESS && !role_value.is_null() {
|
||||
let _role_guard = scopeguard::guard(role_value, |e| CFRelease(e));
|
||||
cfstring_to_rust(role_value as CFStringRef)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
}
|
||||
};
|
||||
|
||||
let bundle_id = bundle_id_for_pid(pid);
|
||||
|
||||
Ok(FocusSnapshot {
|
||||
pid,
|
||||
bundle_id,
|
||||
role,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Bring the app owning `pid` to the foreground, re-activating its
|
||||
/// last-focused window. Paired with [`capture_focus`] at chord-start so a
|
||||
/// post-transcription synthetic ⌘V lands where the user started, not
|
||||
/// wherever focus drifted to during the transcribe / refine window.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn activate_pid(pid: i32) -> Result<(), String> {
|
||||
unsafe {
|
||||
let _pool = AutoreleasePool::new();
|
||||
let app: Id = msg_send![
|
||||
class!(NSRunningApplication),
|
||||
runningApplicationWithProcessIdentifier: pid
|
||||
];
|
||||
if app.is_null() {
|
||||
return Err(format!("No running application for PID {}", pid));
|
||||
}
|
||||
// NSApplicationActivateIgnoringOtherApps = 1 << 1 = 2.
|
||||
//
|
||||
// macOS 14 deprecated this in favour of `activate()` but kept it
|
||||
// functional when the caller has Accessibility permission — which
|
||||
// we require for the paste event anyway.
|
||||
let _: bool = msg_send![app, activateWithOptions: 2u64];
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
use std::path::Path;
|
||||
|
||||
use windows::core::{IUnknown, BSTR, PWSTR};
|
||||
use windows::Win32::Foundation::{CloseHandle, BOOL, HWND, LPARAM};
|
||||
use windows::Win32::System::Com::{
|
||||
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
|
||||
};
|
||||
use windows::Win32::System::Threading::{
|
||||
AttachThreadInput, GetCurrentThreadId, OpenProcess, QueryFullProcessImageNameW,
|
||||
PROCESS_NAME_FORMAT, PROCESS_QUERY_LIMITED_INFORMATION,
|
||||
};
|
||||
use windows::Win32::UI::Accessibility::{CUIAutomation, IUIAutomation, IUIAutomationElement};
|
||||
use windows::Win32::UI::WindowsAndMessaging::{
|
||||
EnumWindows, GetForegroundWindow, GetWindow, GetWindowThreadProcessId, IsWindowVisible,
|
||||
SetForegroundWindow, GW_OWNER,
|
||||
};
|
||||
|
||||
/// Read the PID that owns `hwnd`. Returns 0 on failure.
|
||||
pub unsafe fn hwnd_pid(hwnd: HWND) -> u32 {
|
||||
let mut pid: u32 = 0;
|
||||
let _ = GetWindowThreadProcessId(hwnd, Some(&mut pid as *mut _));
|
||||
pid
|
||||
}
|
||||
|
||||
/// Query a PID's executable path and return its lowercased basename
|
||||
/// (e.g. `"voicebox.exe"`). This is the Windows analogue of macOS'
|
||||
/// `bundleIdentifier`, just less globally unique — two apps with the
|
||||
/// same exe name can collide, but that's rare enough to accept for
|
||||
/// the self-paste short-circuit.
|
||||
pub fn exe_basename(pid: u32) -> Option<String> {
|
||||
unsafe {
|
||||
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
|
||||
let mut buf = [0u16; 1024];
|
||||
let mut size = buf.len() as u32;
|
||||
let ok = QueryFullProcessImageNameW(
|
||||
handle,
|
||||
PROCESS_NAME_FORMAT(0),
|
||||
PWSTR(buf.as_mut_ptr()),
|
||||
&mut size,
|
||||
);
|
||||
let _ = CloseHandle(handle);
|
||||
if ok.is_err() || size == 0 {
|
||||
return None;
|
||||
}
|
||||
let full = String::from_utf16(&buf[..size as usize]).ok()?;
|
||||
let basename = Path::new(&full)
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())?;
|
||||
Some(basename)
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort `UIAutomation::GetFocusedElement().CurrentClassName()`.
|
||||
/// Returns `None` when COM init, CoCreateInstance, or any UIA call
|
||||
/// fails — role info is nice-to-have, not load-bearing for paste.
|
||||
pub fn focused_control_class() -> Option<String> {
|
||||
unsafe {
|
||||
// MTA per-thread init. Ignore HRESULT: S_OK / S_FALSE /
|
||||
// RPC_E_CHANGED_MODE are all benign for our uses here, and
|
||||
// we deliberately never call CoUninitialize (the Tauri
|
||||
// runtime thread lives for the life of the process, so
|
||||
// leaving COM init in place is fine).
|
||||
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
|
||||
|
||||
let automation: IUIAutomation =
|
||||
CoCreateInstance(&CUIAutomation, None::<&IUnknown>, CLSCTX_INPROC_SERVER).ok()?;
|
||||
let element: IUIAutomationElement = automation.GetFocusedElement().ok()?;
|
||||
// UIAutomationElement's CurrentClassName allocates a BSTR
|
||||
// the caller has to drop. `BSTR` in `windows` crate is a
|
||||
// Drop-wrapped owned string, so just returning `.to_string()`
|
||||
// is safe.
|
||||
let class: BSTR = element.CurrentClassName().ok()?;
|
||||
let s = class.to_string();
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a visible top-level window owned by `pid`. Returns the first
|
||||
/// match via `EnumWindows`. Top-level ≡ no owner window.
|
||||
pub fn find_top_level_window(pid: u32) -> Option<HWND> {
|
||||
struct Ctx {
|
||||
target_pid: u32,
|
||||
found: Option<HWND>,
|
||||
}
|
||||
let mut ctx = Ctx {
|
||||
target_pid: pid,
|
||||
found: None,
|
||||
};
|
||||
unsafe extern "system" fn callback(hwnd: HWND, lparam: LPARAM) -> BOOL {
|
||||
let ctx = &mut *(lparam.0 as *mut Ctx);
|
||||
if hwnd_pid(hwnd) != ctx.target_pid {
|
||||
return BOOL(1);
|
||||
}
|
||||
// Skip tool windows / invisible shells. `GetWindow(GW_OWNER)`
|
||||
// is non-null for modal dialogs and other secondary windows;
|
||||
// we want the real app frame, which has no owner.
|
||||
if !IsWindowVisible(hwnd).as_bool() {
|
||||
return BOOL(1);
|
||||
}
|
||||
if !GetWindow(hwnd, GW_OWNER).unwrap_or(HWND(std::ptr::null_mut())).is_invalid() {
|
||||
return BOOL(1);
|
||||
}
|
||||
ctx.found = Some(hwnd);
|
||||
BOOL(0)
|
||||
}
|
||||
unsafe {
|
||||
let _ = EnumWindows(
|
||||
Some(callback),
|
||||
LPARAM(&mut ctx as *mut _ as isize),
|
||||
);
|
||||
}
|
||||
ctx.found
|
||||
}
|
||||
|
||||
/// Bring `hwnd` to the foreground reliably.
|
||||
///
|
||||
/// Plain `SetForegroundWindow` loses to Windows' foreground-lock
|
||||
/// rules — when our process isn't already foreground it can't hand
|
||||
/// focus to another app. The documented workaround is to attach the
|
||||
/// current thread's input queue to the current foreground window's
|
||||
/// thread for the duration of the call, which temporarily lets us
|
||||
/// share that thread's "last user activity" stamp.
|
||||
pub fn activate_hwnd(hwnd: HWND) -> Result<(), String> {
|
||||
unsafe {
|
||||
let fg = GetForegroundWindow();
|
||||
if fg == hwnd {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let our_thread = GetCurrentThreadId();
|
||||
let fg_thread = if fg.is_invalid() {
|
||||
0
|
||||
} else {
|
||||
let mut _pid: u32 = 0;
|
||||
GetWindowThreadProcessId(fg, Some(&mut _pid as *mut _))
|
||||
};
|
||||
|
||||
let attached = fg_thread != 0
|
||||
&& fg_thread != our_thread
|
||||
&& AttachThreadInput(our_thread, fg_thread, true).as_bool();
|
||||
|
||||
let ok = SetForegroundWindow(hwnd).as_bool();
|
||||
|
||||
if attached {
|
||||
let _ = AttachThreadInput(our_thread, fg_thread, false);
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return Err(format!(
|
||||
"SetForegroundWindow failed for HWND {:?} — Windows foreground-lock may have denied the activation.",
|
||||
hwnd.0
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
|
||||
|
||||
unsafe {
|
||||
let hwnd = GetForegroundWindow();
|
||||
if hwnd.is_invalid() {
|
||||
return Err(
|
||||
"GetForegroundWindow returned null — the desktop has no focused window (secure attention sequence, lock screen, or no user session)."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let pid = win::hwnd_pid(hwnd);
|
||||
if pid == 0 {
|
||||
return Err("GetWindowThreadProcessId returned PID 0 for the foreground window".into());
|
||||
}
|
||||
let bundle_id = win::exe_basename(pid);
|
||||
let role = win::focused_control_class();
|
||||
Ok(FocusSnapshot {
|
||||
pid: pid as i32,
|
||||
bundle_id,
|
||||
role,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn activate_pid(pid: i32) -> Result<(), String> {
|
||||
if pid <= 0 {
|
||||
return Err(format!("Cannot activate invalid PID {pid}"));
|
||||
}
|
||||
let hwnd = win::find_top_level_window(pid as u32)
|
||||
.ok_or_else(|| format!("No visible top-level window for PID {pid}"))?;
|
||||
win::activate_hwnd(hwnd)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
||||
Err("focus capture is not yet implemented on this platform".into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn activate_pid(_pid: i32) -> Result<(), String> {
|
||||
Err("app activation is not yet implemented on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
//! Global keyboard tap + chord dispatcher.
|
||||
//!
|
||||
//! Spawns a dedicated thread running `rdev::listen` (which internally owns a
|
||||
//! CGEventTap on macOS / `SetWindowsHookEx` on Windows / `XRecord` on Linux).
|
||||
//! Feeds raw key events into a private `Chord` state machine and translates
|
||||
//! its effects into Tauri events + window show/hide calls.
|
||||
//!
|
||||
//! Left- and right-hand modifier variants are deliberately kept distinct.
|
||||
//! Defaults bind to right-hand Cmd + right-hand Option so that the usual
|
||||
//! left-hand shortcuts — Cmd+Option+I to open devtools, Cmd+Option+Esc for
|
||||
//! force-quit, etc. — continue to work untouched.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
|
||||
use rdev::{listen, EventType, Key};
|
||||
use tauri::{AppHandle, Emitter, Manager};
|
||||
|
||||
use crate::focus_capture;
|
||||
use crate::DICTATE_WINDOW_LABEL;
|
||||
|
||||
// ========================================================================
|
||||
// Chord state machine
|
||||
// ========================================================================
|
||||
|
||||
/// Semantic action a chord can be bound to. `PushToTalk` = hold chord to
|
||||
/// record, release to stop. `ToggleToTalk` = press chord to start recording,
|
||||
/// press again to stop.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ChordAction {
|
||||
PushToTalk,
|
||||
ToggleToTalk,
|
||||
}
|
||||
|
||||
/// Output of the chord state machine after consuming an input event. Hosts
|
||||
/// translate these into UI / recorder calls.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Effect {
|
||||
StartRecording(ChordAction),
|
||||
StopRecording(ChordAction),
|
||||
/// Emitted when a push-to-talk chord is "upgraded" into the toggle chord
|
||||
/// mid-hold — hosts may want to discard the captured audio and restart
|
||||
/// so the transition moment isn't in the recording.
|
||||
RestartRecording(ChordAction),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum KeyEvent {
|
||||
Down(Key),
|
||||
Up(Key),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum Match {
|
||||
None,
|
||||
Partial,
|
||||
Hit(ChordAction),
|
||||
}
|
||||
|
||||
pub type Bindings = HashMap<ChordAction, HashSet<Key>>;
|
||||
|
||||
/// Private state machine that turns key-down / key-up events into
|
||||
/// `Effect`s. Owns no I/O — just the "which keys are held" and
|
||||
/// "which action is currently driving a recording" bookkeeping.
|
||||
struct Chord {
|
||||
bindings: Bindings,
|
||||
pressed_keys: HashSet<Key>,
|
||||
active_recording_action: Option<ChordAction>,
|
||||
}
|
||||
|
||||
impl Chord {
|
||||
fn new(bindings: Bindings) -> Self {
|
||||
Self {
|
||||
bindings,
|
||||
pressed_keys: HashSet::new(),
|
||||
active_recording_action: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn update_bindings(&mut self, bindings: Bindings) {
|
||||
self.bindings = bindings;
|
||||
}
|
||||
|
||||
fn handle(&mut self, event: KeyEvent) -> Vec<Effect> {
|
||||
let changed = match event {
|
||||
KeyEvent::Down(k) => self.pressed_keys.insert(k),
|
||||
KeyEvent::Up(k) => self.pressed_keys.remove(&k),
|
||||
};
|
||||
if !changed {
|
||||
return Vec::new();
|
||||
}
|
||||
self.step()
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // Used by the chord picker UI in Pass 2 to suspend matching during capture.
|
||||
fn reset(&mut self) {
|
||||
self.pressed_keys.clear();
|
||||
self.active_recording_action = None;
|
||||
}
|
||||
|
||||
fn step(&mut self) -> Vec<Effect> {
|
||||
match self.active_recording_action {
|
||||
Some(ChordAction::PushToTalk) => {
|
||||
if self.classify() == Match::Hit(ChordAction::ToggleToTalk) {
|
||||
self.active_recording_action = Some(ChordAction::ToggleToTalk);
|
||||
return vec![Effect::RestartRecording(ChordAction::ToggleToTalk)];
|
||||
}
|
||||
|
||||
let still_held = self
|
||||
.bindings
|
||||
.get(&ChordAction::PushToTalk)
|
||||
.map(|chord| chord.is_subset(&self.pressed_keys))
|
||||
.unwrap_or(false);
|
||||
|
||||
if !still_held {
|
||||
self.active_recording_action = None;
|
||||
return vec![Effect::StopRecording(ChordAction::PushToTalk)];
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
Some(ChordAction::ToggleToTalk) => {
|
||||
if self.classify() == Match::Hit(ChordAction::ToggleToTalk) {
|
||||
self.active_recording_action = None;
|
||||
return vec![Effect::StopRecording(ChordAction::ToggleToTalk)];
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
None => match self.classify() {
|
||||
Match::Hit(action) => {
|
||||
self.active_recording_action = Some(action);
|
||||
vec![Effect::StartRecording(action)]
|
||||
}
|
||||
Match::None | Match::Partial => Vec::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn classify(&self) -> Match {
|
||||
if self.pressed_keys.is_empty() {
|
||||
return Match::None;
|
||||
}
|
||||
|
||||
// Exact match wins even if the pressed set is also a prefix of another
|
||||
// binding.
|
||||
for (action, chord) in &self.bindings {
|
||||
if self.pressed_keys == *chord {
|
||||
return Match::Hit(*action);
|
||||
}
|
||||
}
|
||||
|
||||
let is_prefix = self
|
||||
.bindings
|
||||
.values()
|
||||
.any(|c| self.pressed_keys.is_subset(c) && self.pressed_keys != *c);
|
||||
|
||||
if is_prefix {
|
||||
Match::Partial
|
||||
} else {
|
||||
Match::None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Monitor
|
||||
// ========================================================================
|
||||
|
||||
/// Hardcoded Pass 1 defaults. Two right-hand modifiers so the usual left-hand
|
||||
/// shortcuts pass through unaffected. Replaced in Pass 2 by reading from the
|
||||
/// server-side `capture_settings` table via a Tauri command the frontend
|
||||
/// invokes whenever `useCaptureSettings` resolves.
|
||||
///
|
||||
/// - **macOS:** `MetaRight + AltGr` — right Command + right Option. (rdev
|
||||
/// labels right-Option as `AltGr` for Linux-convention symmetry; on macOS
|
||||
/// it's the physical right-option key.)
|
||||
/// - **Windows / Linux:** `ControlRight + ShiftRight` — right Ctrl + right
|
||||
/// Shift. Deliberately avoids `AltGr`: on international Windows layouts
|
||||
/// the OS synthesises `AltGr` as `Ctrl+Alt`, so any `AltGr`-involving
|
||||
/// default would fire on every `@`, `€`, `\` keypress on German / French
|
||||
/// / Spanish keyboards.
|
||||
pub fn default_bindings() -> Bindings {
|
||||
#[cfg(target_os = "macos")]
|
||||
let (m1, m2) = (Key::MetaRight, Key::AltGr);
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let (m1, m2) = (Key::ControlRight, Key::ShiftRight);
|
||||
|
||||
let mut b = Bindings::new();
|
||||
b.insert(ChordAction::PushToTalk, {
|
||||
let mut s = HashSet::new();
|
||||
s.insert(m1);
|
||||
s.insert(m2);
|
||||
s
|
||||
});
|
||||
b.insert(ChordAction::ToggleToTalk, {
|
||||
let mut s = HashSet::new();
|
||||
s.insert(m1);
|
||||
s.insert(m2);
|
||||
s.insert(Key::Space);
|
||||
s
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
pub struct HotkeyMonitor {
|
||||
chord: Arc<Mutex<Chord>>,
|
||||
}
|
||||
|
||||
impl HotkeyMonitor {
|
||||
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
||||
let chord = Arc::new(Mutex::new(Chord::new(bindings)));
|
||||
let chord_for_thread = chord.clone();
|
||||
let app_for_thread = app.clone();
|
||||
|
||||
thread::spawn(move || {
|
||||
// Without this call, rdev's convert() calls TSMGetInputSourceProperty
|
||||
// on this background thread, which trips a main-queue assertion on
|
||||
// macOS 14+ and traps the whole process (see Narsil/rdev#165 / #147).
|
||||
#[cfg(target_os = "macos")]
|
||||
rdev::set_is_main_thread(false);
|
||||
|
||||
let result = listen(move |event| {
|
||||
let input = match event.event_type {
|
||||
EventType::KeyPress(k) => KeyEvent::Down(k),
|
||||
EventType::KeyRelease(k) => KeyEvent::Up(k),
|
||||
_ => return,
|
||||
};
|
||||
|
||||
let effects = match chord_for_thread.lock() {
|
||||
Ok(mut chord) => chord.handle(input),
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
for effect in effects {
|
||||
apply_effect(&app_for_thread, effect);
|
||||
}
|
||||
});
|
||||
|
||||
if let Err(err) = result {
|
||||
eprintln!(
|
||||
"HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.",
|
||||
err
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Self { chord }
|
||||
}
|
||||
|
||||
pub fn update_bindings(&self, bindings: Bindings) {
|
||||
if let Ok(mut chord) = self.chord.lock() {
|
||||
chord.update_bindings(bindings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_effect(app: &AppHandle, effect: Effect) {
|
||||
match effect {
|
||||
Effect::StartRecording(_) => {
|
||||
// Snapshot focus BEFORE we touch the window — any AppKit
|
||||
// reshuffle triggered by set_position / show could in principle
|
||||
// steal key focus and poison the reading. In practice those
|
||||
// calls leave keyWindow alone, but capturing first is free.
|
||||
let focus = focus_capture::capture_focus().ok();
|
||||
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
// The previous hide-cycle parked the window off-screen and
|
||||
// made it click-through — undo both before showing, so the
|
||||
// pill lands at top-center and the user can actually click
|
||||
// the error pill / stop button.
|
||||
//
|
||||
// `current_monitor()` returns None when the window is off
|
||||
// any display (our hide handler parks it at -10_000, -10_000
|
||||
// precisely so it never intercepts clicks), so fall back to
|
||||
// the primary monitor for the reposition.
|
||||
let monitor = window
|
||||
.current_monitor()
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| window.primary_monitor().ok().flatten());
|
||||
if let Some(monitor) = monitor {
|
||||
let monitor_pos = monitor.position();
|
||||
let monitor_size = monitor.size();
|
||||
if let Ok(win_size) = window.outer_size() {
|
||||
let x = monitor_pos.x
|
||||
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
|
||||
}
|
||||
}
|
||||
let _ = window.set_ignore_cursor_events(false);
|
||||
// Deliberately no set_focus() — taking key focus would yank
|
||||
// it out of whatever app the user was typing in, which is
|
||||
// the opposite of what a dictation overlay should do.
|
||||
let _ = window.show();
|
||||
let payload = serde_json::json!({ "focus": focus });
|
||||
let _ = window.emit("dictate:start", payload);
|
||||
}
|
||||
}
|
||||
Effect::StopRecording(_) => {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
let _ = window.emit("dictate:stop", ());
|
||||
}
|
||||
}
|
||||
Effect::RestartRecording(_) => {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
let _ = window.emit("dictate:restart", ());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Tests
|
||||
// ========================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn keys(keys: &[Key]) -> HashSet<Key> {
|
||||
keys.iter().copied().collect()
|
||||
}
|
||||
|
||||
fn test_bindings() -> Bindings {
|
||||
let mut b = Bindings::new();
|
||||
b.insert(ChordAction::PushToTalk, keys(&[Key::MetaLeft, Key::Alt]));
|
||||
b.insert(
|
||||
ChordAction::ToggleToTalk,
|
||||
keys(&[Key::MetaLeft, Key::Alt, Key::Space]),
|
||||
);
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_to_talk_starts_on_exact_hold_and_stops_on_release() {
|
||||
let mut c = Chord::new(test_bindings());
|
||||
assert_eq!(c.handle(KeyEvent::Down(Key::MetaLeft)), vec![]);
|
||||
assert_eq!(
|
||||
c.handle(KeyEvent::Down(Key::Alt)),
|
||||
vec![Effect::StartRecording(ChordAction::PushToTalk)],
|
||||
);
|
||||
assert_eq!(
|
||||
c.handle(KeyEvent::Up(Key::Alt)),
|
||||
vec![Effect::StopRecording(ChordAction::PushToTalk)],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggle_starts_on_exact_and_stops_on_second_exact() {
|
||||
let mut c = Chord::new(test_bindings());
|
||||
c.handle(KeyEvent::Down(Key::MetaLeft));
|
||||
c.handle(KeyEvent::Down(Key::Alt));
|
||||
// At this point PTT is active.
|
||||
assert_eq!(c.active_recording_action, Some(ChordAction::PushToTalk));
|
||||
assert_eq!(
|
||||
c.handle(KeyEvent::Down(Key::Space)),
|
||||
vec![Effect::RestartRecording(ChordAction::ToggleToTalk)],
|
||||
);
|
||||
// Releasing cmd/opt must not stop toggle recording.
|
||||
assert_eq!(c.handle(KeyEvent::Up(Key::MetaLeft)), vec![]);
|
||||
assert_eq!(c.handle(KeyEvent::Up(Key::Alt)), vec![]);
|
||||
assert_eq!(c.handle(KeyEvent::Up(Key::Space)), vec![]);
|
||||
|
||||
// Second press of toggle chord stops it.
|
||||
c.handle(KeyEvent::Down(Key::MetaLeft));
|
||||
c.handle(KeyEvent::Down(Key::Alt));
|
||||
assert_eq!(
|
||||
c.handle(KeyEvent::Down(Key::Space)),
|
||||
vec![Effect::StopRecording(ChordAction::ToggleToTalk)],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toggle_from_idle_starts_immediately_on_full_chord() {
|
||||
let mut c = Chord::new(test_bindings());
|
||||
c.handle(KeyEvent::Down(Key::MetaLeft));
|
||||
c.handle(KeyEvent::Down(Key::Alt));
|
||||
// Drop MetaLeft before Space — we're not in the exact toggle match
|
||||
// yet, just prefix. No start for toggle.
|
||||
assert_eq!(c.active_recording_action, Some(ChordAction::PushToTalk));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
//! Stable string ↔ `rdev::Key` mapping for chord persistence.
|
||||
//!
|
||||
//! The frontend captures keypresses through the browser keyboard API (which
|
||||
//! exposes `event.code` like `"MetaRight"`, `"AltRight"`, `"Space"`, `"KeyA"`)
|
||||
//! and stores chords in capture_settings as JSON arrays of canonical names.
|
||||
//! On the way back the same names need to round-trip into `rdev::Key`
|
||||
//! variants the chord engine actually matches against.
|
||||
//!
|
||||
//! Names follow the rdev variant identifiers exactly (`"MetaRight"`,
|
||||
//! `"AltGr"`, `"KeyA"`, …) with one alias: the browser reports right-Option
|
||||
//! as `"AltRight"` while rdev calls it `"AltGr"`. Both map to the same key.
|
||||
|
||||
use rdev::Key;
|
||||
|
||||
/// Resolve a canonical key name to its `rdev::Key`. Returns `None` for
|
||||
/// names that don't have a corresponding variant — the command surface
|
||||
/// rejects those so we never silently drop keys from a chord.
|
||||
pub fn key_from_str(name: &str) -> Option<Key> {
|
||||
Some(match name {
|
||||
// Modifiers — left/right distinction matters for chord defaults.
|
||||
"Alt" | "AltLeft" => Key::Alt,
|
||||
"AltGr" | "AltRight" => Key::AltGr,
|
||||
"ControlLeft" => Key::ControlLeft,
|
||||
"ControlRight" => Key::ControlRight,
|
||||
"MetaLeft" => Key::MetaLeft,
|
||||
"MetaRight" => Key::MetaRight,
|
||||
"ShiftLeft" => Key::ShiftLeft,
|
||||
"ShiftRight" => Key::ShiftRight,
|
||||
"CapsLock" => Key::CapsLock,
|
||||
"Function" => Key::Function,
|
||||
|
||||
// Whitespace / navigation
|
||||
"Space" => Key::Space,
|
||||
"Tab" => Key::Tab,
|
||||
"Return" | "Enter" => Key::Return,
|
||||
"Backspace" => Key::Backspace,
|
||||
"Delete" => Key::Delete,
|
||||
"Escape" => Key::Escape,
|
||||
"Insert" => Key::Insert,
|
||||
"Home" => Key::Home,
|
||||
"End" => Key::End,
|
||||
"PageUp" => Key::PageUp,
|
||||
"PageDown" => Key::PageDown,
|
||||
"ArrowUp" | "UpArrow" => Key::UpArrow,
|
||||
"ArrowDown" | "DownArrow" => Key::DownArrow,
|
||||
"ArrowLeft" | "LeftArrow" => Key::LeftArrow,
|
||||
"ArrowRight" | "RightArrow" => Key::RightArrow,
|
||||
|
||||
// Function row
|
||||
"F1" => Key::F1, "F2" => Key::F2, "F3" => Key::F3, "F4" => Key::F4,
|
||||
"F5" => Key::F5, "F6" => Key::F6, "F7" => Key::F7, "F8" => Key::F8,
|
||||
"F9" => Key::F9, "F10" => Key::F10, "F11" => Key::F11, "F12" => Key::F12,
|
||||
|
||||
// Digits
|
||||
"Digit0" | "Num0" => Key::Num0,
|
||||
"Digit1" | "Num1" => Key::Num1,
|
||||
"Digit2" | "Num2" => Key::Num2,
|
||||
"Digit3" | "Num3" => Key::Num3,
|
||||
"Digit4" | "Num4" => Key::Num4,
|
||||
"Digit5" | "Num5" => Key::Num5,
|
||||
"Digit6" | "Num6" => Key::Num6,
|
||||
"Digit7" | "Num7" => Key::Num7,
|
||||
"Digit8" | "Num8" => Key::Num8,
|
||||
"Digit9" | "Num9" => Key::Num9,
|
||||
|
||||
// Letters — browser uses "KeyA" style which already matches rdev.
|
||||
"KeyA" => Key::KeyA, "KeyB" => Key::KeyB, "KeyC" => Key::KeyC,
|
||||
"KeyD" => Key::KeyD, "KeyE" => Key::KeyE, "KeyF" => Key::KeyF,
|
||||
"KeyG" => Key::KeyG, "KeyH" => Key::KeyH, "KeyI" => Key::KeyI,
|
||||
"KeyJ" => Key::KeyJ, "KeyK" => Key::KeyK, "KeyL" => Key::KeyL,
|
||||
"KeyM" => Key::KeyM, "KeyN" => Key::KeyN, "KeyO" => Key::KeyO,
|
||||
"KeyP" => Key::KeyP, "KeyQ" => Key::KeyQ, "KeyR" => Key::KeyR,
|
||||
"KeyS" => Key::KeyS, "KeyT" => Key::KeyT, "KeyU" => Key::KeyU,
|
||||
"KeyV" => Key::KeyV, "KeyW" => Key::KeyW, "KeyX" => Key::KeyX,
|
||||
"KeyY" => Key::KeyY, "KeyZ" => Key::KeyZ,
|
||||
|
||||
// Punctuation / symbols
|
||||
"Backquote" | "BackQuote" => Key::BackQuote,
|
||||
"Minus" => Key::Minus,
|
||||
"Equal" => Key::Equal,
|
||||
"BracketLeft" | "LeftBracket" => Key::LeftBracket,
|
||||
"BracketRight" | "RightBracket" => Key::RightBracket,
|
||||
"Semicolon" | "SemiColon" => Key::SemiColon,
|
||||
"Quote" => Key::Quote,
|
||||
"Backslash" | "BackSlash" => Key::BackSlash,
|
||||
"Comma" => Key::Comma,
|
||||
"Period" | "Dot" => Key::Dot,
|
||||
"Slash" => Key::Slash,
|
||||
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
+355
-2
@@ -1,14 +1,62 @@
|
||||
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
|
||||
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
||||
|
||||
mod accessibility;
|
||||
mod audio_capture;
|
||||
mod audio_output;
|
||||
mod clipboard;
|
||||
mod focus_capture;
|
||||
#[cfg(desktop)]
|
||||
mod hotkey_monitor;
|
||||
#[cfg(desktop)]
|
||||
mod key_codes;
|
||||
mod synthetic_keys;
|
||||
|
||||
use std::sync::Mutex;
|
||||
use tauri::{command, State, Manager, WindowEvent, Emitter, Listener, RunEvent};
|
||||
use tauri::{command, State, Manager, WindowEvent, Emitter, Listener, RunEvent, WebviewUrl, WebviewWindowBuilder, PhysicalPosition};
|
||||
use tauri_plugin_shell::ShellExt;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub const DICTATE_WINDOW_LABEL: &str = "dictate";
|
||||
const DICTATE_WINDOW_WIDTH: f64 = 420.0;
|
||||
const DICTATE_WINDOW_HEIGHT: f64 = 64.0;
|
||||
|
||||
/// Create the floating dictate webview up front, hidden. The HotkeyMonitor
|
||||
/// shows it on chord-start; the frontend hides it when the capture pipeline
|
||||
/// finishes. Starting it at setup avoids a race where the first chord fires
|
||||
/// before the webview has had a chance to subscribe to the `dictate:*` events.
|
||||
#[cfg(desktop)]
|
||||
fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewWindow> {
|
||||
let window = WebviewWindowBuilder::new(
|
||||
app,
|
||||
DICTATE_WINDOW_LABEL,
|
||||
WebviewUrl::App("?view=dictate".into()),
|
||||
)
|
||||
.title("Voicebox Dictate")
|
||||
.inner_size(DICTATE_WINDOW_WIDTH, DICTATE_WINDOW_HEIGHT)
|
||||
.decorations(false)
|
||||
.transparent(true)
|
||||
.always_on_top(true)
|
||||
// Follow the user across macOS Spaces / virtual desktops instead of
|
||||
// being pinned to the Space where the window was first created.
|
||||
.visible_on_all_workspaces(true)
|
||||
.skip_taskbar(true)
|
||||
.resizable(false)
|
||||
.shadow(false)
|
||||
.visible(false)
|
||||
.build()?;
|
||||
|
||||
if let Some(monitor) = window.current_monitor()? {
|
||||
let monitor_size = monitor.size();
|
||||
let win_size = window.outer_size()?;
|
||||
let x = (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = (monitor_size.height as f64 * 0.04) as i32;
|
||||
window.set_position(PhysicalPosition::new(x, y))?;
|
||||
}
|
||||
|
||||
Ok(window)
|
||||
}
|
||||
|
||||
const LEGACY_PORT: u16 = 8000;
|
||||
const SERVER_PORT: u16 = 17493;
|
||||
|
||||
@@ -709,6 +757,273 @@ fn stop_audio_playback(
|
||||
state.stop_all_playback()
|
||||
}
|
||||
|
||||
/// Identifier of the Voicebox app itself — used to short-circuit auto-paste
|
||||
/// when the user fires a chord while focus was inside one of our own
|
||||
/// windows. Paste into Voicebox-internal targets is step 6 territory and
|
||||
/// goes through a different (JS-side) injection path.
|
||||
///
|
||||
/// Value matches what `focus_capture::capture_focus` writes into
|
||||
/// `FocusSnapshot::bundle_id` on the current platform — reverse-DNS bundle
|
||||
/// id on macOS, lowercased exe basename on Windows/Linux.
|
||||
#[cfg(target_os = "macos")]
|
||||
const VOICEBOX_BUNDLE_ID: &str = "sh.voicebox.app";
|
||||
#[cfg(target_os = "windows")]
|
||||
const VOICEBOX_BUNDLE_ID: &str = "voicebox.exe";
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
const VOICEBOX_BUNDLE_ID: &str = "voicebox";
|
||||
|
||||
/// Milliseconds to wait between activating the target app and firing the
|
||||
/// synthetic ⌘V, giving AppKit time to finish re-ordering windows and
|
||||
/// restoring its last-focused field.
|
||||
const POST_ACTIVATE_SETTLE_MS: u64 = 120;
|
||||
|
||||
/// Milliseconds the staged text lives on the clipboard after the paste
|
||||
/// keystroke, before we restore the user's original clipboard contents.
|
||||
/// Too short and slow apps haven't consumed the paste yet; too long and
|
||||
/// the user sees our text if they look at their clipboard manager.
|
||||
const PASTE_CONSUME_MS: u64 = 400;
|
||||
|
||||
/// Reports whether the process currently has macOS Accessibility trust.
|
||||
/// Used by the settings UI and the paste debug harness to decide whether
|
||||
/// synthetic key events will actually land.
|
||||
#[command]
|
||||
fn check_accessibility_permission() -> bool {
|
||||
accessibility::is_trusted()
|
||||
}
|
||||
|
||||
/// Push a new chord configuration into the running `HotkeyMonitor`. The
|
||||
/// frontend calls this both at startup (replaying the saved chord from
|
||||
/// capture_settings) and any time the user edits the chord in the picker —
|
||||
/// no app restart needed because the engine swap is atomic under the
|
||||
/// monitor's mutex.
|
||||
///
|
||||
/// Returns an error when a key name doesn't map to an `rdev::Key`, so the
|
||||
/// picker UI can surface "this key isn't supported" instead of silently
|
||||
/// dropping it from the chord.
|
||||
#[cfg(desktop)]
|
||||
#[command]
|
||||
fn update_chord_bindings(
|
||||
monitor: State<'_, hotkey_monitor::HotkeyMonitor>,
|
||||
push_to_talk: Vec<String>,
|
||||
toggle_to_talk: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
use hotkey_monitor::{Bindings, ChordAction};
|
||||
use rdev::Key;
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn build_chord(name: &str, names: &[String]) -> Result<HashSet<Key>, String> {
|
||||
if names.is_empty() {
|
||||
return Err(format!("{name} chord must have at least one key"));
|
||||
}
|
||||
let mut chord = HashSet::new();
|
||||
for raw in names {
|
||||
let key = key_codes::key_from_str(raw)
|
||||
.ok_or_else(|| format!("Unsupported key in {name} chord: {raw}"))?;
|
||||
chord.insert(key);
|
||||
}
|
||||
Ok(chord)
|
||||
}
|
||||
|
||||
let push_chord = build_chord("push-to-talk", &push_to_talk)?;
|
||||
let toggle_chord = build_chord("toggle-to-talk", &toggle_to_talk)?;
|
||||
|
||||
let mut bindings = Bindings::new();
|
||||
bindings.insert(ChordAction::PushToTalk, push_chord);
|
||||
bindings.insert(ChordAction::ToggleToTalk, toggle_chord);
|
||||
|
||||
monitor.update_bindings(bindings);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Open the Privacy & Security → Accessibility pane in System Settings so
|
||||
/// the user can grant the permission. The URL scheme is stable across
|
||||
/// macOS 10.14–15; no-op on other platforms.
|
||||
#[command]
|
||||
fn open_accessibility_settings(app: tauri::AppHandle) -> Result<(), String> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let url = "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility";
|
||||
app.shell()
|
||||
.open(url, None)
|
||||
.map_err(|e| format!("Failed to open Accessibility settings: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
let _ = app;
|
||||
Err("Accessibility settings pane is only implemented on macOS".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Deliver `text` into the UI that had focus when the chord fired.
|
||||
///
|
||||
/// Pipeline: activate the captured PID → settle → save the user's
|
||||
/// clipboard → write `text` → fire ⌘V → wait for the target to consume it
|
||||
/// → restore the original clipboard.
|
||||
///
|
||||
/// Skips (returns `false`) without touching anything when:
|
||||
/// - `focus.bundle_id` is Voicebox itself — step 6 will inject directly
|
||||
/// into our own webview; pasting would just double-insert or miss the
|
||||
/// real target.
|
||||
/// - Accessibility is not trusted — `CGEventPost` would silently drop the
|
||||
/// keystroke, leaving the user's clipboard clobbered with nothing to
|
||||
/// show for it.
|
||||
///
|
||||
/// Returns `true` when the paste sequence completed end-to-end.
|
||||
#[command]
|
||||
async fn paste_final_text(
|
||||
text: String,
|
||||
focus: focus_capture::FocusSnapshot,
|
||||
) -> Result<bool, String> {
|
||||
if focus.bundle_id.as_deref() == Some(VOICEBOX_BUNDLE_ID) {
|
||||
return Ok(false);
|
||||
}
|
||||
if !accessibility::is_trusted() {
|
||||
return Err(
|
||||
"Accessibility permission required for auto-paste. Open System Settings → Privacy & Security → Accessibility and enable Voicebox."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
focus_capture::activate_pid(focus.pid)?;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await;
|
||||
|
||||
let snapshot = clipboard::save_clipboard()?;
|
||||
clipboard::write_text(&text)?;
|
||||
synthetic_keys::send_paste()?;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(PASTE_CONSUME_MS)).await;
|
||||
clipboard::restore_clipboard(&snapshot)?;
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Inspect the currently focused UI element. Returns the owning app's PID,
|
||||
/// bundle id, and AX role. Useful for sanity-checking the focus pipeline
|
||||
/// before committing to a paste.
|
||||
#[command]
|
||||
fn debug_capture_focus() -> Result<focus_capture::FocusSnapshot, String> {
|
||||
focus_capture::capture_focus()
|
||||
}
|
||||
|
||||
/// Full auto-paste rehearsal: snapshot the focus target now, sleep
|
||||
/// `drift_ms` so the user can deliberately switch to a different app
|
||||
/// (proving we don't paste into whichever window is frontmost when the
|
||||
/// transcribe finishes), then activate the captured PID, stage `text`,
|
||||
/// fire ⌘V, and restore the clipboard.
|
||||
#[command]
|
||||
async fn debug_focus_roundtrip(
|
||||
text: String,
|
||||
drift_ms: u64,
|
||||
post_paste_delay_ms: u64,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
if !accessibility::is_trusted() {
|
||||
return Err(
|
||||
"Accessibility permission not granted. Open System Settings → Privacy & Security → Accessibility and enable Voicebox."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot = focus_capture::capture_focus()?;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(drift_ms)).await;
|
||||
|
||||
focus_capture::activate_pid(snapshot.pid)?;
|
||||
// Give AppKit a beat to process the activation before the synthetic
|
||||
// Cmd+V arrives — without this the paste sometimes races ahead of the
|
||||
// window-ordering animation and lands in the previous frontmost app.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(120)).await;
|
||||
|
||||
let clip = clipboard::save_clipboard()?;
|
||||
let after_write = clipboard::write_text(&text)?;
|
||||
synthetic_keys::send_paste()?;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(post_paste_delay_ms)).await;
|
||||
let before_restore = clipboard::current_change_count()?;
|
||||
clipboard::restore_clipboard(&clip)?;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"focus": snapshot,
|
||||
"change_count_after_write": after_write,
|
||||
"change_count_before_restore": before_restore,
|
||||
"clobbered_during_paste": before_restore != after_write,
|
||||
}))
|
||||
}
|
||||
|
||||
/// End-to-end smoke test for the auto-paste pipeline: save the user's
|
||||
/// clipboard, stage `text`, optionally wait `pre_paste_delay_ms` so the
|
||||
/// caller has time to focus the target app, synthesise ⌘V, wait
|
||||
/// `post_paste_delay_ms` for the target app to consume the event, and put
|
||||
/// the original clipboard back.
|
||||
///
|
||||
/// Short-circuits when Accessibility permission is missing — without it
|
||||
/// `CGEventPost` silently drops events, so running the full sequence
|
||||
/// would just clobber the clipboard with nothing to show for it.
|
||||
#[command]
|
||||
async fn debug_paste_text(
|
||||
text: String,
|
||||
pre_paste_delay_ms: u64,
|
||||
post_paste_delay_ms: u64,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
if !accessibility::is_trusted() {
|
||||
return Err(
|
||||
"Accessibility permission not granted. Open System Settings → Privacy & Security → Accessibility and enable Voicebox, then try again."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot = clipboard::save_clipboard()?;
|
||||
let before = snapshot.change_count();
|
||||
let after_write = clipboard::write_text(&text)?;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(pre_paste_delay_ms)).await;
|
||||
|
||||
synthetic_keys::send_paste()?;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(post_paste_delay_ms)).await;
|
||||
|
||||
let before_restore = clipboard::current_change_count()?;
|
||||
clipboard::restore_clipboard(&snapshot)?;
|
||||
let after_restore = clipboard::current_change_count()?;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"change_count_before": before,
|
||||
"change_count_after_write": after_write,
|
||||
"change_count_before_restore": before_restore,
|
||||
"change_count_after_restore": after_restore,
|
||||
"clobbered_during_paste": before_restore != after_write,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Manual smoke test for the clipboard snapshot/restore primitives used by
|
||||
/// the auto-paste pipeline. Stages `text` on the pasteboard, waits
|
||||
/// `hold_ms` so the caller can ⌘V into another app, then puts the original
|
||||
/// clipboard contents back. The return value reports the change-count deltas
|
||||
/// so the harness can verify no third party mutated the clipboard mid-paste.
|
||||
#[command]
|
||||
async fn debug_clipboard_roundtrip(
|
||||
text: String,
|
||||
hold_ms: u64,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let snapshot = clipboard::save_clipboard()?;
|
||||
let before = snapshot.change_count();
|
||||
let item_count = snapshot.item_count();
|
||||
let after_write = clipboard::write_text(&text)?;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(hold_ms)).await;
|
||||
|
||||
let before_restore = clipboard::current_change_count()?;
|
||||
clipboard::restore_clipboard(&snapshot)?;
|
||||
let after_restore = clipboard::current_change_count()?;
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"saved_items": item_count,
|
||||
"change_count_before": before,
|
||||
"change_count_after_write": after_write,
|
||||
"change_count_before_restore": before_restore,
|
||||
"change_count_after_restore": after_restore,
|
||||
"clobbered_during_hold": before_restore != after_write,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
tauri::Builder::default()
|
||||
@@ -728,6 +1043,36 @@ pub fn run() {
|
||||
{
|
||||
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
|
||||
app.handle().plugin(tauri_plugin_process::init())?;
|
||||
|
||||
if let Err(e) = build_dictate_window(app.handle()) {
|
||||
eprintln!("Failed to pre-create dictate window: {}", e);
|
||||
}
|
||||
|
||||
let monitor = hotkey_monitor::HotkeyMonitor::spawn(
|
||||
app.handle().clone(),
|
||||
hotkey_monitor::default_bindings(),
|
||||
);
|
||||
// Stored as state so the chord-picker UI can call
|
||||
// `update_chord_bindings` to live-swap the engine's chords
|
||||
// without restarting the listener thread.
|
||||
app.manage(monitor);
|
||||
|
||||
// The frontend emits `dictate:hide` whenever the pill cycle
|
||||
// finishes (rest-fade → hidden). `hide()` alone has been
|
||||
// unreliable for transparent always-on-top windows on macOS
|
||||
// — the NSWindow lingers as an invisible click target that
|
||||
// steals focus to the Voicebox app when the user clicks
|
||||
// where it used to be. Park the window off-screen and mark
|
||||
// it click-through as well, so even if `hide()` no-ops the
|
||||
// user sees and interacts with nothing.
|
||||
let handle_for_hide = app.handle().clone();
|
||||
app.handle().listen("dictate:hide", move |_event| {
|
||||
if let Some(window) = handle_for_hide.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
let _ = window.set_ignore_cursor_events(true);
|
||||
let _ = window.set_position(PhysicalPosition::new(-10_000, -10_000));
|
||||
let _ = window.hide();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Hide title bar icon on Windows
|
||||
@@ -797,7 +1142,15 @@ pub fn run() {
|
||||
is_system_audio_supported,
|
||||
list_audio_output_devices,
|
||||
play_audio_to_devices,
|
||||
stop_audio_playback
|
||||
stop_audio_playback,
|
||||
debug_clipboard_roundtrip,
|
||||
debug_paste_text,
|
||||
debug_capture_focus,
|
||||
debug_focus_roundtrip,
|
||||
check_accessibility_permission,
|
||||
open_accessibility_settings,
|
||||
paste_final_text,
|
||||
update_chord_bindings
|
||||
])
|
||||
.on_window_event({
|
||||
let closing = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
//! Synthetic keyboard event posting for the auto-paste pipeline.
|
||||
//!
|
||||
//! `send_paste` fires the four-event paste sequence onto the OS input
|
||||
//! pipeline so the focused app performs its native paste action against
|
||||
//! whatever the clipboard module has just staged.
|
||||
//!
|
||||
//! - **macOS** — Cmd down, V down with Cmd flag, V up with Cmd flag, Cmd
|
||||
//! up via `CGEventPost` at `kCGHIDEventTap`. Accessibility permission is
|
||||
//! load-bearing: without it the system swallows the events silently, so
|
||||
//! callers must gate on [`crate::accessibility::is_trusted`].
|
||||
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
|
||||
//! permission gate, but UAC/UIPI blocks delivery into elevated target
|
||||
//! windows when we run non-elevated — nothing we can do short of also
|
||||
//! running elevated.
|
||||
//!
|
||||
//! The virtual keycode used for V is `kVK_ANSI_V` (9) on macOS and `VK_V`
|
||||
//! (0x56) on Windows. Both are layout-dependent — they mean "the physical
|
||||
//! key in the QWERTY V position" — so on Dvorak / Colemak this would fire
|
||||
//! the wrong shortcut. A later pass will resolve the current layout's V
|
||||
//! keycode per-platform (`TISCopyCurrentKeyboardInputSource` +
|
||||
//! `UCKeyTranslate` on macOS; `VkKeyScanExW` on Windows).
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use std::ffi::c_void;
|
||||
|
||||
#[repr(C)]
|
||||
pub struct CGEvent {
|
||||
_opaque: [u8; 0],
|
||||
}
|
||||
pub type CGEventRef = *mut CGEvent;
|
||||
|
||||
#[repr(C)]
|
||||
pub struct CGEventSource {
|
||||
_opaque: [u8; 0],
|
||||
}
|
||||
pub type CGEventSourceRef = *mut CGEventSource;
|
||||
|
||||
pub type CGEventTapLocation = u32;
|
||||
pub type CGKeyCode = u16;
|
||||
pub type CGEventFlags = u64;
|
||||
pub type CGEventSourceStateID = i32;
|
||||
|
||||
/// `kCGHIDEventTap` — posted events enter at the HID level so every
|
||||
/// downstream tap (including the target app) sees them exactly as if the
|
||||
/// hardware had produced them.
|
||||
pub const K_CG_HID_EVENT_TAP: CGEventTapLocation = 0;
|
||||
|
||||
/// `kCGEventSourceStateHIDSystemState` — mimics hardware, which is what
|
||||
/// we want: modifier bookkeeping inside target apps stays consistent.
|
||||
pub const K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE: CGEventSourceStateID = 1;
|
||||
|
||||
/// `kCGEventFlagMaskCommand` — the Cmd modifier bit inside `CGEventFlags`.
|
||||
pub const K_CG_EVENT_FLAG_MASK_COMMAND: CGEventFlags = 0x00100000;
|
||||
|
||||
/// `kVK_ANSI_V`.
|
||||
pub const KEYCODE_V: CGKeyCode = 9;
|
||||
/// `kVK_Command` (left Cmd).
|
||||
pub const KEYCODE_LEFT_CMD: CGKeyCode = 0x37;
|
||||
|
||||
#[link(name = "CoreGraphics", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn CGEventSourceCreate(state_id: CGEventSourceStateID) -> CGEventSourceRef;
|
||||
pub fn CGEventCreateKeyboardEvent(
|
||||
source: CGEventSourceRef,
|
||||
virtual_key: CGKeyCode,
|
||||
key_down: bool,
|
||||
) -> CGEventRef;
|
||||
pub fn CGEventSetFlags(event: CGEventRef, flags: CGEventFlags);
|
||||
pub fn CGEventPost(tap: CGEventTapLocation, event: CGEventRef);
|
||||
}
|
||||
|
||||
#[link(name = "CoreFoundation", kind = "framework")]
|
||||
extern "C" {
|
||||
pub fn CFRelease(cf: *const c_void);
|
||||
}
|
||||
}
|
||||
|
||||
/// Post the four-event Cmd+V sequence to the HID event tap.
|
||||
///
|
||||
/// Returns after the events are queued — there's no completion callback,
|
||||
/// so callers should sleep briefly afterwards to let the target app
|
||||
/// process the paste before any follow-up (e.g. clipboard restore).
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
use ffi::*;
|
||||
|
||||
unsafe {
|
||||
let source = CGEventSourceCreate(K_CG_EVENT_SOURCE_STATE_HID_SYSTEM_STATE);
|
||||
if source.is_null() {
|
||||
return Err("CGEventSourceCreate returned null".into());
|
||||
}
|
||||
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
||||
|
||||
let events = [
|
||||
(KEYCODE_LEFT_CMD, true, 0),
|
||||
(KEYCODE_V, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(KEYCODE_V, false, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(KEYCODE_LEFT_CMD, false, 0),
|
||||
];
|
||||
|
||||
// Build the four events up front so CFRelease happens after all posts.
|
||||
// Posting in a loop that interleaved create → post → release would
|
||||
// work, but keeping the events alive for the full sequence matches
|
||||
// the pattern CGEventPost's docs show and is easier to reason about.
|
||||
let mut guards = Vec::with_capacity(events.len());
|
||||
let mut created = Vec::with_capacity(events.len());
|
||||
|
||||
for (key, down, flags) in events {
|
||||
let event = CGEventCreateKeyboardEvent(source, key, down);
|
||||
if event.is_null() {
|
||||
return Err(format!(
|
||||
"CGEventCreateKeyboardEvent(key={}, down={}) returned null",
|
||||
key, down
|
||||
));
|
||||
}
|
||||
let guard = scopeguard::guard(event, |e| CFRelease(e as *const c_void));
|
||||
if flags != 0 {
|
||||
CGEventSetFlags(event, flags);
|
||||
}
|
||||
created.push(event);
|
||||
guards.push(guard);
|
||||
}
|
||||
|
||||
for event in created {
|
||||
CGEventPost(K_CG_HID_EVENT_TAP, event);
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod win {
|
||||
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
||||
INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_KEYUP,
|
||||
VIRTUAL_KEY,
|
||||
};
|
||||
|
||||
pub fn make_key(vk: VIRTUAL_KEY, up: bool) -> INPUT {
|
||||
let flags = if up {
|
||||
KEYEVENTF_KEYUP
|
||||
} else {
|
||||
KEYBD_EVENT_FLAGS(0)
|
||||
};
|
||||
INPUT {
|
||||
r#type: INPUT_KEYBOARD,
|
||||
Anonymous: INPUT_0 {
|
||||
ki: KEYBDINPUT {
|
||||
wVk: vk,
|
||||
wScan: 0,
|
||||
dwFlags: flags,
|
||||
time: 0,
|
||||
dwExtraInfo: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
||||
SendInput, INPUT, VK_CONTROL, VK_V,
|
||||
};
|
||||
|
||||
// Four-event Ctrl+V sequence. Matches the macOS CGEvent pattern: the
|
||||
// modifier brackets the letter so the target app sees a fully formed
|
||||
// accelerator rather than a lone V. `dwExtraInfo` is zero — we're not
|
||||
// tagging these as "ours" because no consumer in the paste path needs
|
||||
// to distinguish synthetic events from hardware ones.
|
||||
let events = [
|
||||
win::make_key(VK_CONTROL, false),
|
||||
win::make_key(VK_V, false),
|
||||
win::make_key(VK_V, true),
|
||||
win::make_key(VK_CONTROL, true),
|
||||
];
|
||||
|
||||
unsafe {
|
||||
let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32);
|
||||
if sent as usize != events.len() {
|
||||
return Err(format!(
|
||||
"SendInput delivered {} of {} events — the input desktop may be locked (secure attention sequence) or a higher-integrity window is intercepting.",
|
||||
sent,
|
||||
events.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
|
||||
pub fn send_paste() -> Result<(), String> {
|
||||
Err("synthetic paste is not yet implemented on this platform".into())
|
||||
}
|
||||
Reference in New Issue
Block a user