fix(dictation): preserve native focus and fullscreen injection

Carry focus and auto-paste permission per capture, support dictation over macOS fullscreen Spaces, preserve native window behavior flags, and abort paste when the pill cannot be hidden safely.\n\nVerified: frontend CI; cargo check; diff and security scans. Packaged multi-Space validation remains a release gate.
This commit is contained in:
Jamie Pine
2026-07-19 17:44:25 -07:00
parent 5787e36611
commit 88e72d5da2
6 changed files with 299 additions and 43 deletions
@@ -39,20 +39,13 @@ export function DictateWindow() {
// user opted into keeping the microphone ready.
const [micWarm, setMicWarm] = useState(false);
// Snapshot of the focused UI element at chord-start, shipped over from
// Rust on the ``dictate:start`` payload. Held in a ref so it survives
// the 12 s transcribe + refine window — the paste only fires once the
// final text comes back.
const focusRef = useRef<FocusSnapshot | null>(null);
const session = useCaptureRecordingSession({
keepMicWarm: micWarm,
onFinalText: async (text, _capture, allowAutoPaste) => {
const focus = focusRef.current;
// Consume-once: a second chord before this fires would overwrite
// focusRef, but nulling it here guards against the late-arriving
// refine-result firing a paste after the user has moved on.
focusRef.current = null;
onFinalText: async (text, _capture, allowAutoPaste, context) => {
// Focus is the snapshot taken at chord-start and threaded through as this
// take's context, so it survives the 12 s transcribe + refine window and
// overlapping dictations can't paste into each other's target.
const focus = context as FocusSnapshot | null;
if (!allowAutoPaste) return;
if (!focus || !text.trim()) return;
try {
@@ -81,8 +74,7 @@ export function DictateWindow() {
const unlistens: UnlistenFn[] = [];
const registrations = [
listen<{ focus: FocusSnapshot | null }>('dictate:start', (event) => {
focusRef.current = event.payload?.focus ?? null;
sessionRef.current.startRecording();
sessionRef.current.startRecording(event.payload?.focus ?? null);
}),
listen('dictate:stop', () => {
// Forward stops that arrive while getUserMedia is still resolving.
+8 -4
View File
@@ -4,7 +4,11 @@ import { convertToWav } from '@/lib/utils/audio';
interface UseAudioRecordingOptions {
maxDurationSeconds?: number;
onRecordingComplete?: (blob: Blob, duration?: number) => void;
// ``context`` is whatever was handed to ``startRecording`` for this take,
// threaded back untouched so callers can correlate the result with the
// recording it came from (the dictate window pairs it with the focus
// snapshot captured at chord-start).
onRecordingComplete?: (blob: Blob, duration?: number, context?: unknown) => void;
/**
* Keep the microphone ``MediaStream`` open between recordings instead of
* tearing it down on every stop. This is what removes the "first words get
@@ -184,7 +188,7 @@ export function useAudioRecording({
}, [keepWarm, acquireStream]);
const startRecording = useCallback(
async () => {
async (context?: unknown) => {
// A second chord can arrive while the first one is still waiting on
// getUserMedia. Never create overlapping MediaRecorders on the same
// coalesced stream; the original take will honor any deferred stop.
@@ -275,11 +279,11 @@ export function useAudioRecording({
// Convert to WAV format to avoid needing ffmpeg on backend
try {
const wavBlob = await convertToWav(webmBlob);
onRecordingComplete?.(wavBlob, recordedDuration);
onRecordingComplete?.(wavBlob, recordedDuration, context);
} catch (err) {
console.error('Error converting audio to WAV:', err);
// Fallback to original blob if conversion fails
onRecordingComplete?.(webmBlob, recordedDuration);
onRecordingComplete?.(webmBlob, recordedDuration, context);
}
};
+50 -23
View File
@@ -60,8 +60,9 @@ export interface UseCaptureRecordingSessionOptions {
/**
* Fired after a capture row is created on the server. Callers can use this
* to select the new capture or emit a Tauri event to a sibling window.
* ``context`` is whatever was passed to ``startRecording`` for this take.
*/
onCaptureCreated?: (capture: CaptureResponse) => void;
onCaptureCreated?: (capture: CaptureResponse, context?: unknown) => void;
/**
* Fired with the final delivered text — refined if ``auto_refine`` was on
* for this capture, raw transcript otherwise. Used by the floating
@@ -69,12 +70,14 @@ export interface UseCaptureRecordingSessionOptions {
*
* ``allowAutoPaste`` snapshots the setting at chord-start so a refine that
* lands after the user flips the toggle still uses the value the capture
* was created under.
* was created under. ``context`` is the value passed to ``startRecording``
* for this take, so overlapping dictations can't cross their targets.
*/
onFinalText?: (
text: string,
capture: CaptureResponse,
allowAutoPaste: boolean,
context?: unknown,
) => void;
}
@@ -85,7 +88,7 @@ export interface UseCaptureRecordingSessionResult {
isRecording: boolean;
isUploading: boolean;
isRefining: boolean;
startRecording: () => void;
startRecording: (context?: unknown) => void;
stopRecording: () => void;
toggleRecording: () => void;
dismissError: () => void;
@@ -128,10 +131,13 @@ export function useCaptureRecordingSession(
const onFinalTextRef = useRef(options.onFinalText);
onFinalTextRef.current = options.onFinalText;
// Snapshot of ``allow_auto_paste`` from the capture-create response —
// held so the refine onSuccess (which only sees the plain CaptureResponse)
// can still pass the original setting through to onFinalText.
const allowAutoPasteRef = useRef<boolean>(true);
// Per-capture recording context and its ``allow_auto_paste`` snapshot, keyed
// by capture id so a refine that resolves after another dictation started
// still delivers to the right target with the setting the capture was created
// under. Populated on capture-create and consumed once the final text lands.
const captureDeliveryRef = useRef<Map<string, { context: unknown; allowAutoPaste: boolean }>>(
new Map(),
);
const clearRestTimer = useCallback(() => {
if (restTimerRef.current !== null) {
@@ -197,20 +203,34 @@ export function useCaptureRecordingSession(
queryClient.invalidateQueries({ queryKey: ['captures'] });
broadcastUpdated(captureId);
if (pillStateRef.current === 'refining') scheduleHidePill();
const delivery = captureDeliveryRef.current.get(captureId);
captureDeliveryRef.current.delete(captureId);
const finalText = data.transcript_refined ?? data.transcript_raw;
if (finalText) {
onFinalTextRef.current?.(finalText, data, allowAutoPasteRef.current);
onFinalTextRef.current?.(
finalText,
data,
delivery?.allowAutoPaste ?? true,
delivery?.context,
);
}
},
onError: (err: Error) => {
onError: (err: Error, captureId) => {
captureDeliveryRef.current.delete(captureId);
showError(err.message || 'Refinement failed');
},
});
const uploadMutation = useMutation({
mutationFn: async ({ file, source }: { file: File; source: CaptureSource }) =>
apiClient.createCapture(file, { source }),
onSuccess: (capture) => {
mutationFn: async ({
file,
source,
}: {
file: File;
source: CaptureSource;
context?: unknown;
}) => apiClient.createCapture(file, { source }),
onSuccess: (capture, { context }) => {
queryClient.setQueryData<CaptureListResponse>(['captures'], (prev) => {
if (!prev) return prev;
if (prev.items.some((c) => c.id === capture.id)) return prev;
@@ -218,9 +238,12 @@ export function useCaptureRecordingSession(
});
queryClient.invalidateQueries({ queryKey: ['captures'] });
broadcastCreated(capture);
onCaptureCreatedRef.current?.(capture);
allowAutoPasteRef.current = capture.allow_auto_paste;
onCaptureCreatedRef.current?.(capture, context);
if (capture.auto_refine) {
captureDeliveryRef.current.set(capture.id, {
context,
allowAutoPaste: capture.allow_auto_paste,
});
setPillState('refining');
refineMutation.mutate(capture.id);
} else {
@@ -230,6 +253,7 @@ export function useCaptureRecordingSession(
capture.transcript_raw,
capture,
capture.allow_auto_paste,
context,
);
}
}
@@ -258,7 +282,7 @@ export function useCaptureRecordingSession(
releaseWarm,
} = useAudioRecording({
keepWarm: options.keepMicWarm ?? false,
onRecordingComplete: (blob, recordedDuration) => {
onRecordingComplete: (blob, recordedDuration, context) => {
// Trigger-happy tap — MediaRecorder hasn't emitted a usable chunk yet
// so the blob is empty or unparseable. Surface it as a transient pill
// so the user sees their recording was recognised and canceled.
@@ -276,7 +300,7 @@ export function useCaptureRecordingSession(
const file = new File([blob], `dictation-${Date.now()}.${extension}`, {
type: blob.type,
});
uploadMutation.mutate({ file, source: 'dictation' });
uploadMutation.mutate({ file, source: 'dictation', context });
},
});
@@ -286,13 +310,16 @@ export function useCaptureRecordingSession(
}
}, [recordError, showError]);
const startRecording = useCallback(() => {
if (isRecording) return;
clearRestTimer();
setFrozenElapsedMs(0);
setPillState('recording');
beginAudioRecording();
}, [isRecording, beginAudioRecording, clearRestTimer]);
const startRecording = useCallback(
(context?: unknown) => {
if (isRecording) return;
clearRestTimer();
setFrozenElapsedMs(0);
setPillState('recording');
beginAudioRecording(context);
},
[isRecording, beginAudioRecording, clearRestTimer],
);
const toggleRecording = useCallback(() => {
if (isRecording) {
+53
View File
@@ -198,6 +198,17 @@ pub fn capture_focus() -> Result<FocusSnapshot, String> {
&mut focused as *mut _,
);
if err != AX_ERROR_SUCCESS || focused.is_null() {
// Some apps (terminals, some Electron windows) expose no
// system-wide focused element. Rather than drop the dictation,
// fall back to the frontmost app so the transcript still injects
// there via activate + ⌘V.
if let Some(fp) = frontmost_pid() {
return Ok(FocusSnapshot {
pid: fp,
bundle_id: bundle_id_for_pid(fp),
role: None,
});
}
return Err(format!(
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
err
@@ -213,6 +224,26 @@ pub fn capture_focus() -> Result<FocusSnapshot, String> {
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
}
// If the focused element belongs to our OWN process, the dictate pill
// has transiently taken key focus for its WebKit mic capture — the
// system-wide AXFocusedUIElement then resolves to the pill instead of
// the user's real target, which would make us paste into ourselves (a
// no-op) or drop the text entirely. The pill is a non-activating panel
// so it never becomes the frontmost application; remap to the
// frontmost app, which is always the real dictation target.
let our_pid = std::process::id() as Pid;
if pid == our_pid {
if let Some(fp) = frontmost_pid() {
if fp != our_pid {
return Ok(FocusSnapshot {
pid: fp,
bundle_id: bundle_id_for_pid(fp),
role: None,
});
}
}
}
let role = {
let role_attr = cf_string_const("AXRole");
match role_attr {
@@ -301,6 +332,28 @@ pub fn activate_pid(pid: i32) -> Result<(), String> {
}
}
/// PID of the app the user is currently in (NSWorkspace frontmost). Used to
/// skip re-activation when the paste target never lost frontmost status —
/// activating an already-frontmost app is a no-op at best, and on
/// fullscreen Spaces macOS 26's cooperative activation returns NO for it,
/// which previously aborted the whole paste.
#[cfg(target_os = "macos")]
pub fn frontmost_pid() -> Option<i32> {
unsafe {
let _pool = AutoreleasePool::new();
let ws: Id = msg_send![class!(NSWorkspace), sharedWorkspace];
if ws.is_null() {
return None;
}
let app: Id = msg_send![ws, frontmostApplication];
if app.is_null() {
return None;
}
let pid: i32 = msg_send![app, processIdentifier];
Some(pid)
}
}
/// `true` when `NSRunningApplication` responds to
/// `yieldActivationToApplication:` — the macOS 14+ discriminator for the
/// cooperative-activation APIs. Cached since the answer doesn't change
+4
View File
@@ -269,6 +269,10 @@ fn apply_effect(app: &AppHandle, effect: Effect) {
// it out of whatever app the user was typing in, which is
// the opposite of what a dictation overlay should do.
let _ = window.show();
// Order the pill into the currently-active Space (incl. a
// foreign app's fullscreen Space) — see main.rs.
#[cfg(target_os = "macos")]
crate::force_order_front(&window);
let payload = serde_json::json!({ "focus": focus });
let _ = window.emit("dictate:start", payload);
}
+178 -2
View File
@@ -57,9 +57,141 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewW
window.set_position(PhysicalPosition::new(x, y))?;
}
// Make the pill able to float over other apps' native fullscreen Spaces.
#[cfg(target_os = "macos")]
apply_fullscreen_overlay_behavior(&window);
Ok(window)
}
// `object_setClass` — reclass a live object. Not re-exported by `objc`.
#[cfg(target_os = "macos")]
extern "C" {
fn object_setClass(
obj: *mut objc::runtime::Object,
cls: *const objc::runtime::Class,
) -> *const objc::runtime::Class;
}
/// `canBecomeKeyWindow` override for the pill panel. Borderless NSPanels
/// refuse key status by default, and WebKit rejects `getUserMedia` from a
/// document whose window can never become key — so recording silently fails.
/// Returning YES restores capture; the panel stays non-activating, so showing
/// it never steals focus from the app being dictated into.
#[cfg(target_os = "macos")]
extern "C" fn pill_panel_can_become_key(
_this: &objc::runtime::Object,
_sel: objc::runtime::Sel,
) -> objc::runtime::BOOL {
objc::runtime::YES
}
/// Lazily-registered NSPanel subclass for the dictate pill: key-capable (for
/// WebKit media capture) while remaining a panel (for fullscreen-Space join).
#[cfg(target_os = "macos")]
fn pill_panel_class() -> &'static objc::runtime::Class {
use objc::declare::ClassDecl;
use objc::runtime::{Class, Object, Sel, BOOL};
use objc::{class, sel, sel_impl};
static INIT: std::sync::Once = std::sync::Once::new();
INIT.call_once(|| {
let superclass = class!(NSPanel);
let mut decl =
ClassDecl::new("VoiceboxPillPanel", superclass).expect("register VoiceboxPillPanel");
unsafe {
decl.add_method(
sel!(canBecomeKeyWindow),
pill_panel_can_become_key as extern "C" fn(&Object, Sel) -> BOOL,
);
}
decl.register();
});
Class::get("VoiceboxPillPanel").expect("VoiceboxPillPanel registered")
}
/// Convert the dictate pill's NSWindow into a key-capable NSPanel and set the
/// collection behavior + window level required to appear over another app's
/// native macOS fullscreen Space.
///
/// A regular (Dock-icon) app's plain NSWindow is never admitted to a foreign
/// fullscreen Space regardless of collection-behavior flags or window level;
/// an NSPanel with the same flags is. NSPanel adds no instance variables over
/// NSWindow, so re-classing the live object is safe (the tauri-nspanel plugin
/// uses the same technique). Idempotent via an `isKindOfClass` guard. Runs on
/// the main thread because AppKit window mutation is main-thread-only.
#[cfg(target_os = "macos")]
pub fn apply_fullscreen_overlay_behavior(window: &tauri::WebviewWindow) {
let w = window.clone();
let dispatched = window.run_on_main_thread(move || {
use objc::runtime::{Object, NO, YES};
use objc::{class, msg_send, sel, sel_impl};
// NSWindowCollectionBehavior bit flags.
const CAN_JOIN_ALL_SPACES: u64 = 1 << 0;
const STATIONARY: u64 = 1 << 4;
const FULL_SCREEN_AUXILIARY: u64 = 1 << 8; // the flag stock Tauri never sets
const NONACTIVATING_PANEL: u64 = 1 << 7; // NSWindowStyleMaskNonactivatingPanel
// NSScreenSaverWindowLevel — floats above fullscreen app content.
const OVERLAY_WINDOW_LEVEL: i64 = 1000;
let ns_window = match w.ns_window() {
Ok(ptr) => ptr as *mut Object,
Err(e) => {
eprintln!("apply_fullscreen_overlay_behavior: ns_window() failed: {e}");
return;
}
};
if ns_window.is_null() {
return;
}
// SAFETY: ns_window is a valid, non-null NSWindow* owned by Tauri for
// the lifetime of the webview window; all selectors are standard AppKit
// calls and we are on the main thread.
unsafe {
let is_panel: objc::runtime::BOOL =
msg_send![ns_window, isKindOfClass: class!(NSPanel)];
if is_panel == NO {
object_setClass(ns_window, pill_panel_class());
let style: u64 = msg_send![ns_window, styleMask];
let _: () = msg_send![ns_window, setStyleMask: style | NONACTIVATING_PANEL];
let _: () = msg_send![ns_window, setHidesOnDeactivate: NO];
let _: () = msg_send![ns_window, setBecomesKeyOnlyIfNeeded: YES];
let _: () = msg_send![ns_window, setFloatingPanel: YES];
}
// Preserve behavior bits installed by Tauri/Tao instead of
// replacing them wholesale when adding the fullscreen flags.
let current_behavior: u64 = msg_send![ns_window, collectionBehavior];
let behavior =
current_behavior | CAN_JOIN_ALL_SPACES | FULL_SCREEN_AUXILIARY | STATIONARY;
let _: () = msg_send![ns_window, setCollectionBehavior: behavior];
let _: () = msg_send![ns_window, setLevel: OVERLAY_WINDOW_LEVEL];
}
});
if let Err(e) = dispatched {
eprintln!("apply_fullscreen_overlay_behavior: main-thread dispatch failed: {e}");
}
}
/// Order the pill front over whatever Space is active. `orderFrontRegardless`
/// works even though the app is inactive (it always is mid-dictation — the
/// user is typing in some other app). Called right after `window.show()`.
#[cfg(target_os = "macos")]
pub fn force_order_front(window: &tauri::WebviewWindow) {
let w = window.clone();
let _ = window.run_on_main_thread(move || {
use objc::runtime::Object;
use objc::{msg_send, sel, sel_impl};
if let Ok(ptr) = w.ns_window() {
let ns_window = ptr as *mut Object;
if !ns_window.is_null() {
unsafe {
let _: () = msg_send![ns_window, orderFrontRegardless];
}
}
}
});
}
/// Position, undo click-through, and show the dictate pill window.
///
/// The hide path parks the window at (-10_000, -10_000) and toggles
@@ -114,6 +246,8 @@ pub fn show_dictate_window(app: &tauri::AppHandle) {
}
let _ = window.set_ignore_cursor_events(false);
let _ = window.show();
#[cfg(target_os = "macos")]
force_order_front(&window);
}
const LEGACY_PORT: u16 = 8000;
@@ -1210,6 +1344,7 @@ fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> {
/// Returns `true` when the paste sequence completed end-to-end.
#[command]
async fn paste_final_text(
app: tauri::AppHandle,
text: String,
focus: focus_capture::FocusSnapshot,
) -> Result<bool, String> {
@@ -1223,15 +1358,56 @@ async fn paste_final_text(
);
}
focus_capture::activate_pid(focus.pid)?;
tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await;
// Only re-activate the target when the user actually left it. When it is
// still frontmost (the common case — the dictate pill is non-activating,
// and in a fullscreen Space the target never loses frontmost), activation
// is a no-op; on macOS 26 fullscreen Spaces `activate` returns NO for an
// already-frontmost app, which would otherwise abort the paste entirely.
#[cfg(target_os = "macos")]
let already_front = focus_capture::frontmost_pid() == Some(focus.pid);
#[cfg(not(target_os = "macos"))]
let already_front = false;
if !already_front {
focus_capture::activate_pid(focus.pid)?;
tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await;
}
let snapshot = clipboard::save_clipboard()?;
let after_write = clipboard::write_text(&text)?;
// Order the pill out for the synthetic ⌘V. The pill is a key-capable
// panel (WebKit needs that for getUserMedia), and over a fullscreen Space
// it holds key focus Spotlight-style — the keystroke would land in the
// pill instead of the target app. Hidden it can't swallow keys; restored
// immediately after so the webview never suspends between dictations.
#[cfg(target_os = "macos")]
let pill = app.get_webview_window(DICTATE_WINDOW_LABEL);
#[cfg(target_os = "macos")]
if let Some(ref w) = pill {
if let Err(e) = w.hide() {
// Never emit Cmd+V while the key-capable pill may still own focus.
// Undo our clipboard write when it is still safe, then abort.
if matches!(
clipboard::current_change_count(),
Ok(current) if current == after_write
) {
clipboard::restore_clipboard(&snapshot)?;
}
return Err(format!("Failed to hide dictate window before paste: {e}"));
}
tokio::time::sleep(std::time::Duration::from_millis(80)).await;
}
let paste_result = synthetic_keys::send_paste();
tokio::time::sleep(std::time::Duration::from_millis(PASTE_CONSUME_MS)).await;
#[cfg(target_os = "macos")]
if let Some(ref w) = pill {
let _ = w.show();
force_order_front(w);
}
let safe_to_restore = matches!(
clipboard::current_change_count(),
Ok(current) if current == after_write