diff --git a/app/src/components/DictateWindow/DictateWindow.tsx b/app/src/components/DictateWindow/DictateWindow.tsx index 99795d10..1d917e69 100644 --- a/app/src/components/DictateWindow/DictateWindow.tsx +++ b/app/src/components/DictateWindow/DictateWindow.tsx @@ -39,20 +39,13 @@ export function DictateWindow() { // user opted into keeping the microphone ready. const [micWarm, setMicWarm] = useState(false); - // Snapshot of the focused UI element at chord-start, shipped over from - // Rust on the ``dictate:start`` payload. Held in a ref so it survives - // the 1–2 s transcribe + refine window — the paste only fires once the - // final text comes back. - const focusRef = useRef(null); - const session = useCaptureRecordingSession({ keepMicWarm: micWarm, - onFinalText: async (text, _capture, allowAutoPaste) => { - const focus = focusRef.current; - // Consume-once: a second chord before this fires would overwrite - // focusRef, but nulling it here guards against the late-arriving - // refine-result firing a paste after the user has moved on. - focusRef.current = null; + onFinalText: async (text, _capture, allowAutoPaste, context) => { + // Focus is the snapshot taken at chord-start and threaded through as this + // take's context, so it survives the 1–2 s transcribe + refine window and + // overlapping dictations can't paste into each other's target. + const focus = context as FocusSnapshot | null; if (!allowAutoPaste) return; if (!focus || !text.trim()) return; try { @@ -81,8 +74,7 @@ export function DictateWindow() { const unlistens: UnlistenFn[] = []; const registrations = [ listen<{ focus: FocusSnapshot | null }>('dictate:start', (event) => { - focusRef.current = event.payload?.focus ?? null; - sessionRef.current.startRecording(); + sessionRef.current.startRecording(event.payload?.focus ?? null); }), listen('dictate:stop', () => { // Forward stops that arrive while getUserMedia is still resolving. diff --git a/app/src/lib/hooks/useAudioRecording.ts b/app/src/lib/hooks/useAudioRecording.ts index 5ee2200e..411a291b 100644 --- a/app/src/lib/hooks/useAudioRecording.ts +++ b/app/src/lib/hooks/useAudioRecording.ts @@ -4,7 +4,11 @@ import { convertToWav } from '@/lib/utils/audio'; interface UseAudioRecordingOptions { maxDurationSeconds?: number; - onRecordingComplete?: (blob: Blob, duration?: number) => void; + // ``context`` is whatever was handed to ``startRecording`` for this take, + // threaded back untouched so callers can correlate the result with the + // recording it came from (the dictate window pairs it with the focus + // snapshot captured at chord-start). + onRecordingComplete?: (blob: Blob, duration?: number, context?: unknown) => void; /** * Keep the microphone ``MediaStream`` open between recordings instead of * tearing it down on every stop. This is what removes the "first words get @@ -184,7 +188,7 @@ export function useAudioRecording({ }, [keepWarm, acquireStream]); const startRecording = useCallback( - async () => { + async (context?: unknown) => { // A second chord can arrive while the first one is still waiting on // getUserMedia. Never create overlapping MediaRecorders on the same // coalesced stream; the original take will honor any deferred stop. @@ -275,11 +279,11 @@ export function useAudioRecording({ // Convert to WAV format to avoid needing ffmpeg on backend try { const wavBlob = await convertToWav(webmBlob); - onRecordingComplete?.(wavBlob, recordedDuration); + onRecordingComplete?.(wavBlob, recordedDuration, context); } catch (err) { console.error('Error converting audio to WAV:', err); // Fallback to original blob if conversion fails - onRecordingComplete?.(webmBlob, recordedDuration); + onRecordingComplete?.(webmBlob, recordedDuration, context); } }; diff --git a/app/src/lib/hooks/useCaptureRecordingSession.ts b/app/src/lib/hooks/useCaptureRecordingSession.ts index 42457030..51141e97 100644 --- a/app/src/lib/hooks/useCaptureRecordingSession.ts +++ b/app/src/lib/hooks/useCaptureRecordingSession.ts @@ -60,8 +60,9 @@ export interface UseCaptureRecordingSessionOptions { /** * Fired after a capture row is created on the server. Callers can use this * to select the new capture or emit a Tauri event to a sibling window. + * ``context`` is whatever was passed to ``startRecording`` for this take. */ - onCaptureCreated?: (capture: CaptureResponse) => void; + onCaptureCreated?: (capture: CaptureResponse, context?: unknown) => void; /** * Fired with the final delivered text — refined if ``auto_refine`` was on * for this capture, raw transcript otherwise. Used by the floating @@ -69,12 +70,14 @@ export interface UseCaptureRecordingSessionOptions { * * ``allowAutoPaste`` snapshots the setting at chord-start so a refine that * lands after the user flips the toggle still uses the value the capture - * was created under. + * was created under. ``context`` is the value passed to ``startRecording`` + * for this take, so overlapping dictations can't cross their targets. */ onFinalText?: ( text: string, capture: CaptureResponse, allowAutoPaste: boolean, + context?: unknown, ) => void; } @@ -85,7 +88,7 @@ export interface UseCaptureRecordingSessionResult { isRecording: boolean; isUploading: boolean; isRefining: boolean; - startRecording: () => void; + startRecording: (context?: unknown) => void; stopRecording: () => void; toggleRecording: () => void; dismissError: () => void; @@ -128,10 +131,13 @@ export function useCaptureRecordingSession( const onFinalTextRef = useRef(options.onFinalText); onFinalTextRef.current = options.onFinalText; - // Snapshot of ``allow_auto_paste`` from the capture-create response — - // held so the refine onSuccess (which only sees the plain CaptureResponse) - // can still pass the original setting through to onFinalText. - const allowAutoPasteRef = useRef(true); + // Per-capture recording context and its ``allow_auto_paste`` snapshot, keyed + // by capture id so a refine that resolves after another dictation started + // still delivers to the right target with the setting the capture was created + // under. Populated on capture-create and consumed once the final text lands. + const captureDeliveryRef = useRef>( + new Map(), + ); const clearRestTimer = useCallback(() => { if (restTimerRef.current !== null) { @@ -197,20 +203,34 @@ export function useCaptureRecordingSession( queryClient.invalidateQueries({ queryKey: ['captures'] }); broadcastUpdated(captureId); if (pillStateRef.current === 'refining') scheduleHidePill(); + const delivery = captureDeliveryRef.current.get(captureId); + captureDeliveryRef.current.delete(captureId); const finalText = data.transcript_refined ?? data.transcript_raw; if (finalText) { - onFinalTextRef.current?.(finalText, data, allowAutoPasteRef.current); + onFinalTextRef.current?.( + finalText, + data, + delivery?.allowAutoPaste ?? true, + delivery?.context, + ); } }, - onError: (err: Error) => { + onError: (err: Error, captureId) => { + captureDeliveryRef.current.delete(captureId); showError(err.message || 'Refinement failed'); }, }); const uploadMutation = useMutation({ - mutationFn: async ({ file, source }: { file: File; source: CaptureSource }) => - apiClient.createCapture(file, { source }), - onSuccess: (capture) => { + mutationFn: async ({ + file, + source, + }: { + file: File; + source: CaptureSource; + context?: unknown; + }) => apiClient.createCapture(file, { source }), + onSuccess: (capture, { context }) => { queryClient.setQueryData(['captures'], (prev) => { if (!prev) return prev; if (prev.items.some((c) => c.id === capture.id)) return prev; @@ -218,9 +238,12 @@ export function useCaptureRecordingSession( }); queryClient.invalidateQueries({ queryKey: ['captures'] }); broadcastCreated(capture); - onCaptureCreatedRef.current?.(capture); - allowAutoPasteRef.current = capture.allow_auto_paste; + onCaptureCreatedRef.current?.(capture, context); if (capture.auto_refine) { + captureDeliveryRef.current.set(capture.id, { + context, + allowAutoPaste: capture.allow_auto_paste, + }); setPillState('refining'); refineMutation.mutate(capture.id); } else { @@ -230,6 +253,7 @@ export function useCaptureRecordingSession( capture.transcript_raw, capture, capture.allow_auto_paste, + context, ); } } @@ -258,7 +282,7 @@ export function useCaptureRecordingSession( releaseWarm, } = useAudioRecording({ keepWarm: options.keepMicWarm ?? false, - onRecordingComplete: (blob, recordedDuration) => { + onRecordingComplete: (blob, recordedDuration, context) => { // Trigger-happy tap — MediaRecorder hasn't emitted a usable chunk yet // so the blob is empty or unparseable. Surface it as a transient pill // so the user sees their recording was recognised and canceled. @@ -276,7 +300,7 @@ export function useCaptureRecordingSession( const file = new File([blob], `dictation-${Date.now()}.${extension}`, { type: blob.type, }); - uploadMutation.mutate({ file, source: 'dictation' }); + uploadMutation.mutate({ file, source: 'dictation', context }); }, }); @@ -286,13 +310,16 @@ export function useCaptureRecordingSession( } }, [recordError, showError]); - const startRecording = useCallback(() => { - if (isRecording) return; - clearRestTimer(); - setFrozenElapsedMs(0); - setPillState('recording'); - beginAudioRecording(); - }, [isRecording, beginAudioRecording, clearRestTimer]); + const startRecording = useCallback( + (context?: unknown) => { + if (isRecording) return; + clearRestTimer(); + setFrozenElapsedMs(0); + setPillState('recording'); + beginAudioRecording(context); + }, + [isRecording, beginAudioRecording, clearRestTimer], + ); const toggleRecording = useCallback(() => { if (isRecording) { diff --git a/tauri/src-tauri/src/focus_capture.rs b/tauri/src-tauri/src/focus_capture.rs index e64b1761..38774087 100644 --- a/tauri/src-tauri/src/focus_capture.rs +++ b/tauri/src-tauri/src/focus_capture.rs @@ -198,6 +198,17 @@ pub fn capture_focus() -> Result { &mut focused as *mut _, ); if err != AX_ERROR_SUCCESS || focused.is_null() { + // Some apps (terminals, some Electron windows) expose no + // system-wide focused element. Rather than drop the dictation, + // fall back to the frontmost app so the transcript still injects + // there via activate + ⌘V. + if let Some(fp) = frontmost_pid() { + return Ok(FocusSnapshot { + pid: fp, + bundle_id: bundle_id_for_pid(fp), + role: None, + }); + } return Err(format!( "No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.", err @@ -213,6 +224,26 @@ pub fn capture_focus() -> Result { return Err(format!("AXUIElementGetPid failed (AXError {})", err)); } + // If the focused element belongs to our OWN process, the dictate pill + // has transiently taken key focus for its WebKit mic capture — the + // system-wide AXFocusedUIElement then resolves to the pill instead of + // the user's real target, which would make us paste into ourselves (a + // no-op) or drop the text entirely. The pill is a non-activating panel + // so it never becomes the frontmost application; remap to the + // frontmost app, which is always the real dictation target. + let our_pid = std::process::id() as Pid; + if pid == our_pid { + if let Some(fp) = frontmost_pid() { + if fp != our_pid { + return Ok(FocusSnapshot { + pid: fp, + bundle_id: bundle_id_for_pid(fp), + role: None, + }); + } + } + } + let role = { let role_attr = cf_string_const("AXRole"); match role_attr { @@ -301,6 +332,28 @@ pub fn activate_pid(pid: i32) -> Result<(), String> { } } +/// PID of the app the user is currently in (NSWorkspace frontmost). Used to +/// skip re-activation when the paste target never lost frontmost status — +/// activating an already-frontmost app is a no-op at best, and on +/// fullscreen Spaces macOS 26's cooperative activation returns NO for it, +/// which previously aborted the whole paste. +#[cfg(target_os = "macos")] +pub fn frontmost_pid() -> Option { + unsafe { + let _pool = AutoreleasePool::new(); + let ws: Id = msg_send![class!(NSWorkspace), sharedWorkspace]; + if ws.is_null() { + return None; + } + let app: Id = msg_send![ws, frontmostApplication]; + if app.is_null() { + return None; + } + let pid: i32 = msg_send![app, processIdentifier]; + Some(pid) + } +} + /// `true` when `NSRunningApplication` responds to /// `yieldActivationToApplication:` — the macOS 14+ discriminator for the /// cooperative-activation APIs. Cached since the answer doesn't change diff --git a/tauri/src-tauri/src/hotkey_monitor.rs b/tauri/src-tauri/src/hotkey_monitor.rs index c2c01e98..280a0462 100644 --- a/tauri/src-tauri/src/hotkey_monitor.rs +++ b/tauri/src-tauri/src/hotkey_monitor.rs @@ -269,6 +269,10 @@ fn apply_effect(app: &AppHandle, effect: Effect) { // it out of whatever app the user was typing in, which is // the opposite of what a dictation overlay should do. let _ = window.show(); + // Order the pill into the currently-active Space (incl. a + // foreign app's fullscreen Space) — see main.rs. + #[cfg(target_os = "macos")] + crate::force_order_front(&window); let payload = serde_json::json!({ "focus": focus }); let _ = window.emit("dictate:start", payload); } diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index cd5f405b..a17d217a 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -57,9 +57,141 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result *const objc::runtime::Class; +} + +/// `canBecomeKeyWindow` override for the pill panel. Borderless NSPanels +/// refuse key status by default, and WebKit rejects `getUserMedia` from a +/// document whose window can never become key — so recording silently fails. +/// Returning YES restores capture; the panel stays non-activating, so showing +/// it never steals focus from the app being dictated into. +#[cfg(target_os = "macos")] +extern "C" fn pill_panel_can_become_key( + _this: &objc::runtime::Object, + _sel: objc::runtime::Sel, +) -> objc::runtime::BOOL { + objc::runtime::YES +} + +/// Lazily-registered NSPanel subclass for the dictate pill: key-capable (for +/// WebKit media capture) while remaining a panel (for fullscreen-Space join). +#[cfg(target_os = "macos")] +fn pill_panel_class() -> &'static objc::runtime::Class { + use objc::declare::ClassDecl; + use objc::runtime::{Class, Object, Sel, BOOL}; + use objc::{class, sel, sel_impl}; + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + let superclass = class!(NSPanel); + let mut decl = + ClassDecl::new("VoiceboxPillPanel", superclass).expect("register VoiceboxPillPanel"); + unsafe { + decl.add_method( + sel!(canBecomeKeyWindow), + pill_panel_can_become_key as extern "C" fn(&Object, Sel) -> BOOL, + ); + } + decl.register(); + }); + Class::get("VoiceboxPillPanel").expect("VoiceboxPillPanel registered") +} + +/// Convert the dictate pill's NSWindow into a key-capable NSPanel and set the +/// collection behavior + window level required to appear over another app's +/// native macOS fullscreen Space. +/// +/// A regular (Dock-icon) app's plain NSWindow is never admitted to a foreign +/// fullscreen Space regardless of collection-behavior flags or window level; +/// an NSPanel with the same flags is. NSPanel adds no instance variables over +/// NSWindow, so re-classing the live object is safe (the tauri-nspanel plugin +/// uses the same technique). Idempotent via an `isKindOfClass` guard. Runs on +/// the main thread because AppKit window mutation is main-thread-only. +#[cfg(target_os = "macos")] +pub fn apply_fullscreen_overlay_behavior(window: &tauri::WebviewWindow) { + let w = window.clone(); + let dispatched = window.run_on_main_thread(move || { + use objc::runtime::{Object, NO, YES}; + use objc::{class, msg_send, sel, sel_impl}; + + // NSWindowCollectionBehavior bit flags. + const CAN_JOIN_ALL_SPACES: u64 = 1 << 0; + const STATIONARY: u64 = 1 << 4; + const FULL_SCREEN_AUXILIARY: u64 = 1 << 8; // the flag stock Tauri never sets + const NONACTIVATING_PANEL: u64 = 1 << 7; // NSWindowStyleMaskNonactivatingPanel + // NSScreenSaverWindowLevel — floats above fullscreen app content. + const OVERLAY_WINDOW_LEVEL: i64 = 1000; + + let ns_window = match w.ns_window() { + Ok(ptr) => ptr as *mut Object, + Err(e) => { + eprintln!("apply_fullscreen_overlay_behavior: ns_window() failed: {e}"); + return; + } + }; + if ns_window.is_null() { + return; + } + // SAFETY: ns_window is a valid, non-null NSWindow* owned by Tauri for + // the lifetime of the webview window; all selectors are standard AppKit + // calls and we are on the main thread. + unsafe { + let is_panel: objc::runtime::BOOL = + msg_send![ns_window, isKindOfClass: class!(NSPanel)]; + if is_panel == NO { + object_setClass(ns_window, pill_panel_class()); + let style: u64 = msg_send![ns_window, styleMask]; + let _: () = msg_send![ns_window, setStyleMask: style | NONACTIVATING_PANEL]; + let _: () = msg_send![ns_window, setHidesOnDeactivate: NO]; + let _: () = msg_send![ns_window, setBecomesKeyOnlyIfNeeded: YES]; + let _: () = msg_send![ns_window, setFloatingPanel: YES]; + } + // Preserve behavior bits installed by Tauri/Tao instead of + // replacing them wholesale when adding the fullscreen flags. + let current_behavior: u64 = msg_send![ns_window, collectionBehavior]; + let behavior = + current_behavior | CAN_JOIN_ALL_SPACES | FULL_SCREEN_AUXILIARY | STATIONARY; + let _: () = msg_send![ns_window, setCollectionBehavior: behavior]; + let _: () = msg_send![ns_window, setLevel: OVERLAY_WINDOW_LEVEL]; + } + }); + if let Err(e) = dispatched { + eprintln!("apply_fullscreen_overlay_behavior: main-thread dispatch failed: {e}"); + } +} + +/// Order the pill front over whatever Space is active. `orderFrontRegardless` +/// works even though the app is inactive (it always is mid-dictation — the +/// user is typing in some other app). Called right after `window.show()`. +#[cfg(target_os = "macos")] +pub fn force_order_front(window: &tauri::WebviewWindow) { + let w = window.clone(); + let _ = window.run_on_main_thread(move || { + use objc::runtime::Object; + use objc::{msg_send, sel, sel_impl}; + if let Ok(ptr) = w.ns_window() { + let ns_window = ptr as *mut Object; + if !ns_window.is_null() { + unsafe { + let _: () = msg_send![ns_window, orderFrontRegardless]; + } + } + } + }); +} + /// Position, undo click-through, and show the dictate pill window. /// /// The hide path parks the window at (-10_000, -10_000) and toggles @@ -114,6 +246,8 @@ pub fn show_dictate_window(app: &tauri::AppHandle) { } let _ = window.set_ignore_cursor_events(false); let _ = window.show(); + #[cfg(target_os = "macos")] + force_order_front(&window); } const LEGACY_PORT: u16 = 8000; @@ -1210,6 +1344,7 @@ fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> { /// Returns `true` when the paste sequence completed end-to-end. #[command] async fn paste_final_text( + app: tauri::AppHandle, text: String, focus: focus_capture::FocusSnapshot, ) -> Result { @@ -1223,15 +1358,56 @@ async fn paste_final_text( ); } - focus_capture::activate_pid(focus.pid)?; - tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await; + // Only re-activate the target when the user actually left it. When it is + // still frontmost (the common case — the dictate pill is non-activating, + // and in a fullscreen Space the target never loses frontmost), activation + // is a no-op; on macOS 26 fullscreen Spaces `activate` returns NO for an + // already-frontmost app, which would otherwise abort the paste entirely. + #[cfg(target_os = "macos")] + let already_front = focus_capture::frontmost_pid() == Some(focus.pid); + #[cfg(not(target_os = "macos"))] + let already_front = false; + + if !already_front { + focus_capture::activate_pid(focus.pid)?; + tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await; + } let snapshot = clipboard::save_clipboard()?; let after_write = clipboard::write_text(&text)?; + // Order the pill out for the synthetic ⌘V. The pill is a key-capable + // panel (WebKit needs that for getUserMedia), and over a fullscreen Space + // it holds key focus Spotlight-style — the keystroke would land in the + // pill instead of the target app. Hidden it can't swallow keys; restored + // immediately after so the webview never suspends between dictations. + #[cfg(target_os = "macos")] + let pill = app.get_webview_window(DICTATE_WINDOW_LABEL); + #[cfg(target_os = "macos")] + if let Some(ref w) = pill { + if let Err(e) = w.hide() { + // Never emit Cmd+V while the key-capable pill may still own focus. + // Undo our clipboard write when it is still safe, then abort. + if matches!( + clipboard::current_change_count(), + Ok(current) if current == after_write + ) { + clipboard::restore_clipboard(&snapshot)?; + } + return Err(format!("Failed to hide dictate window before paste: {e}")); + } + tokio::time::sleep(std::time::Duration::from_millis(80)).await; + } + let paste_result = synthetic_keys::send_paste(); tokio::time::sleep(std::time::Duration::from_millis(PASTE_CONSUME_MS)).await; + #[cfg(target_os = "macos")] + if let Some(ref w) = pill { + let _ = w.show(); + force_order_front(w); + } + let safe_to_restore = matches!( clipboard::current_change_count(), Ok(current) if current == after_write