mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-29 15:15:27 -07:00
fix(dictation): preserve native focus and fullscreen injection
Carry focus and auto-paste permission per capture, support dictation over macOS fullscreen Spaces, preserve native window behavior flags, and abort paste when the pill cannot be hidden safely.\n\nVerified: frontend CI; cargo check; diff and security scans. Packaged multi-Space validation remains a release gate.
This commit is contained in:
@@ -39,20 +39,13 @@ export function DictateWindow() {
|
|||||||
// user opted into keeping the microphone ready.
|
// user opted into keeping the microphone ready.
|
||||||
const [micWarm, setMicWarm] = useState(false);
|
const [micWarm, setMicWarm] = useState(false);
|
||||||
|
|
||||||
// Snapshot of the focused UI element at chord-start, shipped over from
|
|
||||||
// Rust on the ``dictate:start`` payload. Held in a ref so it survives
|
|
||||||
// the 1–2 s transcribe + refine window — the paste only fires once the
|
|
||||||
// final text comes back.
|
|
||||||
const focusRef = useRef<FocusSnapshot | null>(null);
|
|
||||||
|
|
||||||
const session = useCaptureRecordingSession({
|
const session = useCaptureRecordingSession({
|
||||||
keepMicWarm: micWarm,
|
keepMicWarm: micWarm,
|
||||||
onFinalText: async (text, _capture, allowAutoPaste) => {
|
onFinalText: async (text, _capture, allowAutoPaste, context) => {
|
||||||
const focus = focusRef.current;
|
// Focus is the snapshot taken at chord-start and threaded through as this
|
||||||
// Consume-once: a second chord before this fires would overwrite
|
// take's context, so it survives the 1–2 s transcribe + refine window and
|
||||||
// focusRef, but nulling it here guards against the late-arriving
|
// overlapping dictations can't paste into each other's target.
|
||||||
// refine-result firing a paste after the user has moved on.
|
const focus = context as FocusSnapshot | null;
|
||||||
focusRef.current = null;
|
|
||||||
if (!allowAutoPaste) return;
|
if (!allowAutoPaste) return;
|
||||||
if (!focus || !text.trim()) return;
|
if (!focus || !text.trim()) return;
|
||||||
try {
|
try {
|
||||||
@@ -81,8 +74,7 @@ export function DictateWindow() {
|
|||||||
const unlistens: UnlistenFn[] = [];
|
const unlistens: UnlistenFn[] = [];
|
||||||
const registrations = [
|
const registrations = [
|
||||||
listen<{ focus: FocusSnapshot | null }>('dictate:start', (event) => {
|
listen<{ focus: FocusSnapshot | null }>('dictate:start', (event) => {
|
||||||
focusRef.current = event.payload?.focus ?? null;
|
sessionRef.current.startRecording(event.payload?.focus ?? null);
|
||||||
sessionRef.current.startRecording();
|
|
||||||
}),
|
}),
|
||||||
listen('dictate:stop', () => {
|
listen('dictate:stop', () => {
|
||||||
// Forward stops that arrive while getUserMedia is still resolving.
|
// Forward stops that arrive while getUserMedia is still resolving.
|
||||||
|
|||||||
@@ -4,7 +4,11 @@ import { convertToWav } from '@/lib/utils/audio';
|
|||||||
|
|
||||||
interface UseAudioRecordingOptions {
|
interface UseAudioRecordingOptions {
|
||||||
maxDurationSeconds?: number;
|
maxDurationSeconds?: number;
|
||||||
onRecordingComplete?: (blob: Blob, duration?: number) => void;
|
// ``context`` is whatever was handed to ``startRecording`` for this take,
|
||||||
|
// threaded back untouched so callers can correlate the result with the
|
||||||
|
// recording it came from (the dictate window pairs it with the focus
|
||||||
|
// snapshot captured at chord-start).
|
||||||
|
onRecordingComplete?: (blob: Blob, duration?: number, context?: unknown) => void;
|
||||||
/**
|
/**
|
||||||
* Keep the microphone ``MediaStream`` open between recordings instead of
|
* Keep the microphone ``MediaStream`` open between recordings instead of
|
||||||
* tearing it down on every stop. This is what removes the "first words get
|
* tearing it down on every stop. This is what removes the "first words get
|
||||||
@@ -184,7 +188,7 @@ export function useAudioRecording({
|
|||||||
}, [keepWarm, acquireStream]);
|
}, [keepWarm, acquireStream]);
|
||||||
|
|
||||||
const startRecording = useCallback(
|
const startRecording = useCallback(
|
||||||
async () => {
|
async (context?: unknown) => {
|
||||||
// A second chord can arrive while the first one is still waiting on
|
// A second chord can arrive while the first one is still waiting on
|
||||||
// getUserMedia. Never create overlapping MediaRecorders on the same
|
// getUserMedia. Never create overlapping MediaRecorders on the same
|
||||||
// coalesced stream; the original take will honor any deferred stop.
|
// coalesced stream; the original take will honor any deferred stop.
|
||||||
@@ -275,11 +279,11 @@ export function useAudioRecording({
|
|||||||
// Convert to WAV format to avoid needing ffmpeg on backend
|
// Convert to WAV format to avoid needing ffmpeg on backend
|
||||||
try {
|
try {
|
||||||
const wavBlob = await convertToWav(webmBlob);
|
const wavBlob = await convertToWav(webmBlob);
|
||||||
onRecordingComplete?.(wavBlob, recordedDuration);
|
onRecordingComplete?.(wavBlob, recordedDuration, context);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error converting audio to WAV:', err);
|
console.error('Error converting audio to WAV:', err);
|
||||||
// Fallback to original blob if conversion fails
|
// Fallback to original blob if conversion fails
|
||||||
onRecordingComplete?.(webmBlob, recordedDuration);
|
onRecordingComplete?.(webmBlob, recordedDuration, context);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -60,8 +60,9 @@ export interface UseCaptureRecordingSessionOptions {
|
|||||||
/**
|
/**
|
||||||
* Fired after a capture row is created on the server. Callers can use this
|
* Fired after a capture row is created on the server. Callers can use this
|
||||||
* to select the new capture or emit a Tauri event to a sibling window.
|
* to select the new capture or emit a Tauri event to a sibling window.
|
||||||
|
* ``context`` is whatever was passed to ``startRecording`` for this take.
|
||||||
*/
|
*/
|
||||||
onCaptureCreated?: (capture: CaptureResponse) => void;
|
onCaptureCreated?: (capture: CaptureResponse, context?: unknown) => void;
|
||||||
/**
|
/**
|
||||||
* Fired with the final delivered text — refined if ``auto_refine`` was on
|
* Fired with the final delivered text — refined if ``auto_refine`` was on
|
||||||
* for this capture, raw transcript otherwise. Used by the floating
|
* for this capture, raw transcript otherwise. Used by the floating
|
||||||
@@ -69,12 +70,14 @@ export interface UseCaptureRecordingSessionOptions {
|
|||||||
*
|
*
|
||||||
* ``allowAutoPaste`` snapshots the setting at chord-start so a refine that
|
* ``allowAutoPaste`` snapshots the setting at chord-start so a refine that
|
||||||
* lands after the user flips the toggle still uses the value the capture
|
* lands after the user flips the toggle still uses the value the capture
|
||||||
* was created under.
|
* was created under. ``context`` is the value passed to ``startRecording``
|
||||||
|
* for this take, so overlapping dictations can't cross their targets.
|
||||||
*/
|
*/
|
||||||
onFinalText?: (
|
onFinalText?: (
|
||||||
text: string,
|
text: string,
|
||||||
capture: CaptureResponse,
|
capture: CaptureResponse,
|
||||||
allowAutoPaste: boolean,
|
allowAutoPaste: boolean,
|
||||||
|
context?: unknown,
|
||||||
) => void;
|
) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -85,7 +88,7 @@ export interface UseCaptureRecordingSessionResult {
|
|||||||
isRecording: boolean;
|
isRecording: boolean;
|
||||||
isUploading: boolean;
|
isUploading: boolean;
|
||||||
isRefining: boolean;
|
isRefining: boolean;
|
||||||
startRecording: () => void;
|
startRecording: (context?: unknown) => void;
|
||||||
stopRecording: () => void;
|
stopRecording: () => void;
|
||||||
toggleRecording: () => void;
|
toggleRecording: () => void;
|
||||||
dismissError: () => void;
|
dismissError: () => void;
|
||||||
@@ -128,10 +131,13 @@ export function useCaptureRecordingSession(
|
|||||||
const onFinalTextRef = useRef(options.onFinalText);
|
const onFinalTextRef = useRef(options.onFinalText);
|
||||||
onFinalTextRef.current = options.onFinalText;
|
onFinalTextRef.current = options.onFinalText;
|
||||||
|
|
||||||
// Snapshot of ``allow_auto_paste`` from the capture-create response —
|
// Per-capture recording context and its ``allow_auto_paste`` snapshot, keyed
|
||||||
// held so the refine onSuccess (which only sees the plain CaptureResponse)
|
// by capture id so a refine that resolves after another dictation started
|
||||||
// can still pass the original setting through to onFinalText.
|
// still delivers to the right target with the setting the capture was created
|
||||||
const allowAutoPasteRef = useRef<boolean>(true);
|
// under. Populated on capture-create and consumed once the final text lands.
|
||||||
|
const captureDeliveryRef = useRef<Map<string, { context: unknown; allowAutoPaste: boolean }>>(
|
||||||
|
new Map(),
|
||||||
|
);
|
||||||
|
|
||||||
const clearRestTimer = useCallback(() => {
|
const clearRestTimer = useCallback(() => {
|
||||||
if (restTimerRef.current !== null) {
|
if (restTimerRef.current !== null) {
|
||||||
@@ -197,20 +203,34 @@ export function useCaptureRecordingSession(
|
|||||||
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
||||||
broadcastUpdated(captureId);
|
broadcastUpdated(captureId);
|
||||||
if (pillStateRef.current === 'refining') scheduleHidePill();
|
if (pillStateRef.current === 'refining') scheduleHidePill();
|
||||||
|
const delivery = captureDeliveryRef.current.get(captureId);
|
||||||
|
captureDeliveryRef.current.delete(captureId);
|
||||||
const finalText = data.transcript_refined ?? data.transcript_raw;
|
const finalText = data.transcript_refined ?? data.transcript_raw;
|
||||||
if (finalText) {
|
if (finalText) {
|
||||||
onFinalTextRef.current?.(finalText, data, allowAutoPasteRef.current);
|
onFinalTextRef.current?.(
|
||||||
|
finalText,
|
||||||
|
data,
|
||||||
|
delivery?.allowAutoPaste ?? true,
|
||||||
|
delivery?.context,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onError: (err: Error) => {
|
onError: (err: Error, captureId) => {
|
||||||
|
captureDeliveryRef.current.delete(captureId);
|
||||||
showError(err.message || 'Refinement failed');
|
showError(err.message || 'Refinement failed');
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const uploadMutation = useMutation({
|
const uploadMutation = useMutation({
|
||||||
mutationFn: async ({ file, source }: { file: File; source: CaptureSource }) =>
|
mutationFn: async ({
|
||||||
apiClient.createCapture(file, { source }),
|
file,
|
||||||
onSuccess: (capture) => {
|
source,
|
||||||
|
}: {
|
||||||
|
file: File;
|
||||||
|
source: CaptureSource;
|
||||||
|
context?: unknown;
|
||||||
|
}) => apiClient.createCapture(file, { source }),
|
||||||
|
onSuccess: (capture, { context }) => {
|
||||||
queryClient.setQueryData<CaptureListResponse>(['captures'], (prev) => {
|
queryClient.setQueryData<CaptureListResponse>(['captures'], (prev) => {
|
||||||
if (!prev) return prev;
|
if (!prev) return prev;
|
||||||
if (prev.items.some((c) => c.id === capture.id)) return prev;
|
if (prev.items.some((c) => c.id === capture.id)) return prev;
|
||||||
@@ -218,9 +238,12 @@ export function useCaptureRecordingSession(
|
|||||||
});
|
});
|
||||||
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
||||||
broadcastCreated(capture);
|
broadcastCreated(capture);
|
||||||
onCaptureCreatedRef.current?.(capture);
|
onCaptureCreatedRef.current?.(capture, context);
|
||||||
allowAutoPasteRef.current = capture.allow_auto_paste;
|
|
||||||
if (capture.auto_refine) {
|
if (capture.auto_refine) {
|
||||||
|
captureDeliveryRef.current.set(capture.id, {
|
||||||
|
context,
|
||||||
|
allowAutoPaste: capture.allow_auto_paste,
|
||||||
|
});
|
||||||
setPillState('refining');
|
setPillState('refining');
|
||||||
refineMutation.mutate(capture.id);
|
refineMutation.mutate(capture.id);
|
||||||
} else {
|
} else {
|
||||||
@@ -230,6 +253,7 @@ export function useCaptureRecordingSession(
|
|||||||
capture.transcript_raw,
|
capture.transcript_raw,
|
||||||
capture,
|
capture,
|
||||||
capture.allow_auto_paste,
|
capture.allow_auto_paste,
|
||||||
|
context,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -258,7 +282,7 @@ export function useCaptureRecordingSession(
|
|||||||
releaseWarm,
|
releaseWarm,
|
||||||
} = useAudioRecording({
|
} = useAudioRecording({
|
||||||
keepWarm: options.keepMicWarm ?? false,
|
keepWarm: options.keepMicWarm ?? false,
|
||||||
onRecordingComplete: (blob, recordedDuration) => {
|
onRecordingComplete: (blob, recordedDuration, context) => {
|
||||||
// Trigger-happy tap — MediaRecorder hasn't emitted a usable chunk yet
|
// Trigger-happy tap — MediaRecorder hasn't emitted a usable chunk yet
|
||||||
// so the blob is empty or unparseable. Surface it as a transient pill
|
// so the blob is empty or unparseable. Surface it as a transient pill
|
||||||
// so the user sees their recording was recognised and canceled.
|
// so the user sees their recording was recognised and canceled.
|
||||||
@@ -276,7 +300,7 @@ export function useCaptureRecordingSession(
|
|||||||
const file = new File([blob], `dictation-${Date.now()}.${extension}`, {
|
const file = new File([blob], `dictation-${Date.now()}.${extension}`, {
|
||||||
type: blob.type,
|
type: blob.type,
|
||||||
});
|
});
|
||||||
uploadMutation.mutate({ file, source: 'dictation' });
|
uploadMutation.mutate({ file, source: 'dictation', context });
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -286,13 +310,16 @@ export function useCaptureRecordingSession(
|
|||||||
}
|
}
|
||||||
}, [recordError, showError]);
|
}, [recordError, showError]);
|
||||||
|
|
||||||
const startRecording = useCallback(() => {
|
const startRecording = useCallback(
|
||||||
if (isRecording) return;
|
(context?: unknown) => {
|
||||||
clearRestTimer();
|
if (isRecording) return;
|
||||||
setFrozenElapsedMs(0);
|
clearRestTimer();
|
||||||
setPillState('recording');
|
setFrozenElapsedMs(0);
|
||||||
beginAudioRecording();
|
setPillState('recording');
|
||||||
}, [isRecording, beginAudioRecording, clearRestTimer]);
|
beginAudioRecording(context);
|
||||||
|
},
|
||||||
|
[isRecording, beginAudioRecording, clearRestTimer],
|
||||||
|
);
|
||||||
|
|
||||||
const toggleRecording = useCallback(() => {
|
const toggleRecording = useCallback(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
|
|||||||
@@ -198,6 +198,17 @@ pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
|||||||
&mut focused as *mut _,
|
&mut focused as *mut _,
|
||||||
);
|
);
|
||||||
if err != AX_ERROR_SUCCESS || focused.is_null() {
|
if err != AX_ERROR_SUCCESS || focused.is_null() {
|
||||||
|
// Some apps (terminals, some Electron windows) expose no
|
||||||
|
// system-wide focused element. Rather than drop the dictation,
|
||||||
|
// fall back to the frontmost app so the transcript still injects
|
||||||
|
// there via activate + ⌘V.
|
||||||
|
if let Some(fp) = frontmost_pid() {
|
||||||
|
return Ok(FocusSnapshot {
|
||||||
|
pid: fp,
|
||||||
|
bundle_id: bundle_id_for_pid(fp),
|
||||||
|
role: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
|
"No focused element (AXError {}). Verify Accessibility permission is granted and a focused text field exists.",
|
||||||
err
|
err
|
||||||
@@ -213,6 +224,26 @@ pub fn capture_focus() -> Result<FocusSnapshot, String> {
|
|||||||
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
|
return Err(format!("AXUIElementGetPid failed (AXError {})", err));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If the focused element belongs to our OWN process, the dictate pill
|
||||||
|
// has transiently taken key focus for its WebKit mic capture — the
|
||||||
|
// system-wide AXFocusedUIElement then resolves to the pill instead of
|
||||||
|
// the user's real target, which would make us paste into ourselves (a
|
||||||
|
// no-op) or drop the text entirely. The pill is a non-activating panel
|
||||||
|
// so it never becomes the frontmost application; remap to the
|
||||||
|
// frontmost app, which is always the real dictation target.
|
||||||
|
let our_pid = std::process::id() as Pid;
|
||||||
|
if pid == our_pid {
|
||||||
|
if let Some(fp) = frontmost_pid() {
|
||||||
|
if fp != our_pid {
|
||||||
|
return Ok(FocusSnapshot {
|
||||||
|
pid: fp,
|
||||||
|
bundle_id: bundle_id_for_pid(fp),
|
||||||
|
role: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let role = {
|
let role = {
|
||||||
let role_attr = cf_string_const("AXRole");
|
let role_attr = cf_string_const("AXRole");
|
||||||
match role_attr {
|
match role_attr {
|
||||||
@@ -301,6 +332,28 @@ pub fn activate_pid(pid: i32) -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// PID of the app the user is currently in (NSWorkspace frontmost). Used to
|
||||||
|
/// skip re-activation when the paste target never lost frontmost status —
|
||||||
|
/// activating an already-frontmost app is a no-op at best, and on
|
||||||
|
/// fullscreen Spaces macOS 26's cooperative activation returns NO for it,
|
||||||
|
/// which previously aborted the whole paste.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn frontmost_pid() -> Option<i32> {
|
||||||
|
unsafe {
|
||||||
|
let _pool = AutoreleasePool::new();
|
||||||
|
let ws: Id = msg_send![class!(NSWorkspace), sharedWorkspace];
|
||||||
|
if ws.is_null() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let app: Id = msg_send![ws, frontmostApplication];
|
||||||
|
if app.is_null() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let pid: i32 = msg_send![app, processIdentifier];
|
||||||
|
Some(pid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// `true` when `NSRunningApplication` responds to
|
/// `true` when `NSRunningApplication` responds to
|
||||||
/// `yieldActivationToApplication:` — the macOS 14+ discriminator for the
|
/// `yieldActivationToApplication:` — the macOS 14+ discriminator for the
|
||||||
/// cooperative-activation APIs. Cached since the answer doesn't change
|
/// cooperative-activation APIs. Cached since the answer doesn't change
|
||||||
|
|||||||
@@ -269,6 +269,10 @@ fn apply_effect(app: &AppHandle, effect: Effect) {
|
|||||||
// it out of whatever app the user was typing in, which is
|
// it out of whatever app the user was typing in, which is
|
||||||
// the opposite of what a dictation overlay should do.
|
// the opposite of what a dictation overlay should do.
|
||||||
let _ = window.show();
|
let _ = window.show();
|
||||||
|
// Order the pill into the currently-active Space (incl. a
|
||||||
|
// foreign app's fullscreen Space) — see main.rs.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
crate::force_order_front(&window);
|
||||||
let payload = serde_json::json!({ "focus": focus });
|
let payload = serde_json::json!({ "focus": focus });
|
||||||
let _ = window.emit("dictate:start", payload);
|
let _ = window.emit("dictate:start", payload);
|
||||||
}
|
}
|
||||||
|
|||||||
+178
-2
@@ -57,9 +57,141 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewW
|
|||||||
window.set_position(PhysicalPosition::new(x, y))?;
|
window.set_position(PhysicalPosition::new(x, y))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Make the pill able to float over other apps' native fullscreen Spaces.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
apply_fullscreen_overlay_behavior(&window);
|
||||||
|
|
||||||
Ok(window)
|
Ok(window)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// `object_setClass` — reclass a live object. Not re-exported by `objc`.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
extern "C" {
|
||||||
|
fn object_setClass(
|
||||||
|
obj: *mut objc::runtime::Object,
|
||||||
|
cls: *const objc::runtime::Class,
|
||||||
|
) -> *const objc::runtime::Class;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `canBecomeKeyWindow` override for the pill panel. Borderless NSPanels
|
||||||
|
/// refuse key status by default, and WebKit rejects `getUserMedia` from a
|
||||||
|
/// document whose window can never become key — so recording silently fails.
|
||||||
|
/// Returning YES restores capture; the panel stays non-activating, so showing
|
||||||
|
/// it never steals focus from the app being dictated into.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
extern "C" fn pill_panel_can_become_key(
|
||||||
|
_this: &objc::runtime::Object,
|
||||||
|
_sel: objc::runtime::Sel,
|
||||||
|
) -> objc::runtime::BOOL {
|
||||||
|
objc::runtime::YES
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lazily-registered NSPanel subclass for the dictate pill: key-capable (for
|
||||||
|
/// WebKit media capture) while remaining a panel (for fullscreen-Space join).
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
fn pill_panel_class() -> &'static objc::runtime::Class {
|
||||||
|
use objc::declare::ClassDecl;
|
||||||
|
use objc::runtime::{Class, Object, Sel, BOOL};
|
||||||
|
use objc::{class, sel, sel_impl};
|
||||||
|
static INIT: std::sync::Once = std::sync::Once::new();
|
||||||
|
INIT.call_once(|| {
|
||||||
|
let superclass = class!(NSPanel);
|
||||||
|
let mut decl =
|
||||||
|
ClassDecl::new("VoiceboxPillPanel", superclass).expect("register VoiceboxPillPanel");
|
||||||
|
unsafe {
|
||||||
|
decl.add_method(
|
||||||
|
sel!(canBecomeKeyWindow),
|
||||||
|
pill_panel_can_become_key as extern "C" fn(&Object, Sel) -> BOOL,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
decl.register();
|
||||||
|
});
|
||||||
|
Class::get("VoiceboxPillPanel").expect("VoiceboxPillPanel registered")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert the dictate pill's NSWindow into a key-capable NSPanel and set the
|
||||||
|
/// collection behavior + window level required to appear over another app's
|
||||||
|
/// native macOS fullscreen Space.
|
||||||
|
///
|
||||||
|
/// A regular (Dock-icon) app's plain NSWindow is never admitted to a foreign
|
||||||
|
/// fullscreen Space regardless of collection-behavior flags or window level;
|
||||||
|
/// an NSPanel with the same flags is. NSPanel adds no instance variables over
|
||||||
|
/// NSWindow, so re-classing the live object is safe (the tauri-nspanel plugin
|
||||||
|
/// uses the same technique). Idempotent via an `isKindOfClass` guard. Runs on
|
||||||
|
/// the main thread because AppKit window mutation is main-thread-only.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn apply_fullscreen_overlay_behavior(window: &tauri::WebviewWindow) {
|
||||||
|
let w = window.clone();
|
||||||
|
let dispatched = window.run_on_main_thread(move || {
|
||||||
|
use objc::runtime::{Object, NO, YES};
|
||||||
|
use objc::{class, msg_send, sel, sel_impl};
|
||||||
|
|
||||||
|
// NSWindowCollectionBehavior bit flags.
|
||||||
|
const CAN_JOIN_ALL_SPACES: u64 = 1 << 0;
|
||||||
|
const STATIONARY: u64 = 1 << 4;
|
||||||
|
const FULL_SCREEN_AUXILIARY: u64 = 1 << 8; // the flag stock Tauri never sets
|
||||||
|
const NONACTIVATING_PANEL: u64 = 1 << 7; // NSWindowStyleMaskNonactivatingPanel
|
||||||
|
// NSScreenSaverWindowLevel — floats above fullscreen app content.
|
||||||
|
const OVERLAY_WINDOW_LEVEL: i64 = 1000;
|
||||||
|
|
||||||
|
let ns_window = match w.ns_window() {
|
||||||
|
Ok(ptr) => ptr as *mut Object,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("apply_fullscreen_overlay_behavior: ns_window() failed: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if ns_window.is_null() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// SAFETY: ns_window is a valid, non-null NSWindow* owned by Tauri for
|
||||||
|
// the lifetime of the webview window; all selectors are standard AppKit
|
||||||
|
// calls and we are on the main thread.
|
||||||
|
unsafe {
|
||||||
|
let is_panel: objc::runtime::BOOL =
|
||||||
|
msg_send![ns_window, isKindOfClass: class!(NSPanel)];
|
||||||
|
if is_panel == NO {
|
||||||
|
object_setClass(ns_window, pill_panel_class());
|
||||||
|
let style: u64 = msg_send![ns_window, styleMask];
|
||||||
|
let _: () = msg_send![ns_window, setStyleMask: style | NONACTIVATING_PANEL];
|
||||||
|
let _: () = msg_send![ns_window, setHidesOnDeactivate: NO];
|
||||||
|
let _: () = msg_send![ns_window, setBecomesKeyOnlyIfNeeded: YES];
|
||||||
|
let _: () = msg_send![ns_window, setFloatingPanel: YES];
|
||||||
|
}
|
||||||
|
// Preserve behavior bits installed by Tauri/Tao instead of
|
||||||
|
// replacing them wholesale when adding the fullscreen flags.
|
||||||
|
let current_behavior: u64 = msg_send![ns_window, collectionBehavior];
|
||||||
|
let behavior =
|
||||||
|
current_behavior | CAN_JOIN_ALL_SPACES | FULL_SCREEN_AUXILIARY | STATIONARY;
|
||||||
|
let _: () = msg_send![ns_window, setCollectionBehavior: behavior];
|
||||||
|
let _: () = msg_send![ns_window, setLevel: OVERLAY_WINDOW_LEVEL];
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if let Err(e) = dispatched {
|
||||||
|
eprintln!("apply_fullscreen_overlay_behavior: main-thread dispatch failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Order the pill front over whatever Space is active. `orderFrontRegardless`
|
||||||
|
/// works even though the app is inactive (it always is mid-dictation — the
|
||||||
|
/// user is typing in some other app). Called right after `window.show()`.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn force_order_front(window: &tauri::WebviewWindow) {
|
||||||
|
let w = window.clone();
|
||||||
|
let _ = window.run_on_main_thread(move || {
|
||||||
|
use objc::runtime::Object;
|
||||||
|
use objc::{msg_send, sel, sel_impl};
|
||||||
|
if let Ok(ptr) = w.ns_window() {
|
||||||
|
let ns_window = ptr as *mut Object;
|
||||||
|
if !ns_window.is_null() {
|
||||||
|
unsafe {
|
||||||
|
let _: () = msg_send![ns_window, orderFrontRegardless];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/// Position, undo click-through, and show the dictate pill window.
|
/// Position, undo click-through, and show the dictate pill window.
|
||||||
///
|
///
|
||||||
/// The hide path parks the window at (-10_000, -10_000) and toggles
|
/// The hide path parks the window at (-10_000, -10_000) and toggles
|
||||||
@@ -114,6 +246,8 @@ pub fn show_dictate_window(app: &tauri::AppHandle) {
|
|||||||
}
|
}
|
||||||
let _ = window.set_ignore_cursor_events(false);
|
let _ = window.set_ignore_cursor_events(false);
|
||||||
let _ = window.show();
|
let _ = window.show();
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
force_order_front(&window);
|
||||||
}
|
}
|
||||||
|
|
||||||
const LEGACY_PORT: u16 = 8000;
|
const LEGACY_PORT: u16 = 8000;
|
||||||
@@ -1210,6 +1344,7 @@ fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> {
|
|||||||
/// Returns `true` when the paste sequence completed end-to-end.
|
/// Returns `true` when the paste sequence completed end-to-end.
|
||||||
#[command]
|
#[command]
|
||||||
async fn paste_final_text(
|
async fn paste_final_text(
|
||||||
|
app: tauri::AppHandle,
|
||||||
text: String,
|
text: String,
|
||||||
focus: focus_capture::FocusSnapshot,
|
focus: focus_capture::FocusSnapshot,
|
||||||
) -> Result<bool, String> {
|
) -> Result<bool, String> {
|
||||||
@@ -1223,15 +1358,56 @@ async fn paste_final_text(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
focus_capture::activate_pid(focus.pid)?;
|
// Only re-activate the target when the user actually left it. When it is
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await;
|
// still frontmost (the common case — the dictate pill is non-activating,
|
||||||
|
// and in a fullscreen Space the target never loses frontmost), activation
|
||||||
|
// is a no-op; on macOS 26 fullscreen Spaces `activate` returns NO for an
|
||||||
|
// already-frontmost app, which would otherwise abort the paste entirely.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
let already_front = focus_capture::frontmost_pid() == Some(focus.pid);
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
let already_front = false;
|
||||||
|
|
||||||
|
if !already_front {
|
||||||
|
focus_capture::activate_pid(focus.pid)?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(POST_ACTIVATE_SETTLE_MS)).await;
|
||||||
|
}
|
||||||
|
|
||||||
let snapshot = clipboard::save_clipboard()?;
|
let snapshot = clipboard::save_clipboard()?;
|
||||||
let after_write = clipboard::write_text(&text)?;
|
let after_write = clipboard::write_text(&text)?;
|
||||||
|
|
||||||
|
// Order the pill out for the synthetic ⌘V. The pill is a key-capable
|
||||||
|
// panel (WebKit needs that for getUserMedia), and over a fullscreen Space
|
||||||
|
// it holds key focus Spotlight-style — the keystroke would land in the
|
||||||
|
// pill instead of the target app. Hidden it can't swallow keys; restored
|
||||||
|
// immediately after so the webview never suspends between dictations.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
let pill = app.get_webview_window(DICTATE_WINDOW_LABEL);
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
if let Some(ref w) = pill {
|
||||||
|
if let Err(e) = w.hide() {
|
||||||
|
// Never emit Cmd+V while the key-capable pill may still own focus.
|
||||||
|
// Undo our clipboard write when it is still safe, then abort.
|
||||||
|
if matches!(
|
||||||
|
clipboard::current_change_count(),
|
||||||
|
Ok(current) if current == after_write
|
||||||
|
) {
|
||||||
|
clipboard::restore_clipboard(&snapshot)?;
|
||||||
|
}
|
||||||
|
return Err(format!("Failed to hide dictate window before paste: {e}"));
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(80)).await;
|
||||||
|
}
|
||||||
|
|
||||||
let paste_result = synthetic_keys::send_paste();
|
let paste_result = synthetic_keys::send_paste();
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(PASTE_CONSUME_MS)).await;
|
tokio::time::sleep(std::time::Duration::from_millis(PASTE_CONSUME_MS)).await;
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
if let Some(ref w) = pill {
|
||||||
|
let _ = w.show();
|
||||||
|
force_order_front(w);
|
||||||
|
}
|
||||||
|
|
||||||
let safe_to_restore = matches!(
|
let safe_to_restore = matches!(
|
||||||
clipboard::current_change_count(),
|
clipboard::current_change_count(),
|
||||||
Ok(current) if current == after_write
|
Ok(current) if current == after_write
|
||||||
|
|||||||
Reference in New Issue
Block a user