mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 06:10:43 -07:00
fix(dictation): preserve native focus and fullscreen injection
Carry focus and auto-paste permission per capture, support dictation over macOS fullscreen Spaces, preserve native window behavior flags, and abort paste when the pill cannot be hidden safely.\n\nVerified: frontend CI; cargo check; diff and security scans. Packaged multi-Space validation remains a release gate.
This commit is contained in:
@@ -39,20 +39,13 @@ export function DictateWindow() {
|
||||
// user opted into keeping the microphone ready.
|
||||
const [micWarm, setMicWarm] = useState(false);
|
||||
|
||||
// Snapshot of the focused UI element at chord-start, shipped over from
|
||||
// Rust on the ``dictate:start`` payload. Held in a ref so it survives
|
||||
// the 1–2 s transcribe + refine window — the paste only fires once the
|
||||
// final text comes back.
|
||||
const focusRef = useRef<FocusSnapshot | null>(null);
|
||||
|
||||
const session = useCaptureRecordingSession({
|
||||
keepMicWarm: micWarm,
|
||||
onFinalText: async (text, _capture, allowAutoPaste) => {
|
||||
const focus = focusRef.current;
|
||||
// Consume-once: a second chord before this fires would overwrite
|
||||
// focusRef, but nulling it here guards against the late-arriving
|
||||
// refine-result firing a paste after the user has moved on.
|
||||
focusRef.current = null;
|
||||
onFinalText: async (text, _capture, allowAutoPaste, context) => {
|
||||
// Focus is the snapshot taken at chord-start and threaded through as this
|
||||
// take's context, so it survives the 1–2 s transcribe + refine window and
|
||||
// overlapping dictations can't paste into each other's target.
|
||||
const focus = context as FocusSnapshot | null;
|
||||
if (!allowAutoPaste) return;
|
||||
if (!focus || !text.trim()) return;
|
||||
try {
|
||||
@@ -81,8 +74,7 @@ export function DictateWindow() {
|
||||
const unlistens: UnlistenFn[] = [];
|
||||
const registrations = [
|
||||
listen<{ focus: FocusSnapshot | null }>('dictate:start', (event) => {
|
||||
focusRef.current = event.payload?.focus ?? null;
|
||||
sessionRef.current.startRecording();
|
||||
sessionRef.current.startRecording(event.payload?.focus ?? null);
|
||||
}),
|
||||
listen('dictate:stop', () => {
|
||||
// Forward stops that arrive while getUserMedia is still resolving.
|
||||
|
||||
@@ -4,7 +4,11 @@ import { convertToWav } from '@/lib/utils/audio';
|
||||
|
||||
interface UseAudioRecordingOptions {
|
||||
maxDurationSeconds?: number;
|
||||
onRecordingComplete?: (blob: Blob, duration?: number) => void;
|
||||
// ``context`` is whatever was handed to ``startRecording`` for this take,
|
||||
// threaded back untouched so callers can correlate the result with the
|
||||
// recording it came from (the dictate window pairs it with the focus
|
||||
// snapshot captured at chord-start).
|
||||
onRecordingComplete?: (blob: Blob, duration?: number, context?: unknown) => void;
|
||||
/**
|
||||
* Keep the microphone ``MediaStream`` open between recordings instead of
|
||||
* tearing it down on every stop. This is what removes the "first words get
|
||||
@@ -184,7 +188,7 @@ export function useAudioRecording({
|
||||
}, [keepWarm, acquireStream]);
|
||||
|
||||
const startRecording = useCallback(
|
||||
async () => {
|
||||
async (context?: unknown) => {
|
||||
// A second chord can arrive while the first one is still waiting on
|
||||
// getUserMedia. Never create overlapping MediaRecorders on the same
|
||||
// coalesced stream; the original take will honor any deferred stop.
|
||||
@@ -275,11 +279,11 @@ export function useAudioRecording({
|
||||
// Convert to WAV format to avoid needing ffmpeg on backend
|
||||
try {
|
||||
const wavBlob = await convertToWav(webmBlob);
|
||||
onRecordingComplete?.(wavBlob, recordedDuration);
|
||||
onRecordingComplete?.(wavBlob, recordedDuration, context);
|
||||
} catch (err) {
|
||||
console.error('Error converting audio to WAV:', err);
|
||||
// Fallback to original blob if conversion fails
|
||||
onRecordingComplete?.(webmBlob, recordedDuration);
|
||||
onRecordingComplete?.(webmBlob, recordedDuration, context);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -60,8 +60,9 @@ export interface UseCaptureRecordingSessionOptions {
|
||||
/**
|
||||
* Fired after a capture row is created on the server. Callers can use this
|
||||
* to select the new capture or emit a Tauri event to a sibling window.
|
||||
* ``context`` is whatever was passed to ``startRecording`` for this take.
|
||||
*/
|
||||
onCaptureCreated?: (capture: CaptureResponse) => void;
|
||||
onCaptureCreated?: (capture: CaptureResponse, context?: unknown) => void;
|
||||
/**
|
||||
* Fired with the final delivered text — refined if ``auto_refine`` was on
|
||||
* for this capture, raw transcript otherwise. Used by the floating
|
||||
@@ -69,12 +70,14 @@ export interface UseCaptureRecordingSessionOptions {
|
||||
*
|
||||
* ``allowAutoPaste`` snapshots the setting at chord-start so a refine that
|
||||
* lands after the user flips the toggle still uses the value the capture
|
||||
* was created under.
|
||||
* was created under. ``context`` is the value passed to ``startRecording``
|
||||
* for this take, so overlapping dictations can't cross their targets.
|
||||
*/
|
||||
onFinalText?: (
|
||||
text: string,
|
||||
capture: CaptureResponse,
|
||||
allowAutoPaste: boolean,
|
||||
context?: unknown,
|
||||
) => void;
|
||||
}
|
||||
|
||||
@@ -85,7 +88,7 @@ export interface UseCaptureRecordingSessionResult {
|
||||
isRecording: boolean;
|
||||
isUploading: boolean;
|
||||
isRefining: boolean;
|
||||
startRecording: () => void;
|
||||
startRecording: (context?: unknown) => void;
|
||||
stopRecording: () => void;
|
||||
toggleRecording: () => void;
|
||||
dismissError: () => void;
|
||||
@@ -128,10 +131,13 @@ export function useCaptureRecordingSession(
|
||||
const onFinalTextRef = useRef(options.onFinalText);
|
||||
onFinalTextRef.current = options.onFinalText;
|
||||
|
||||
// Snapshot of ``allow_auto_paste`` from the capture-create response —
|
||||
// held so the refine onSuccess (which only sees the plain CaptureResponse)
|
||||
// can still pass the original setting through to onFinalText.
|
||||
const allowAutoPasteRef = useRef<boolean>(true);
|
||||
// Per-capture recording context and its ``allow_auto_paste`` snapshot, keyed
|
||||
// by capture id so a refine that resolves after another dictation started
|
||||
// still delivers to the right target with the setting the capture was created
|
||||
// under. Populated on capture-create and consumed once the final text lands.
|
||||
const captureDeliveryRef = useRef<Map<string, { context: unknown; allowAutoPaste: boolean }>>(
|
||||
new Map(),
|
||||
);
|
||||
|
||||
const clearRestTimer = useCallback(() => {
|
||||
if (restTimerRef.current !== null) {
|
||||
@@ -197,20 +203,34 @@ export function useCaptureRecordingSession(
|
||||
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
||||
broadcastUpdated(captureId);
|
||||
if (pillStateRef.current === 'refining') scheduleHidePill();
|
||||
const delivery = captureDeliveryRef.current.get(captureId);
|
||||
captureDeliveryRef.current.delete(captureId);
|
||||
const finalText = data.transcript_refined ?? data.transcript_raw;
|
||||
if (finalText) {
|
||||
onFinalTextRef.current?.(finalText, data, allowAutoPasteRef.current);
|
||||
onFinalTextRef.current?.(
|
||||
finalText,
|
||||
data,
|
||||
delivery?.allowAutoPaste ?? true,
|
||||
delivery?.context,
|
||||
);
|
||||
}
|
||||
},
|
||||
onError: (err: Error) => {
|
||||
onError: (err: Error, captureId) => {
|
||||
captureDeliveryRef.current.delete(captureId);
|
||||
showError(err.message || 'Refinement failed');
|
||||
},
|
||||
});
|
||||
|
||||
const uploadMutation = useMutation({
|
||||
mutationFn: async ({ file, source }: { file: File; source: CaptureSource }) =>
|
||||
apiClient.createCapture(file, { source }),
|
||||
onSuccess: (capture) => {
|
||||
mutationFn: async ({
|
||||
file,
|
||||
source,
|
||||
}: {
|
||||
file: File;
|
||||
source: CaptureSource;
|
||||
context?: unknown;
|
||||
}) => apiClient.createCapture(file, { source }),
|
||||
onSuccess: (capture, { context }) => {
|
||||
queryClient.setQueryData<CaptureListResponse>(['captures'], (prev) => {
|
||||
if (!prev) return prev;
|
||||
if (prev.items.some((c) => c.id === capture.id)) return prev;
|
||||
@@ -218,9 +238,12 @@ export function useCaptureRecordingSession(
|
||||
});
|
||||
queryClient.invalidateQueries({ queryKey: ['captures'] });
|
||||
broadcastCreated(capture);
|
||||
onCaptureCreatedRef.current?.(capture);
|
||||
allowAutoPasteRef.current = capture.allow_auto_paste;
|
||||
onCaptureCreatedRef.current?.(capture, context);
|
||||
if (capture.auto_refine) {
|
||||
captureDeliveryRef.current.set(capture.id, {
|
||||
context,
|
||||
allowAutoPaste: capture.allow_auto_paste,
|
||||
});
|
||||
setPillState('refining');
|
||||
refineMutation.mutate(capture.id);
|
||||
} else {
|
||||
@@ -230,6 +253,7 @@ export function useCaptureRecordingSession(
|
||||
capture.transcript_raw,
|
||||
capture,
|
||||
capture.allow_auto_paste,
|
||||
context,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -258,7 +282,7 @@ export function useCaptureRecordingSession(
|
||||
releaseWarm,
|
||||
} = useAudioRecording({
|
||||
keepWarm: options.keepMicWarm ?? false,
|
||||
onRecordingComplete: (blob, recordedDuration) => {
|
||||
onRecordingComplete: (blob, recordedDuration, context) => {
|
||||
// Trigger-happy tap — MediaRecorder hasn't emitted a usable chunk yet
|
||||
// so the blob is empty or unparseable. Surface it as a transient pill
|
||||
// so the user sees their recording was recognised and canceled.
|
||||
@@ -276,7 +300,7 @@ export function useCaptureRecordingSession(
|
||||
const file = new File([blob], `dictation-${Date.now()}.${extension}`, {
|
||||
type: blob.type,
|
||||
});
|
||||
uploadMutation.mutate({ file, source: 'dictation' });
|
||||
uploadMutation.mutate({ file, source: 'dictation', context });
|
||||
},
|
||||
});
|
||||
|
||||
@@ -286,13 +310,16 @@ export function useCaptureRecordingSession(
|
||||
}
|
||||
}, [recordError, showError]);
|
||||
|
||||
const startRecording = useCallback(() => {
|
||||
if (isRecording) return;
|
||||
clearRestTimer();
|
||||
setFrozenElapsedMs(0);
|
||||
setPillState('recording');
|
||||
beginAudioRecording();
|
||||
}, [isRecording, beginAudioRecording, clearRestTimer]);
|
||||
const startRecording = useCallback(
|
||||
(context?: unknown) => {
|
||||
if (isRecording) return;
|
||||
clearRestTimer();
|
||||
setFrozenElapsedMs(0);
|
||||
setPillState('recording');
|
||||
beginAudioRecording(context);
|
||||
},
|
||||
[isRecording, beginAudioRecording, clearRestTimer],
|
||||
);
|
||||
|
||||
const toggleRecording = useCallback(() => {
|
||||
if (isRecording) {
|
||||
|
||||
Reference in New Issue
Block a user