From 85a3e1363f7e9882872d3afa5351961909cf2fde Mon Sep 17 00:00:00 2001 From: James Pine Date: Thu, 23 Apr 2026 02:23:12 -0700 Subject: [PATCH] feat(capture): gate global hotkey on dictation readiness checklist Stops the "stuck pill" failure where pressing the chord with missing STT/LLM models triggers a recording that has nowhere to land. The hotkey now stays disarmed until every gate (models downloaded, Input Monitoring + Accessibility granted) is green; the empty-state checklist in CapturesTab surfaces each unmet gate with a one-click action and auto-arms the chord once everything turns green. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/CapturesTab/CapturesTab.tsx | 5 + .../DictationReadinessChecklist.tsx | 223 ++++++++++++++++++ app/src/components/ServerTab/CapturesPage.tsx | 30 ++- app/src/lib/api/client.ts | 5 + app/src/lib/api/types.ts | 20 ++ app/src/lib/hooks/useChordSync.ts | 33 ++- app/src/lib/hooks/useDictationReadiness.ts | 97 ++++++++ backend/backends/__init__.py | 5 + backend/models.py | 28 +++ backend/routes/captures.py | 48 ++++ 10 files changed, 480 insertions(+), 14 deletions(-) create mode 100644 app/src/components/CapturesTab/DictationReadinessChecklist.tsx create mode 100644 app/src/lib/hooks/useDictationReadiness.ts diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index 3f158db6..df0f9c54 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -21,6 +21,7 @@ import { } from 'lucide-react'; import { useEffect, useMemo, useRef, useState } from 'react'; import { CapturePill } from '@/components/CapturePill/CapturePill'; +import { DictationReadinessChecklist } from '@/components/CapturesTab/DictationReadinessChecklist'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; import { @@ -44,6 +45,7 @@ import type { import type { LanguageCode } from '@/lib/constants/languages'; import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui'; import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession'; +import { useDictationReadiness } from '@/lib/hooks/useDictationReadiness'; import { useCaptureSettings } from '@/lib/hooks/useSettings'; import { cn } from '@/lib/utils/cn'; import { displayLabelForKey, modifierSideHint } from '@/lib/utils/keyCodes'; @@ -188,6 +190,7 @@ export function CapturesTab() { const hotkeyEnabled = captureSettings?.hotkey_enabled ?? false; const pushToTalkKeys = captureSettings?.chord_push_to_talk_keys ?? []; const toggleToTalkKeys = captureSettings?.chord_toggle_to_talk_keys ?? []; + const readiness = useDictationReadiness(); const session = useCaptureRecordingSession({ onCaptureCreated: (capture) => setSelectedId(capture.id), @@ -776,6 +779,8 @@ export function CapturesTab() {

Pick a capture to see the transcript.

+ ) : hotkeyEnabled && !readiness.allReady ? ( + ) : hotkeyEnabled && (pushToTalkKeys.length || toggleToTalkKeys.length) ? (
diff --git a/app/src/components/CapturesTab/DictationReadinessChecklist.tsx b/app/src/components/CapturesTab/DictationReadinessChecklist.tsx new file mode 100644 index 00000000..f20ba76b --- /dev/null +++ b/app/src/components/CapturesTab/DictationReadinessChecklist.tsx @@ -0,0 +1,223 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { + Accessibility, + CheckCircle2, + Circle, + Cpu, + Download, + ExternalLink, + Keyboard, + Loader2, +} from 'lucide-react'; +import { useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { useToast } from '@/components/ui/use-toast'; +import { apiClient } from '@/lib/api/client'; +import type { DictationReadiness, ReadinessGate } from '@/lib/hooks/useDictationReadiness'; +import { cn } from '@/lib/utils/cn'; + +interface RowProps { + icon: React.ReactNode; + title: string; + description: string; + ready: boolean; + action?: React.ReactNode; +} + +function ChecklistRow({ icon, title, description, ready, action }: RowProps) { + return ( +
+
+ {ready ? ( + + ) : ( + + )} +
+
+
+ {icon} +

{title}

+
+

{description}

+ {!ready && action ?
{action}
: null} +
+
+ ); +} + +/** + * Renders one row per dictation-readiness gate. Each unmet gate gets an + * inline action — Download for missing models, Open Settings for missing + * TCC permissions — so the user can resolve everything without leaving + * Captures. + * + * The chord stays disarmed until every row is green; this is what stops the + * "stuck pill" failure mode of pressing the chord with a missing model. + */ +export function DictationReadinessChecklist({ readiness }: { readiness: DictationReadiness }) { + const queryClient = useQueryClient(); + const { toast } = useToast(); + const [downloading, setDownloading] = useState>(new Set()); + + const downloadMutation = useMutation({ + mutationFn: async ({ modelName }: { gate: ReadinessGate; modelName: string }) => + apiClient.triggerModelDownload(modelName), + onSuccess: (_data, vars) => { + // Bump model status + readiness so the checklist row flips green as + // soon as the cache is populated. Keep the gate in `downloading` until + // readiness reports `ready: true` to avoid a flash of "Download" on + // post-completion polls. + queryClient.invalidateQueries({ queryKey: ['modelStatus'] }); + queryClient.invalidateQueries({ queryKey: ['capture-readiness'] }); + toast({ + title: 'Download started', + description: `${vars.gate === 'stt' ? readiness.stt?.display_name : readiness.llm?.display_name} is downloading. The shortcut will arm itself when it finishes.`, + }); + }, + onError: (err: Error, vars) => { + setDownloading((prev) => { + const next = new Set(prev); + next.delete(vars.gate); + return next; + }); + toast({ + title: 'Download failed', + description: err.message, + variant: 'destructive', + }); + }, + }); + + const startDownload = (gate: ReadinessGate, modelName: string) => { + setDownloading((prev) => new Set(prev).add(gate)); + downloadMutation.mutate({ gate, modelName }); + }; + + // Once readiness flips ready=true for a gate, drop it from `downloading` + // so subsequent reopens show the green check, not "Downloading…". + const isDownloading = (gate: ReadinessGate, ready: boolean) => !ready && downloading.has(gate); + + const sttSize = + readiness.stt?.size_mb != null ? `${(readiness.stt.size_mb / 1000).toFixed(1)} GB` : null; + const llmSize = + readiness.llm?.size_mb != null ? `${(readiness.llm.size_mb / 1000).toFixed(1)} GB` : null; + + return ( +
+
+

+ A few things before you can dictate +

+

+ The shortcut stays off until everything below is ready. +

+
+ + {readiness.stt && ( + } + title={`${readiness.stt.display_name} (speech-to-text)`} + description={ + readiness.stt.ready + ? 'Model downloaded.' + : `Needed to transcribe your audio${sttSize ? ` · ${sttSize}` : ''}.` + } + ready={readiness.stt.ready} + action={ + + } + /> + )} + + {readiness.llm && ( + } + title={`${readiness.llm.display_name} (refinement)`} + description={ + readiness.llm.ready + ? 'Model downloaded.' + : `Cleans up the raw transcript before paste${llmSize ? ` · ${llmSize}` : ''}.` + } + ready={readiness.llm.ready} + action={ + + } + /> + )} + + } + title="Input Monitoring permission" + description={ + readiness.inputMonitoring + ? 'macOS allows Voicebox to detect your global shortcut.' + : 'macOS needs to allow Voicebox to detect the global shortcut.' + } + ready={readiness.inputMonitoring} + action={ + + } + /> + + } + title="Accessibility permission" + description={ + readiness.accessibility + ? 'Voicebox can paste transcriptions into other apps.' + : 'Required so transcriptions can paste into the focused app.' + } + ready={readiness.accessibility} + action={ + + } + /> +
+ ); +} diff --git a/app/src/components/ServerTab/CapturesPage.tsx b/app/src/components/ServerTab/CapturesPage.tsx index de4761d6..94bb78a5 100644 --- a/app/src/components/ServerTab/CapturesPage.tsx +++ b/app/src/components/ServerTab/CapturesPage.tsx @@ -21,6 +21,8 @@ import { SelectValue, } from '@/components/ui/select'; import { Toggle } from '@/components/ui/toggle'; +import { useToast } from '@/components/ui/use-toast'; +import { useDictationReadiness } from '@/lib/hooks/useDictationReadiness'; import { useCaptureSettings } from '@/lib/hooks/useSettings'; import { useProfiles } from '@/lib/hooks/useProfiles'; import { cn } from '@/lib/utils/cn'; @@ -132,6 +134,8 @@ function HotkeyPillPreview({ enabled }: { enabled: boolean }) { export function CapturesPage() { const { settings, update } = useCaptureSettings(); const { data: profiles } = useProfiles(); + const { toast } = useToast(); + const readiness = useDictationReadiness(); const sttModel = settings?.stt_model ?? 'turbo'; const language = settings?.language ?? 'auto'; const autoRefine = settings?.auto_refine ?? true; @@ -172,7 +176,31 @@ export function CapturesPage() { update({ hotkey_enabled: v })} + onCheckedChange={(v) => { + update({ hotkey_enabled: v }); + // Surface model-readiness blocks at the toggle. The + // InputMonitoringNotice below already covers TCC, but + // missing models would otherwise be invisible from this + // page — the user toggles on, presses the chord, and + // nothing happens because useChordSync gates on readiness. + if (!v) return; + const missingModels = readiness.missing.filter( + (g) => g === 'stt' || g === 'llm', + ); + if (missingModels.length === 0) return; + const names = [ + missingModels.includes('stt') ? readiness.stt?.display_name : null, + missingModels.includes('llm') ? readiness.llm?.display_name : null, + ] + .filter(Boolean) + .join(' and '); + toast({ + title: 'Shortcut on, but not yet armed', + description: `${names} still need${ + missingModels.length === 1 ? 's' : '' + } to download. Open the Captures tab to start.`, + }); + }} /> } /> diff --git a/app/src/lib/api/client.ts b/app/src/lib/api/client.ts index f88c6c1a..fcfcef74 100644 --- a/app/src/lib/api/client.ts +++ b/app/src/lib/api/client.ts @@ -38,6 +38,7 @@ import type { CaptureListResponse, CaptureResponse, CaptureCreateResponse, + CaptureReadinessResponse, CaptureRefineRequest, CaptureRetranscribeRequest, CaptureSettings, @@ -486,6 +487,10 @@ class ApiClient { return this.request('/settings/captures'); } + async getCaptureReadiness(): Promise { + return this.request('/capture/readiness'); + } + async updateCaptureSettings(patch: CaptureSettingsUpdate): Promise { return this.request('/settings/captures', { method: 'PUT', diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index f5959b06..e887831e 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -219,6 +219,26 @@ export interface CaptureSettings { export type CaptureSettingsUpdate = Partial; +/** + * One row in the dictation readiness checklist. ``model_name`` is the + * canonical id understood by ``POST /models/download`` so the UI can wire a + * one-click "Download" button without a second lookup. + */ +export interface ModelReadiness { + ready: boolean; + model_name: string; + display_name: string; + size: string; + size_mb?: number | null; +} + +/** Backend half of the dictation readiness check. The frontend combines this + * with TCC permission state into the full checklist used by useDictationReadiness. */ +export interface CaptureReadinessResponse { + stt: ModelReadiness; + llm: ModelReadiness; +} + export interface GenerationSettings { max_chunk_chars: number; crossfade_ms: number; diff --git a/app/src/lib/hooks/useChordSync.ts b/app/src/lib/hooks/useChordSync.ts index df95d50a..4758db88 100644 --- a/app/src/lib/hooks/useChordSync.ts +++ b/app/src/lib/hooks/useChordSync.ts @@ -1,27 +1,34 @@ import { invoke } from '@tauri-apps/api/core'; import { useEffect } from 'react'; +import { useDictationReadiness } from '@/lib/hooks/useDictationReadiness'; import { useCaptureSettings } from '@/lib/hooks/useSettings'; import { usePlatform } from '@/platform/PlatformContext'; /** * Spawn (or quiet) the global hotkey monitor based on the saved - * `capture_settings.hotkey_enabled` flag, and keep its bindings in sync with - * the user's chord choices. + * `capture_settings.hotkey_enabled` flag AND the dictation readiness gates, + * and keep its bindings in sync with the user's chord choices. * * Boot sequence: - * - hotkey_enabled = false → call `disable_hotkey` (no-op if monitor was - * never spawned). Crucially, we do *not* call `enable_hotkey`, so the - * macOS Input Monitoring TCC prompt is never triggered for users who - * haven't opted in. - * - hotkey_enabled = true → call `enable_hotkey` with the saved chords. - * This is the call that creates the CGEventTap and triggers the TCC - * prompt on first opt-in. + * - hotkey_enabled = false OR any readiness gate missing → call + * `disable_hotkey` (no-op if monitor was never spawned). Crucially, we do + * *not* call `enable_hotkey` in this state, so the macOS Input Monitoring + * TCC prompt is never triggered for users who haven't opted in, AND the + * chord physically can't fire when models aren't downloaded — preventing + * the "stuck pill" failure mode where dictation triggers but has nowhere + * to land. + * - hotkey_enabled = true AND all gates green → call `enable_hotkey` with + * the saved chords. This creates the CGEventTap and triggers the TCC + * prompt on first opt-in. Re-runs whenever a gate flips green (e.g. the + * user finishes downloading Whisper in another tab) so the chord + * auto-arms without making the user toggle off/on. * * Call once from the main app shell. */ export function useChordSync() { const platform = usePlatform(); const { settings } = useCaptureSettings(); + const { allReady } = useDictationReadiness(); const enabled = settings?.hotkey_enabled; const pushKeys = settings?.chord_push_to_talk_keys; const toggleKeys = settings?.chord_toggle_to_talk_keys; @@ -29,16 +36,16 @@ export function useChordSync() { useEffect(() => { if (!platform.metadata.isTauri) return; if (enabled === undefined || !pushKeys || !toggleKeys) return; - const command = enabled ? 'enable_hotkey' : 'disable_hotkey'; - const args = enabled - ? { pushToTalk: pushKeys, toggleToTalk: toggleKeys } - : {}; + const shouldArm = enabled && allReady; + const command = shouldArm ? 'enable_hotkey' : 'disable_hotkey'; + const args = shouldArm ? { pushToTalk: pushKeys, toggleToTalk: toggleKeys } : {}; invoke(command, args).catch((err) => { console.warn(`[chord-sync] ${command} failed:`, err); }); }, [ platform.metadata.isTauri, enabled, + allReady, // Stringify so a referentially-new array with the same content // doesn't fire a redundant invoke on every settings refetch. pushKeys?.join(','), diff --git a/app/src/lib/hooks/useDictationReadiness.ts b/app/src/lib/hooks/useDictationReadiness.ts new file mode 100644 index 00000000..8f08df76 --- /dev/null +++ b/app/src/lib/hooks/useDictationReadiness.ts @@ -0,0 +1,97 @@ +import { useQuery } from '@tanstack/react-query'; +import { useAccessibilityPermission } from '@/components/AccessibilityGate/AccessibilityGate'; +import { useInputMonitoringPermission } from '@/components/InputMonitoringGate/InputMonitoringGate'; +import { apiClient } from '@/lib/api/client'; +import type { ModelReadiness } from '@/lib/api/types'; +import { usePlatform } from '@/platform/PlatformContext'; + +const READINESS_POLL_INTERVAL_MS = 5_000; + +export type ReadinessGate = 'stt' | 'llm' | 'input_monitoring' | 'accessibility'; + +export interface DictationReadiness { + isLoading: boolean; + allReady: boolean; + /** Subset of gates that are NOT yet satisfied — what the checklist renders. */ + missing: ReadinessGate[]; + stt: ModelReadiness | undefined; + llm: ModelReadiness | undefined; + inputMonitoring: boolean; + accessibility: boolean; + refetch: () => void; + openInputMonitoringSettings: () => Promise; + openAccessibilitySettings: () => Promise; + recheckInputMonitoring: () => Promise; + recheckAccessibility: () => Promise; +} + +/** + * Single source of truth for "can the user trigger dictation right now?" + * + * Combines four gates into one struct so the chord-sync hook can refuse to + * arm the global hotkey unless every gate is green — the "stuck pill" we + * used to get on missing models is solved by never letting the chord fire + * in the first place. + * + * Gates: + * - stt / llm: backend ``/capture/readiness`` (polled, since downloads + * finish out-of-band — e.g. user kicks off a download in another tab and + * expects the toggle to auto-unlock when it lands) + * - input_monitoring / accessibility: macOS TCC checks via Tauri commands + * (rechecked on window focus by the underlying hooks) + * + * Hotkey-enabled is the user's intent toggle and is intentionally *not* + * a gate here — that's `useChordSync`'s concern. + */ +export function useDictationReadiness(): DictationReadiness { + const platform = usePlatform(); + const isTauri = platform.metadata.isTauri; + + const { + needsPermission: inputMonNeeds, + recheck: recheckInputMon, + openSettings: openInputMon, + } = useInputMonitoringPermission(); + const { + needsPermission: a11yNeeds, + recheck: recheckA11y, + openSettings: openA11y, + } = useAccessibilityPermission(); + + const { data, isLoading, refetch } = useQuery({ + queryKey: ['capture-readiness'], + queryFn: () => apiClient.getCaptureReadiness(), + refetchInterval: READINESS_POLL_INTERVAL_MS, + refetchOnWindowFocus: true, + }); + + // On the web build there's no TCC layer — treat both as granted so the + // checklist doesn't block users who can't even open System Settings. + const inputMonitoring = isTauri ? !inputMonNeeds : true; + const accessibility = isTauri ? !a11yNeeds : true; + const sttReady = data?.stt.ready ?? false; + const llmReady = data?.llm.ready ?? false; + + const missing: ReadinessGate[] = []; + if (!sttReady) missing.push('stt'); + if (!llmReady) missing.push('llm'); + if (!inputMonitoring) missing.push('input_monitoring'); + if (!accessibility) missing.push('accessibility'); + + return { + isLoading, + allReady: missing.length === 0, + missing, + stt: data?.stt, + llm: data?.llm, + inputMonitoring, + accessibility, + refetch: () => { + refetch(); + }, + openInputMonitoringSettings: openInputMon, + openAccessibilitySettings: openA11y, + recheckInputMonitoring: recheckInputMon, + recheckAccessibility: recheckA11y, + }; +} diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index e45e9d5b..2437a87b 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -480,6 +480,11 @@ def get_llm_model_configs() -> list[ModelConfig]: return _get_qwen_llm_configs() +def get_stt_model_configs() -> list[ModelConfig]: + """Return only STT (Whisper) model configs.""" + return _get_whisper_configs() + + # Lookup helpers — these replace the if/elif chains in main.py diff --git a/backend/models.py b/backend/models.py index 386a3ede..1e041353 100644 --- a/backend/models.py +++ b/backend/models.py @@ -427,6 +427,34 @@ class PersonalitySpeakRequest(BaseModel): ) +class ModelReadiness(BaseModel): + """Per-model entry in the dictation readiness checklist. + + ``model_name`` is the canonical id used by ``POST /models/download`` so the + frontend can wire a one-click "Download" button without a second lookup. + ``size`` is the user's chosen variant (e.g. "turbo", "0.6B"); ``display_name`` + is what the checklist row should show ("Whisper Turbo"). + """ + + ready: bool + model_name: str + display_name: str + size: str + size_mb: Optional[int] = None + + +class CaptureReadinessResponse(BaseModel): + """Backend gates that must be green before the global hotkey will fire. + + The frontend combines this with its own TCC permission checks (input + monitoring, accessibility) into the full dictation readiness checklist. + Hotkey-enabled is the user's intent toggle and lives outside this struct. + """ + + stt: ModelReadiness + llm: ModelReadiness + + class HealthResponse(BaseModel): """Response model for health check.""" diff --git a/backend/routes/captures.py b/backend/routes/captures.py index 24ef36c5..40a5adc1 100644 --- a/backend/routes/captures.py +++ b/backend/routes/captures.py @@ -7,6 +7,8 @@ from fastapi.responses import FileResponse from sqlalchemy.orm import Session from .. import config, models +from ..backends import get_llm_model_configs, get_stt_model_configs +from ..backends.base import is_model_cached from ..database import Capture as DBCapture, get_db from ..services import captures as captures_service from ..services import settings as settings_service @@ -152,6 +154,52 @@ async def refine_capture_endpoint( return capture +@router.get("/capture/readiness", response_model=models.CaptureReadinessResponse) +async def capture_readiness_endpoint(db: Session = Depends(get_db)): + """Whether the STT and LLM models the user has selected are downloaded. + + The frontend gates the global hotkey on this — pressing the chord with + a missing model would otherwise produce a stuck "transcribing" pill that + waits forever for a download to finish. Checks on-disk cache, not RAM + load, so the answer survives backend restarts. + """ + saved = settings_service.get_capture_settings(db) + + stt_cfg = next( + (c for c in get_stt_model_configs() if c.model_size == saved.stt_model), + None, + ) + llm_cfg = next( + (c for c in get_llm_model_configs() if c.model_size == saved.llm_model), + None, + ) + + if stt_cfg is None or llm_cfg is None: + # Should be impossible — both fields are pattern-validated against + # known sizes — but bail loudly rather than return half a response. + raise HTTPException( + status_code=500, + detail=f"No model config for stt={saved.stt_model} or llm={saved.llm_model}", + ) + + return models.CaptureReadinessResponse( + stt=models.ModelReadiness( + ready=is_model_cached(stt_cfg.hf_repo_id), + model_name=stt_cfg.model_name, + display_name=stt_cfg.display_name, + size=stt_cfg.model_size, + size_mb=stt_cfg.size_mb or None, + ), + llm=models.ModelReadiness( + ready=is_model_cached(llm_cfg.hf_repo_id), + model_name=llm_cfg.model_name, + display_name=llm_cfg.display_name, + size=llm_cfg.model_size, + size_mb=llm_cfg.size_mb or None, + ), + ) + + @router.post("/captures/{capture_id}/retranscribe", response_model=models.CaptureResponse) async def retranscribe_capture_endpoint( capture_id: str,