diff --git a/app/src/components/CapturePill/CapturePill.tsx b/app/src/components/CapturePill/CapturePill.tsx index 46f1e6d5..bf845781 100644 --- a/app/src/components/CapturePill/CapturePill.tsx +++ b/app/src/components/CapturePill/CapturePill.tsx @@ -10,6 +10,7 @@ export type PillState = | 'recording' | 'transcribing' | 'refining' + | 'speaking' | 'completed' | 'rest' | 'error'; @@ -18,13 +19,14 @@ const PILL_LABELS: Record, string> = { recording: 'Recording', transcribing: 'Transcribing', refining: 'Refining', + speaking: 'Speaking', completed: 'Done', }; function barModeFor( state: Exclude, ): 'generating' | 'playing' | 'idle' { - if (state === 'recording') return 'playing'; + if (state === 'recording' || state === 'speaking') return 'playing'; if (state === 'completed' || state === 'rest') return 'idle'; return 'generating'; } diff --git a/app/src/components/CapturesTab/CapturesTab.tsx b/app/src/components/CapturesTab/CapturesTab.tsx index b7ec8692..3f158db6 100644 --- a/app/src/components/CapturesTab/CapturesTab.tsx +++ b/app/src/components/CapturesTab/CapturesTab.tsx @@ -46,6 +46,7 @@ import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui'; import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession'; import { useCaptureSettings } from '@/lib/hooks/useSettings'; import { cn } from '@/lib/utils/cn'; +import { displayLabelForKey, modifierSideHint } from '@/lib/utils/keyCodes'; import { usePlayerStore } from '@/stores/playerStore'; const CAPTURE_AUDIO_MIME = 'audio/*,.wav,.mp3,.m4a,.flac,.ogg,.webm'; @@ -86,6 +87,30 @@ function snippetOf(capture: CaptureResponse): string { return source.trim() || '(no transcript)'; } +function ChordKeys({ keys }: { keys: string[] }) { + if (keys.length === 0) return null; + return ( +
+ {keys.map((k) => { + const side = modifierSideHint(k); + return ( + + {displayLabelForKey(k)} + {side ? ( + + {side} + + ) : null} + + ); + })} +
+ ); +} + function SourceBadge({ source }: { source: CaptureSource }) { const Icon = source === 'dictation' ? Mic : source === 'recording' ? CircleDot : FileAudio; const label = source === 'dictation' ? 'Dictation' : source === 'recording' ? 'Recording' : 'File'; @@ -160,6 +185,9 @@ export function CapturesTab() { const { settings: captureSettings } = useCaptureSettings(); const sttModel = captureSettings?.stt_model ?? 'turbo'; const llmModel = captureSettings?.llm_model ?? '0.6B'; + const hotkeyEnabled = captureSettings?.hotkey_enabled ?? false; + const pushToTalkKeys = captureSettings?.chord_push_to_talk_keys ?? []; + const toggleToTalkKeys = captureSettings?.chord_toggle_to_talk_keys ?? []; const session = useCaptureRecordingSession({ onCaptureCreated: (capture) => setSelectedId(capture.id), @@ -394,17 +422,11 @@ export function CapturesTab() { ) : filtered.length === 0 ? ( -
+
{search ? (

No captures match "{search}"

) : ( - <> -

No captures yet.

- - +

No captures yet.

)}
) : ( @@ -744,22 +766,53 @@ export function CapturesTab() {
) : (
-
- - {capturesLoading ? ( + {capturesLoading ? ( +
+

Loading captures…

- ) : captures.length ? ( +
+ ) : captures.length ? ( +
+

Pick a capture to see the transcript.

- ) : ( - <> -

No captures yet.

- - - )} -
+
+ ) : hotkeyEnabled && (pushToTalkKeys.length || toggleToTalkKeys.length) ? ( +
+
+ {pushToTalkKeys.length ? ( +
+ + + Hold to record + +
+ ) : null} + {toggleToTalkKeys.length ? ( +
+ + + Toggle hands-free + +
+ ) : null} +
+

+ Press the shortcut anywhere on your machine to start your first capture. +

+
+ ) : ( +
+ +

No captures yet.

+

+ Turn on the global shortcut to dictate from anywhere — or click + Dictate above for an in-app capture. +

+ +
+ )}
)} diff --git a/app/src/components/DictateWindow/DictateWindow.tsx b/app/src/components/DictateWindow/DictateWindow.tsx index af7345d4..7c499c6f 100644 --- a/app/src/components/DictateWindow/DictateWindow.tsx +++ b/app/src/components/DictateWindow/DictateWindow.tsx @@ -4,6 +4,7 @@ import { useEffect, useRef } from 'react'; import { CapturePill } from '@/components/CapturePill/CapturePill'; import type { FocusSnapshot } from '@/lib/api/types'; import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession'; +import { useSpeakEvents } from '@/lib/hooks/useSpeakEvents'; /** * Floating dictate surface shown in a separate transparent Tauri window. @@ -82,27 +83,47 @@ export function DictateWindow() { }; }, []); - // When the pill cycle ends, tell Rust to tuck the window away. The Rust - // side is responsible for the hide + park-off-screen + click-through - // combo because calling hide() directly from JS has been unreliable for - // transparent always-on-top windows on macOS. Showing is the reverse — - // the HotkeyMonitor restores position, clicks, and visibility when a - // chord next fires. + // Subscribe to agent-initiated speak events so the pill surfaces while + // voicebox.speak (MCP) or POST /speak is producing audio. We ask Rust + // to show the pill window by emitting `dictate:show` — the existing + // `dictate:hide` happens on cycle end below. + const speaking = useSpeakEvents(); + const prevSpeakingIdRef = useRef(null); useEffect(() => { - if (session.pillState === 'hidden') { + const id = speaking?.generationId ?? null; + if (id && id !== prevSpeakingIdRef.current) { + emit('dictate:show').catch(() => {}); + } + prevSpeakingIdRef.current = id; + }, [speaking?.generationId]); + + // Compose the effective pill state: speak events override the capture + // session when both would render, because agent speech is always + // category-mattering (the user can't hear two pills). Elapsed time + // restarts for speaking so the timer reflects playback length. + const isSpeaking = Boolean(speaking); + const effectiveState = isSpeaking ? 'speaking' : session.pillState; + const effectiveElapsed = isSpeaking ? speaking!.elapsedMs : session.pillElapsedMs; + + // When the pill cycle ends (no capture AND no speak), tell Rust to tuck + // the window away. Rust owns the hide + park-off-screen + click-through + // combo because calling hide() directly from JS has been unreliable for + // transparent always-on-top windows on macOS. + useEffect(() => { + if (effectiveState === 'hidden') { emit('dictate:hide').catch(() => {}); } - }, [session.pillState]); + }, [effectiveState]); return (
- {session.pillState !== 'hidden' ? ( + {effectiveState !== 'hidden' ? ( => { + if (!platform.metadata.isTauri) return true; + setChecking(true); + try { + const trusted = await invoke('check_input_monitoring_permission'); + setNeedsPermission(!trusted); + return trusted; + } catch (err) { + console.warn('[input-monitoring] check failed:', err); + return false; + } finally { + setChecking(false); + } + }, [platform.metadata.isTauri]); + + useEffect(() => { + if (!platform.metadata.isTauri) return; + recheck(); + const onFocus = () => { + recheck(); + }; + window.addEventListener('focus', onFocus); + return () => window.removeEventListener('focus', onFocus); + }, [platform.metadata.isTauri, recheck]); + + const openSettings = useCallback(async () => { + try { + await invoke('open_input_monitoring_settings'); + } catch (err) { + console.warn('[input-monitoring] open settings failed:', err); + } + }, []); + + return { needsPermission, checking, recheck, openSettings }; +} + +/** + * Inline notice rendered under the global-shortcut toggle when the user has + * opted in but macOS Input Monitoring is not granted. Returns null when the + * permission is present (or when the toggle is off and the notice would just + * be noise). + */ +export function InputMonitoringNotice({ enabled }: { enabled: boolean }) { + const { needsPermission, checking, recheck, openSettings } = + useInputMonitoringPermission(); + const [stillMissing, setStillMissing] = useState(false); + + const handleRecheck = useCallback(async () => { + setStillMissing(false); + const trusted = await recheck(); + if (!trusted) setStillMissing(true); + }, [recheck]); + + if (!enabled || !needsPermission) return null; + + return ( +
+
+ +
+

+ Grant Input Monitoring to enable the global shortcut +

+

+ Voicebox needs System Settings → Privacy & Security → Input + Monitoring to detect your dictation chord. The toggle is on, but + macOS is blocking key events until you allow it. +

+
+ + +
+ {stillMissing && !checking && ( +

+ Still not detected. macOS usually requires quitting and reopening + Voicebox after toggling the permission. +

+ )} +
+
+
+ ); +} diff --git a/app/src/components/ServerTab/CapturesPage.tsx b/app/src/components/ServerTab/CapturesPage.tsx index 98c8d359..de4761d6 100644 --- a/app/src/components/ServerTab/CapturesPage.tsx +++ b/app/src/components/ServerTab/CapturesPage.tsx @@ -1,6 +1,7 @@ import { Check, ChevronDown, Keyboard, Laptop, Lock, Trash2, Volume2 } from 'lucide-react'; import { useEffect, useState } from 'react'; import { AccessibilityNotice } from '@/components/AccessibilityGate/AccessibilityGate'; +import { InputMonitoringNotice } from '@/components/InputMonitoringGate/InputMonitoringGate'; import { CapturePill, type PillState } from '@/components/CapturePill/CapturePill'; import { ChordPicker } from '@/components/ChordPicker/ChordPicker'; import { Button } from '@/components/ui/button'; @@ -140,13 +141,13 @@ export function CapturesPage() { const preserveTechnical = settings?.preserve_technical ?? true; const allowAutoPaste = settings?.allow_auto_paste ?? true; const defaultVoiceId = settings?.default_playback_voice_id ?? null; + const hotkeyEnabled = settings?.hotkey_enabled ?? false; const pushToTalkKeys = settings?.chord_push_to_talk_keys ?? ['MetaRight', 'AltGr']; const toggleToTalkKeys = settings?.chord_toggle_to_talk_keys ?? ['MetaRight', 'AltGr', 'Space']; // Mock-only settings — not yet wired to a backend. Keep local so the UI // still responds while Phase 7 (hotkey / clipboard / paste) catches up. const [archiveAudio, setArchiveAudio] = useState(true); - const [hotkeyEnabled, setHotkeyEnabled] = useState(true); const [copyToClipboard, setCopyToClipboard] = useState(true); const [retention, setRetention] = useState('forever'); const [chordEditor, setChordEditor] = useState<'push' | 'toggle' | null>(null); @@ -162,14 +163,21 @@ export function CapturesPage() { title="Dictation" description="Capture from anywhere on your machine with a global shortcut." > - - } - /> +
+ update({ hotkey_enabled: v })} + /> + } + /> + +
s.serverUrl); + const { bindings, upsertAsync, remove } = useMCPBindings(); + const { data: profiles } = useProfiles(); + const { settings: captureSettings, update: updateCapture } = useCaptureSettings(); + + const defaultProfileId = captureSettings?.default_playback_voice_id ?? ''; + const mcpUrl = `${serverUrl}/mcp`; + + const [newClientId, setNewClientId] = useState(''); + const [newLabel, setNewLabel] = useState(''); + const [newProfileId, setNewProfileId] = useState(''); + const [adding, setAdding] = useState(false); + + const handleAdd = async () => { + if (!newClientId.trim()) return; + setAdding(true); + try { + await upsertAsync({ + client_id: newClientId.trim(), + label: newLabel.trim() || null, + profile_id: newProfileId || null, + }); + setNewClientId(''); + setNewLabel(''); + setNewProfileId(''); + } finally { + setAdding(false); + } + }; + + return ( +
+
+ + + + + + + + + updateCapture({ + default_playback_voice_id: e.target.value || null, + }) + } + className="h-8 px-2 rounded-md border bg-background text-sm min-w-[180px]" + > + + {(profiles ?? []).map((p) => ( + + ))} + + } + /> + + + + {bindings.length === 0 ? ( +

+ No bindings yet. Add one below, then configure your MCP client to + send the matching X-Voicebox-Client-Id. +

+ ) : ( +
+ {bindings.map((b) => ( +
+
+
+ {b.label || b.client_id} +
+
+ {b.client_id} + {' · '} + {b.last_seen_at ? ( + + {' '} + last seen {formatRelative(b.last_seen_at)} + + ) : ( + never connected + )} +
+
+ + +
+ ))} +
+ )} + +
+
Add a binding
+
+ setNewClientId(e.target.value)} + className="h-9 px-3 rounded-md border bg-background text-sm" + /> + setNewLabel(e.target.value)} + className="h-9 px-3 rounded-md border bg-background text-sm" + /> + +
+ +
+ +
+ + +
+ ); +} + +function SnippetRow({ + title, + description, + snippet, +}: { + title: string; + description: string; + snippet: string; +}) { + const [copied, setCopied] = useState(false); + const copy = async () => { + try { + await navigator.clipboard.writeText(snippet); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch { + // ignore; user can still select-and-copy the pre content + } + }; + + return ( +
+
+
+
{title}
+
{description}
+
+ +
+ 
+        {snippet}
+
+
+ ); +} + +function formatRelative(iso: string): string { + const then = new Date(iso).getTime(); + const now = Date.now(); + const diff = Math.max(0, now - then); + if (diff < 60_000) return 'just now'; + if (diff < 3600_000) return `${Math.floor(diff / 60_000)} min ago`; + if (diff < 86400_000) return `${Math.floor(diff / 3600_000)} h ago`; + return `${Math.floor(diff / 86400_000)} d ago`; +} diff --git a/app/src/components/ServerTab/ServerTab.tsx b/app/src/components/ServerTab/ServerTab.tsx index 18de8dc6..f124b1c2 100644 --- a/app/src/components/ServerTab/ServerTab.tsx +++ b/app/src/components/ServerTab/ServerTab.tsx @@ -12,6 +12,7 @@ interface SettingsTab { | '/settings' | '/settings/generation' | '/settings/captures' + | '/settings/mcp' | '/settings/gpu' | '/settings/logs' | '/settings/changelog' @@ -23,6 +24,7 @@ const tabs: SettingsTab[] = [ { labelKey: 'settings.tabs.general', path: '/settings' }, { labelKey: 'settings.tabs.generation', path: '/settings/generation' }, { label: 'Captures', path: '/settings/captures' }, + { label: 'MCP', path: '/settings/mcp' }, { labelKey: 'settings.tabs.gpu', path: '/settings/gpu', tauriOnly: true }, { labelKey: 'settings.tabs.logs', path: '/settings/logs', tauriOnly: true }, { labelKey: 'settings.tabs.changelog', path: '/settings/changelog' }, diff --git a/app/src/lib/api/client.ts b/app/src/lib/api/client.ts index 85d7a9e1..f88c6c1a 100644 --- a/app/src/lib/api/client.ts +++ b/app/src/lib/api/client.ts @@ -45,6 +45,9 @@ import type { CaptureSource, GenerationSettings, GenerationSettingsUpdate, + MCPClientBinding, + MCPClientBindingListResponse, + MCPClientBindingUpsert, } from './types'; function formatErrorDetail(detail: unknown, fallback: string): string { @@ -503,6 +506,27 @@ class ApiClient { }); } + // MCP bindings — per-MCP-client voice/engine/intent mapping. + async listMCPBindings(): Promise { + return this.request('/mcp/bindings'); + } + + async upsertMCPBinding( + data: MCPClientBindingUpsert, + ): Promise { + return this.request('/mcp/bindings', { + method: 'PUT', + body: JSON.stringify(data), + }); + } + + async deleteMCPBinding(clientId: string): Promise<{ deleted: string }> { + return this.request<{ deleted: string }>( + `/mcp/bindings/${encodeURIComponent(clientId)}`, + { method: 'DELETE' }, + ); + } + // Model Management async getModelStatus(): Promise { return this.request('/models/status'); diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index c062067d..f5959b06 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -208,6 +208,9 @@ export interface CaptureSettings { preserve_technical: boolean; allow_auto_paste: boolean; default_playback_voice_id: string | null; + /** Whether the global keyboard hotkey is armed. Off by default — turning + * this on triggers the macOS Input Monitoring TCC prompt. */ + hotkey_enabled: boolean; /** rdev::Key variant names. Defaults: ["MetaRight","AltGr"]. */ chord_push_to_talk_keys: string[]; /** rdev::Key variant names. Defaults: ["MetaRight","AltGr","Space"]. */ @@ -465,3 +468,28 @@ export interface ApplyEffectsRequest { label?: string; set_as_default?: boolean; } + +/* ─── MCP ─────────────────────────────────────────────────────────────── */ + +export interface MCPClientBinding { + client_id: string; + label: string | null; + profile_id: string | null; + default_engine: string | null; + default_intent: 'respond' | 'rewrite' | 'compose' | null; + last_seen_at: string | null; + created_at: string; + updated_at: string; +} + +export interface MCPClientBindingUpsert { + client_id: string; + label?: string | null; + profile_id?: string | null; + default_engine?: string | null; + default_intent?: 'respond' | 'rewrite' | 'compose' | null; +} + +export interface MCPClientBindingListResponse { + items: MCPClientBinding[]; +} diff --git a/app/src/lib/hooks/useChordSync.ts b/app/src/lib/hooks/useChordSync.ts index 39af7220..df95d50a 100644 --- a/app/src/lib/hooks/useChordSync.ts +++ b/app/src/lib/hooks/useChordSync.ts @@ -4,32 +4,41 @@ import { useCaptureSettings } from '@/lib/hooks/useSettings'; import { usePlatform } from '@/platform/PlatformContext'; /** - * Push the user's saved chord into the running Rust `HotkeyMonitor`. - * The monitor boots with hard-coded right-hand defaults; this hook - * replaces them as soon as capture_settings resolves and re-applies on - * every subsequent change so chord edits land without a restart. + * Spawn (or quiet) the global hotkey monitor based on the saved + * `capture_settings.hotkey_enabled` flag, and keep its bindings in sync with + * the user's chord choices. * - * Call once from the main app shell — multiple call sites would just - * fire redundant invokes, since the chord engine swap is the same value - * either way. + * Boot sequence: + * - hotkey_enabled = false → call `disable_hotkey` (no-op if monitor was + * never spawned). Crucially, we do *not* call `enable_hotkey`, so the + * macOS Input Monitoring TCC prompt is never triggered for users who + * haven't opted in. + * - hotkey_enabled = true → call `enable_hotkey` with the saved chords. + * This is the call that creates the CGEventTap and triggers the TCC + * prompt on first opt-in. + * + * Call once from the main app shell. */ export function useChordSync() { const platform = usePlatform(); const { settings } = useCaptureSettings(); + const enabled = settings?.hotkey_enabled; const pushKeys = settings?.chord_push_to_talk_keys; const toggleKeys = settings?.chord_toggle_to_talk_keys; useEffect(() => { if (!platform.metadata.isTauri) return; - if (!pushKeys || !toggleKeys) return; - invoke('update_chord_bindings', { - pushToTalk: pushKeys, - toggleToTalk: toggleKeys, - }).catch((err) => { - console.warn('[chord-sync] failed to update bindings:', err); + if (enabled === undefined || !pushKeys || !toggleKeys) return; + const command = enabled ? 'enable_hotkey' : 'disable_hotkey'; + const args = enabled + ? { pushToTalk: pushKeys, toggleToTalk: toggleKeys } + : {}; + invoke(command, args).catch((err) => { + console.warn(`[chord-sync] ${command} failed:`, err); }); }, [ platform.metadata.isTauri, + enabled, // Stringify so a referentially-new array with the same content // doesn't fire a redundant invoke on every settings refetch. pushKeys?.join(','), diff --git a/app/src/lib/hooks/useMCPBindings.ts b/app/src/lib/hooks/useMCPBindings.ts new file mode 100644 index 00000000..a0c9631b --- /dev/null +++ b/app/src/lib/hooks/useMCPBindings.ts @@ -0,0 +1,60 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { apiClient } from '@/lib/api/client'; +import type { + MCPClientBindingListResponse, + MCPClientBindingUpsert, +} from '@/lib/api/types'; + +const MCP_BINDINGS_KEY = ['settings', 'mcp', 'bindings'] as const; + +/** Manage per-MCP-client voice bindings (Claude Code → Morgan, etc.). */ +export function useMCPBindings() { + const queryClient = useQueryClient(); + + const query = useQuery({ + queryKey: MCP_BINDINGS_KEY, + queryFn: () => apiClient.listMCPBindings(), + // Keep fresh while the Settings page is open — the ``last_seen_at`` + // timestamp is useful for confirming an install works, and we want it + // to tick forward when a client connects. + refetchInterval: 10_000, + }); + + const upsertMutation = useMutation({ + mutationFn: (data: MCPClientBindingUpsert) => + apiClient.upsertMCPBinding(data), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY }); + }, + }); + + const deleteMutation = useMutation({ + mutationFn: (clientId: string) => apiClient.deleteMCPBinding(clientId), + onMutate: async (clientId) => { + await queryClient.cancelQueries({ queryKey: MCP_BINDINGS_KEY }); + const prev = + queryClient.getQueryData(MCP_BINDINGS_KEY); + if (prev) { + queryClient.setQueryData( + MCP_BINDINGS_KEY, + { items: prev.items.filter((b) => b.client_id !== clientId) }, + ); + } + return { prev }; + }, + onError: (_err, _id, ctx) => { + if (ctx?.prev) queryClient.setQueryData(MCP_BINDINGS_KEY, ctx.prev); + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY }); + }, + }); + + return { + bindings: query.data?.items ?? [], + isLoading: query.isLoading, + upsert: upsertMutation.mutate, + upsertAsync: upsertMutation.mutateAsync, + remove: deleteMutation.mutate, + }; +} diff --git a/app/src/lib/hooks/useSpeakEvents.ts b/app/src/lib/hooks/useSpeakEvents.ts new file mode 100644 index 00000000..598786cf --- /dev/null +++ b/app/src/lib/hooks/useSpeakEvents.ts @@ -0,0 +1,95 @@ +import { useEffect, useRef, useState } from 'react'; +import { useServerStore } from '@/stores/serverStore'; + +/** Payload for a speak-start SSE event broadcast by the backend. */ +export interface ActiveSpeak { + generationId: string; + profileName: string; + source: 'mcp' | 'rest' | string; + clientId: string | null; + startedAt: number; + elapsedMs: number; +} + +/** + * Subscribes to `/events/speak` and reports whichever agent-initiated + * speak is currently producing audio. Returns ``null`` when nothing is + * speaking. + * + * Multiple concurrent speaks are rare (the model can only really do one + * at a time) and we don't bother stacking them — newest wins, the old + * one's speak-end will clear when it fires. + */ +export function useSpeakEvents(): ActiveSpeak | null { + const [active, setActive] = useState(null); + const activeRef = useRef(null); + activeRef.current = active; + + // Keep a live timer so the pill's elapsed label advances smoothly + // without re-opening the SSE stream. + const [, setTick] = useState(0); + + useEffect(() => { + if (!active) return; + const iv = window.setInterval(() => setTick((t) => t + 1), 250); + return () => window.clearInterval(iv); + }, [active]); + + useEffect(() => { + const baseUrl = useServerStore.getState().serverUrl; + if (!baseUrl) return; + + let cancelled = false; + let source: EventSource | null = null; + + const connect = () => { + if (cancelled) return; + source = new EventSource(`${baseUrl}/events/speak`); + + source.addEventListener('speak-start', (event: MessageEvent) => { + try { + const data = JSON.parse(event.data); + const now = Date.now(); + setActive({ + generationId: String(data.generation_id ?? ''), + profileName: String(data.profile_name ?? ''), + source: String(data.source ?? 'mcp'), + clientId: data.client_id ?? null, + startedAt: now, + elapsedMs: 0, + }); + } catch { + // malformed payload — ignore, don't crash the stream + } + }); + + source.addEventListener('speak-end', (event: MessageEvent) => { + try { + const data = JSON.parse(event.data); + const endedId = String(data.generation_id ?? ''); + // Only clear if this end matches the currently-active id; late + // ends from previous sessions are ignored. + if (activeRef.current?.generationId === endedId) setActive(null); + } catch { + // ignore + } + }); + + source.onerror = () => { + // EventSource auto-reconnects, but if the browser gives up we + // manually retry with backoff. + source?.close(); + if (!cancelled) window.setTimeout(connect, 2000); + }; + }; + + connect(); + return () => { + cancelled = true; + source?.close(); + }; + }, []); + + if (!active) return null; + return { ...active, elapsedMs: Date.now() - active.startedAt }; +} diff --git a/app/src/router.tsx b/app/src/router.tsx index 581a876c..940e7c52 100644 --- a/app/src/router.tsx +++ b/app/src/router.tsx @@ -17,6 +17,7 @@ import { GeneralPage } from '@/components/ServerTab/GeneralPage'; import { GenerationPage } from '@/components/ServerTab/GenerationPage'; import { GpuPage } from '@/components/ServerTab/GpuPage'; import { LogsPage } from '@/components/ServerTab/LogsPage'; +import { MCPPage } from '@/components/ServerTab/MCPPage'; import { SettingsLayout } from '@/components/ServerTab/ServerTab'; import { Sidebar } from '@/components/Sidebar'; import { StoriesTab } from '@/components/StoriesTab/StoriesTab'; @@ -159,6 +160,12 @@ const settingsCapturesRoute = createRoute({ component: CapturesPage, }); +const settingsMCPRoute = createRoute({ + getParentRoute: () => settingsRoute, + path: '/mcp', + component: MCPPage, +}); + const settingsGpuRoute = createRoute({ getParentRoute: () => settingsRoute, path: '/gpu', @@ -204,6 +211,7 @@ const routeTree = rootRoute.addChildren([ settingsGeneralRoute, settingsGenerationRoute, settingsCapturesRoute, + settingsMCPRoute, settingsGpuRoute, settingsLogsRoute, settingsChangelogRoute, diff --git a/backend/app.py b/backend/app.py index c9cf5965..98020562 100644 --- a/backend/app.py +++ b/backend/app.py @@ -4,6 +4,7 @@ import asyncio import logging import os import sys +from contextlib import asynccontextmanager from pathlib import Path @@ -68,15 +69,36 @@ def safe_content_disposition(disposition_type: str, filename: str) -> str: def create_app() -> FastAPI: """Create and configure the FastAPI application.""" + from .mcp_server.server import build_mcp_server + from .mcp_server.context import ClientIdMiddleware + + # Build the MCP app up-front so we can wire its lifespan into FastAPI's — + # FastMCP's Streamable HTTP transport only works if its session manager + # runs inside the parent ASGI lifespan. + mcp = build_mcp_server() + mcp_app = mcp.http_app(path="/", transport="http") + + @asynccontextmanager + async def lifespan(app: FastAPI): + await _run_startup(app) + async with mcp_app.router.lifespan_context(app): + try: + yield + finally: + await _run_shutdown() + application = FastAPI( title="voicebox API", description="Production-quality Qwen3-TTS voice cloning API", version=__version__, + lifespan=lifespan, ) _configure_cors(application) + application.add_middleware(ClientIdMiddleware) register_routers(application) - _register_lifecycle(application) + application.mount("/mcp", mcp_app) + logger.info("MCP: mounted at /mcp") _mount_frontend(application) return application @@ -179,107 +201,104 @@ def _get_gpu_status() -> str: return "None (CPU only)" -def _register_lifecycle(application: FastAPI) -> None: - """Attach startup and shutdown event handlers.""" +async def _run_startup(application: FastAPI) -> None: + """Database init, warnings, model-cache prep. Runs on lifespan entry.""" + import platform + import sys - @application.on_event("startup") - async def startup_event(): - import platform - import sys + logger.info("Voicebox v%s starting up", __version__) + logger.info( + "Python %s on %s %s (%s)", + sys.version.split()[0], + platform.system(), + platform.release(), + platform.machine(), + ) - logger.info("Voicebox v%s starting up", __version__) - logger.info( - "Python %s on %s %s (%s)", - sys.version.split()[0], - platform.system(), - platform.release(), - platform.machine(), - ) + database.init_db() - database.init_db() + from .database.session import _db_path - from .database.session import _db_path + logger.info("Database: %s", _db_path) + logger.info("Data directory: %s", config.get_data_dir()) - logger.info("Database: %s", _db_path) - logger.info("Data directory: %s", config.get_data_dir()) + init_queue() - init_queue() + # Mark stale "generating" records as failed -- leftovers from a killed process + from sqlalchemy import text as sa_text - # Mark stale "generating" records as failed -- leftovers from a killed process - from sqlalchemy import text as sa_text - - db = next(get_db()) - try: - result = db.execute( - sa_text( - "UPDATE generations SET status = 'failed', " - "error = 'Server was shut down during generation' " - "WHERE status IN ('generating', 'loading_model')" - ) + db = next(get_db()) + try: + result = db.execute( + sa_text( + "UPDATE generations SET status = 'failed', " + "error = 'Server was shut down during generation' " + "WHERE status IN ('generating', 'loading_model')" ) - if result.rowcount > 0: - logger.info("Marked %d stale generation(s) as failed", result.rowcount) + ) + if result.rowcount > 0: + logger.info("Marked %d stale generation(s) as failed", result.rowcount) - from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration + from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration - profile_count = db.query(DBVoiceProfile).count() - generation_count = db.query(DBGeneration).count() - logger.info("Profiles: %d, Generations: %d", profile_count, generation_count) + profile_count = db.query(DBVoiceProfile).count() + generation_count = db.query(DBGeneration).count() + logger.info("Profiles: %d, Generations: %d", profile_count, generation_count) - db.commit() - except Exception as e: - db.rollback() - logger.warning("Could not clean up stale generations: %s", e) - finally: - db.close() + db.commit() + except Exception as e: + db.rollback() + logger.warning("Could not clean up stale generations: %s", e) + finally: + db.close() - backend_type = get_backend_type() - logger.info("Backend: %s", backend_type.upper()) - logger.info("GPU: %s", _get_gpu_status()) + backend_type = get_backend_type() + logger.info("Backend: %s", backend_type.upper()) + logger.info("GPU: %s", _get_gpu_status()) - # Warn if GPU architecture is not supported by this PyTorch build - from .backends.base import check_cuda_compatibility + from .backends.base import check_cuda_compatibility - _compatible, _cuda_warning = check_cuda_compatibility() - if not _compatible: - logger.warning("GPU COMPATIBILITY: %s", _cuda_warning) + _compatible, _cuda_warning = check_cuda_compatibility() + if not _compatible: + logger.warning("GPU COMPATIBILITY: %s", _cuda_warning) - from .services.cuda import check_and_update_cuda_binary + from .services.cuda import check_and_update_cuda_binary - create_background_task(check_and_update_cuda_binary()) + create_background_task(check_and_update_cuda_binary()) - try: - progress_manager = get_progress_manager() - progress_manager._set_main_loop(asyncio.get_running_loop()) - except Exception as e: - logger.warning("Could not initialize progress manager event loop: %s", e) + try: + progress_manager = get_progress_manager() + progress_manager._set_main_loop(asyncio.get_running_loop()) + except Exception as e: + logger.warning("Could not initialize progress manager event loop: %s", e) - try: - from huggingface_hub import constants as hf_constants + try: + from huggingface_hub import constants as hf_constants - cache_dir = Path(hf_constants.HF_HUB_CACHE) - cache_dir.mkdir(parents=True, exist_ok=True) - logger.info("Model cache: %s", cache_dir) - except Exception as e: - logger.warning("Could not create HuggingFace cache directory: %s", e) + cache_dir = Path(hf_constants.HF_HUB_CACHE) + cache_dir.mkdir(parents=True, exist_ok=True) + logger.info("Model cache: %s", cache_dir) + except Exception as e: + logger.warning("Could not create HuggingFace cache directory: %s", e) - logger.info("Ready") + logger.info("Ready") - @application.on_event("shutdown") - async def shutdown_event(): - logger.info("Voicebox server shutting down...") - try: - tts.unload_tts_model() - except Exception: - logger.exception("Failed to unload TTS model") - try: - transcribe.unload_whisper_model() - except Exception: - logger.exception("Failed to unload Whisper model") - try: - llm.unload_llm_model() - except Exception: - logger.exception("Failed to unload LLM model") + +async def _run_shutdown() -> None: + """Unload models on lifespan exit.""" + logger.info("Voicebox server shutting down...") + try: + tts.unload_tts_model() + except Exception: + logger.exception("Failed to unload TTS model") + try: + transcribe.unload_whisper_model() + except Exception: + logger.exception("Failed to unload Whisper model") + try: + llm.unload_llm_model() + except Exception: + logger.exception("Failed to unload LLM model") app = create_app() diff --git a/backend/build_binary.py b/backend/build_binary.py index 52bacbfe..278b0473 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -295,6 +295,28 @@ def build_server(cuda=False): "unidic_lite", "--hidden-import", "loguru", + # MCP server — Streamable-HTTP endpoint and the 4 voicebox.* tools. + # FastMCP pulls in a chain of deps (mcp, cyclopts, openapi-pydantic, + # etc.) that don't auto-discover cleanly under PyInstaller, so we + # collect them whole. Small compared to torch. + "--hidden-import", + "backend.mcp_server", + "--hidden-import", + "backend.mcp_server.server", + "--hidden-import", + "backend.mcp_server.tools", + "--hidden-import", + "backend.mcp_server.context", + "--hidden-import", + "backend.mcp_server.resolve", + "--hidden-import", + "backend.mcp_server.events", + "--collect-all", + "fastmcp", + "--collect-all", + "mcp", + "--hidden-import", + "sse_starlette", ] ) @@ -447,12 +469,108 @@ def build_server(cuda=False): logger.info("Binary built in %s", backend_dir / "dist" / binary_name) +def build_shim(): + """Build the voicebox-mcp stdio shim as a tiny standalone binary. + + This is the bridge for MCP clients that only speak stdio — it proxies + JSON-RPC to the main voicebox-server's /mcp endpoint. Keep it small: no + torch, no ML deps, just httpx + asyncio. + """ + backend_dir = Path(__file__).parent + + args = [ + "mcp_shim/__main__.py", + "--onefile", + "--name", + "voicebox-mcp", + # Stdio-only — no console hiding needed on Windows since the parent + # MCP client is spawning this as a child process and wants stdio. + "--hidden-import", + "backend.mcp_shim", + "--hidden-import", + "backend.mcp_shim.__main__", + "--hidden-import", + "httpx", + "--hidden-import", + "httpx._transports.default", + "--hidden-import", + "anyio", + # Exclude everything heavy that httpx/asyncio don't actually need so + # the binary stays tiny (~15 MB instead of ~400 MB). + "--exclude-module", + "torch", + "--exclude-module", + "transformers", + "--exclude-module", + "mlx", + "--exclude-module", + "mlx_audio", + "--exclude-module", + "qwen_tts", + "--exclude-module", + "chatterbox", + "--exclude-module", + "zipvoice", + "--exclude-module", + "tada", + "--exclude-module", + "kokoro", + "--exclude-module", + "misaki", + "--exclude-module", + "spacy", + "--exclude-module", + "librosa", + "--exclude-module", + "numba", + "--exclude-module", + "numpy", + "--exclude-module", + "pedalboard", + "--exclude-module", + "fastapi", + "--exclude-module", + "uvicorn", + "--exclude-module", + "sqlalchemy", + "--exclude-module", + "fastmcp", + "--exclude-module", + "mcp", + ] + + dist_dir = str(backend_dir / "dist") + build_dir = str(backend_dir / "build") + args.extend( + [ + "--distpath", + dist_dir, + "--workpath", + build_dir, + "--noconfirm", + "--clean", + ] + ) + + os.chdir(backend_dir) + PyInstaller.__main__.run(args) + logger.info("Shim built: %s", backend_dir / "dist" / "voicebox-mcp") + + if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Build voicebox-server binary") + parser = argparse.ArgumentParser(description="Build voicebox binaries") parser.add_argument( "--cuda", action="store_true", help="Build CUDA-enabled binary (voicebox-server-cuda)", ) + parser.add_argument( + "--shim", + action="store_true", + help="Build the voicebox-mcp stdio shim binary instead of the server", + ) cli_args = parser.parse_args() - build_server(cuda=cli_args.cuda) + if cli_args.shim: + build_shim() + else: + build_server(cuda=cli_args.cuda) diff --git a/backend/database/__init__.py b/backend/database/__init__.py index 3b68baee..bfb4b124 100644 --- a/backend/database/__init__.py +++ b/backend/database/__init__.py @@ -15,6 +15,7 @@ from .models import ( Generation, GenerationSettings, GenerationVersion, + MCPClientBinding, ProfileChannelMapping, ProfileSample, Project, @@ -35,6 +36,7 @@ __all__ = [ "Generation", "GenerationSettings", "GenerationVersion", + "MCPClientBinding", "ProfileChannelMapping", "ProfileSample", "Project", diff --git a/backend/database/migrations.py b/backend/database/migrations.py index 00bb9447..c4b2b48a 100644 --- a/backend/database/migrations.py +++ b/backend/database/migrations.py @@ -224,6 +224,13 @@ def _migrate_capture_settings(engine, inspector, tables: set[str]) -> None: "chord_toggle_to_talk_keys TEXT NOT NULL DEFAULT '[\"MetaRight\",\"AltGr\",\"Space\"]'", "chord_toggle_to_talk_keys", ) + if "hotkey_enabled" not in columns: + _add_column( + engine, + "capture_settings", + "hotkey_enabled BOOLEAN NOT NULL DEFAULT 0", + "hotkey_enabled", + ) def _normalize_storage_paths(engine, tables: set[str]) -> None: diff --git a/backend/database/models.py b/backend/database/models.py index 2aad301b..277468b4 100644 --- a/backend/database/models.py +++ b/backend/database/models.py @@ -196,6 +196,12 @@ class CaptureSettings(Base): preserve_technical = Column(Boolean, nullable=False, default=True) allow_auto_paste = Column(Boolean, nullable=False, default=True) default_playback_voice_id = Column(String, nullable=True) + # Default OFF — opting in is what triggers the macOS Input Monitoring TCC + # prompt. We deliberately don't spawn the global keyboard tap until the + # user flips this on so a fresh-install user doesn't see a scary + # "Voicebox would like to receive keystrokes from any application" dialog + # before they've even opened the Captures tab. + hotkey_enabled = Column(Boolean, nullable=False, default=False) # Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand # modifiers by default so they don't collide with left-hand system # shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit). @@ -221,6 +227,29 @@ class GenerationSettings(Base): updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) +class MCPClientBinding(Base): + """Per-MCP-client settings (voice profile, engine, intent). + + Lets users bind distinct voices to distinct agents — e.g. Claude Code + speaks in "Morgan," Cursor in "Scarlett." The MCP client identifies + itself via the ``X-Voicebox-Client-Id`` HTTP header; direct-HTTP + clients set it in their MCP config's ``headers`` block, the stdio + shim forwards it from the ``VOICEBOX_CLIENT_ID`` env var. + """ + + __tablename__ = "mcp_client_bindings" + + client_id = Column(String, primary_key=True) + label = Column(String, nullable=True) # display name + profile_id = Column(String, ForeignKey("profiles.id"), nullable=True) + default_engine = Column(String, nullable=True) + # "respond" | "rewrite" | "compose" — null means plain TTS (no LLM transform). + default_intent = Column(String, nullable=True) + last_seen_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + class Capture(Base): """A single voice input capture (dictation, recording, or uploaded file). diff --git a/backend/mcp_server/README.md b/backend/mcp_server/README.md new file mode 100644 index 00000000..4c9b426f --- /dev/null +++ b/backend/mcp_server/README.md @@ -0,0 +1,103 @@ +# Voicebox MCP server + +Local **Model Context Protocol** server — lets any MCP-aware agent +(Claude Code, Cursor, Windsurf, VS Code MCP extensions, etc.) speak text +in your cloned voices, transcribe audio, and browse captures. + +The server runs inside the same `uvicorn` process as the rest of Voicebox +and is mounted at `/mcp` (Streamable HTTP transport). + +## Install into your agent + +Preferred — direct HTTP: + +```json +{ + "mcpServers": { + "voicebox": { + "url": "http://127.0.0.1:17493/mcp", + "headers": { "X-Voicebox-Client-Id": "claude-code" } + } + } +} +``` + +Fallback — stdio shim (when the client doesn't speak HTTP MCP). The +`voicebox-mcp` binary ships inside the Voicebox.app bundle: + +```json +{ + "mcpServers": { + "voicebox": { + "command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp", + "env": { "VOICEBOX_CLIENT_ID": "claude-code" } + } + } +} +``` + +Claude Code one-liner: + +``` +claude mcp add voicebox \ + --transport http \ + --url http://127.0.0.1:17493/mcp \ + --header "X-Voicebox-Client-Id: claude-code" +``` + +## Tools + +| Name | Purpose | +|---|---| +| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. | +| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. | +| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. | +| `voicebox.list_profiles` | Available voice profiles (cloned + preset). | + +All tools resolve voice profiles in this precedence: + +1. Explicit `profile` arg (name or id — case-insensitive) +2. Per-client binding keyed by `X-Voicebox-Client-Id` +3. `capture_settings.default_playback_voice_id` (global default) + +Bindings are managed via `GET|PUT /mcp/bindings` or in the app under +Settings → MCP. + +## Debug with MCP Inspector + +``` +npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp +``` + +Point it at the URL, hit "List tools," call `voicebox.list_profiles` +first to confirm wiring, then `voicebox.speak` for end-to-end. + +## Non-MCP REST surface + +`POST /speak` is a thin wrapper on the same code path for callers that +don't speak MCP (shell scripts, ACP, A2A): + +``` +curl -X POST http://127.0.0.1:17493/speak \ + -H 'Content-Type: application/json' \ + -H 'X-Voicebox-Client-Id: claude-code' \ + -d '{"text":"Build complete.","profile":"Morgan"}' +``` + +## Code layout + +``` +backend/mcp_server/ +├── __init__.py # re-export mount_into +├── server.py # build_mcp_server() + mount_into(app) +├── tools.py # @mcp.tool() implementations +├── context.py # ClientIdMiddleware + current_client_id ContextVar +├── resolve.py # profile resolution precedence +├── events.py # pub/sub queue for /events/speak pill SSE +└── README.md # you are here + +backend/mcp_shim/ # stdio ↔ Streamable-HTTP proxy (see its README) +``` + +The package is **`mcp_server`**, not `mcp`, to avoid shadowing the +installed `mcp` PyPI package that FastMCP imports internally. diff --git a/backend/mcp_server/__init__.py b/backend/mcp_server/__init__.py new file mode 100644 index 00000000..6dcd20c3 --- /dev/null +++ b/backend/mcp_server/__init__.py @@ -0,0 +1,10 @@ +"""Model Context Protocol server — exposes Voicebox tools to local AI agents. + +Mounts a FastMCP instance at /mcp on the main FastAPI app (Streamable HTTP). +A bundled stdio shim (backend/mcp_shim) forwards JSON-RPC into the same +endpoint for MCP clients that only speak stdio. +""" + +from .server import mount_into + +__all__ = ["mount_into"] diff --git a/backend/mcp_server/context.py b/backend/mcp_server/context.py new file mode 100644 index 00000000..20809a77 --- /dev/null +++ b/backend/mcp_server/context.py @@ -0,0 +1,83 @@ +"""Per-request client identity for MCP calls. + +MCP clients identify themselves via an ``X-Voicebox-Client-Id`` HTTP header +(direct-HTTP clients set it in their MCP config; the stdio shim forwards it +from the ``VOICEBOX_CLIENT_ID`` env var). Middleware copies the value into a +ContextVar so tool implementations can read it without plumbing the request +object through every service call. +""" + +import logging +from contextvars import ContextVar +from datetime import datetime + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import ASGIApp + + +logger = logging.getLogger(__name__) + +CLIENT_ID_HEADER = "X-Voicebox-Client-Id" + +# Tool handlers read this to apply per-client voice bindings. +current_client_id: ContextVar[str | None] = ContextVar( + "current_client_id", default=None +) + + +class ClientIdMiddleware(BaseHTTPMiddleware): + """Copy X-Voicebox-Client-Id into a ContextVar and stamp last_seen_at. + + Only stamps on MCP-endpoint requests (anything under ``/mcp``) so + unrelated REST traffic with the header set won't advance the + last-seen timestamp — the Settings UI uses that to show when each + client was last heard from. + """ + + def __init__(self, app: ASGIApp) -> None: + super().__init__(app) + + async def dispatch(self, request: Request, call_next) -> Response: + client_id = request.headers.get(CLIENT_ID_HEADER) + token = current_client_id.set(client_id) + try: + response = await call_next(request) + finally: + current_client_id.reset(token) + + if client_id and request.url.path.startswith("/mcp"): + _stamp_last_seen(client_id) + return response + + +def _stamp_last_seen(client_id: str) -> None: + """Update or create the MCPClientBinding row for this client_id.""" + try: + from ..database import get_db + from ..database.models import MCPClientBinding + except Exception: + return + try: + db = next(get_db()) + except Exception: + return + try: + row = ( + db.query(MCPClientBinding) + .filter(MCPClientBinding.client_id == client_id) + .first() + ) + if row is None: + row = MCPClientBinding(client_id=client_id) + db.add(row) + row.last_seen_at = datetime.utcnow() + db.commit() + except Exception: + logger.debug( + "Could not stamp last_seen_at for %s", client_id, exc_info=True + ) + db.rollback() + finally: + db.close() diff --git a/backend/mcp_server/events.py b/backend/mcp_server/events.py new file mode 100644 index 00000000..a8c944b5 --- /dev/null +++ b/backend/mcp_server/events.py @@ -0,0 +1,35 @@ +"""In-memory pub/sub for speaking-pill SSE broadcasts. + +MCP ``voicebox.speak`` calls and the REST ``POST /speak`` route publish +start/end events that DictateWindow subscribes to via /events/speak, so the +floating pill surfaces whenever an agent is speaking. +""" + +import asyncio +from typing import Any + + +# Each subscriber gets its own queue. Bounded to drop oldest if a client lags. +_subscribers: set[asyncio.Queue[dict[str, Any]]] = set() + + +def subscribe() -> asyncio.Queue[dict[str, Any]]: + """Register a new subscriber; caller must call unsubscribe() when done.""" + queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=64) + _subscribers.add(queue) + return queue + + +def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None: + _subscribers.discard(queue) + + +def publish(kind: str, payload: dict[str, Any]) -> None: + """Fan out to all current subscribers. Non-blocking; drops on full queue.""" + event = {"kind": kind, **payload} + for queue in list(_subscribers): + try: + queue.put_nowait(event) + except asyncio.QueueFull: + # Slow subscriber — skip rather than block publishers. + pass diff --git a/backend/mcp_server/resolve.py b/backend/mcp_server/resolve.py new file mode 100644 index 00000000..bd61e4c3 --- /dev/null +++ b/backend/mcp_server/resolve.py @@ -0,0 +1,57 @@ +"""Voice profile resolution for MCP tool calls. + +Precedence: + 1. Explicit tool arg (profile name or id) + 2. Per-client MCPClientBinding.profile_id + 3. CaptureSettings.default_playback_voice_id (global default) + 4. None — caller raises a helpful error +""" + +from sqlalchemy.orm import Session + +from ..database import VoiceProfile as DBVoiceProfile, get_db +from ..database.models import CaptureSettings +from ..services.profiles import get_profile_orm_by_name_or_id as _lookup_profile + + +def resolve_profile( + explicit: str | None, + client_id: str | None, + db: Session, +) -> DBVoiceProfile | None: + """Apply the full precedence chain and return the profile ORM row (or None).""" + if explicit: + profile = _lookup_profile(explicit, db) + if profile is not None: + return profile + # Explicit but not found — return None so the caller can report it. + return None + + if client_id: + # Per-client binding. Imported lazily so this module stays importable + # even before the migration adds the table on first boot. + from ..database.models import MCPClientBinding # noqa: WPS433 + + binding = ( + db.query(MCPClientBinding) + .filter(MCPClientBinding.client_id == client_id) + .first() + ) + if binding and binding.profile_id: + profile = _lookup_profile(binding.profile_id, db) + if profile is not None: + return profile + + # Global default from capture settings. + settings = db.query(CaptureSettings).filter(CaptureSettings.id == 1).first() + if settings and settings.default_playback_voice_id: + profile = _lookup_profile(settings.default_playback_voice_id, db) + if profile is not None: + return profile + + return None + + +def with_db() -> Session: + """Utility for tool handlers that aren't managed by FastAPI's Depends.""" + return next(get_db()) diff --git a/backend/mcp_server/server.py b/backend/mcp_server/server.py new file mode 100644 index 00000000..3434f7c4 --- /dev/null +++ b/backend/mcp_server/server.py @@ -0,0 +1,79 @@ +"""Construct the FastMCP server and mount it on the FastAPI app. + +The MCP endpoint lives at ``/mcp`` (Streamable HTTP transport). Modern MCP +clients (Claude Code, Cursor, Windsurf, VS Code MCP extensions) connect +directly via URL; older stdio-only clients use the ``voicebox-mcp`` shim +binary bundled with the desktop app. +""" + +from __future__ import annotations + +import logging +from contextlib import AsyncExitStack, asynccontextmanager +from typing import Callable + +from fastapi import FastAPI +from fastmcp import FastMCP + +from .context import ClientIdMiddleware +from .tools import register_tools + + +logger = logging.getLogger(__name__) + + +def build_mcp_server() -> FastMCP: + """Create the FastMCP instance with Voicebox tools registered.""" + mcp = FastMCP( + name="voicebox", + instructions=( + "Voicebox is a local voice I/O layer. Use `voicebox.speak` to " + "play text in a voice profile, `voicebox.transcribe` for " + "audio→text, and the `list_*` tools to discover profiles and " + "captures." + ), + ) + register_tools(mcp) + return mcp + + +def mount_into( + app: FastAPI, + *, + extra_startup: Callable[[], None] | None = None, +) -> None: + """Attach the MCP app to ``app`` at ``/mcp`` and install the client-id middleware. + + ``extra_startup`` — if provided, runs during the FastAPI lifespan. This + is the hook that lets ``app.py`` keep its existing startup/shutdown + bodies while also driving FastMCP's session manager. + """ + mcp = build_mcp_server() + mcp_app = mcp.http_app(path="/", transport="http") + + # ClientIdMiddleware must run before FastMCP so the ContextVar is set + # by the time tool handlers execute. Starlette composes middlewares + # outermost-first, so adding here on the parent app is correct. + app.add_middleware(ClientIdMiddleware) + app.mount("/mcp", mcp_app) + app.state.mcp_lifespan = mcp_app.router.lifespan_context + logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", "")) + + +def compose_lifespan(*lifespans): + """Combine multiple async context managers into a single FastAPI lifespan. + + Used by ``create_app`` to run the existing Voicebox startup/shutdown + together with FastMCP's session manager (which MUST run in the + ASGI lifespan for Streamable HTTP to work). + """ + + @asynccontextmanager + async def _combined(app): + async with AsyncExitStack() as stack: + for cm_factory in lifespans: + cm = cm_factory(app) if callable(cm_factory) else cm_factory + await stack.enter_async_context(cm) + yield + + return _combined diff --git a/backend/mcp_server/tools.py b/backend/mcp_server/tools.py new file mode 100644 index 00000000..e617a0c4 --- /dev/null +++ b/backend/mcp_server/tools.py @@ -0,0 +1,321 @@ +"""Voicebox MCP tool implementations. + +Thin wrappers over existing services/routes. Tools are registered with dotted +names (``voicebox.speak`` etc.) so they look natural in agent logs — +the Python function name stays snake_case. +""" + +from __future__ import annotations + +import asyncio +import base64 as b64 +import logging +import tempfile +from pathlib import Path +from typing import Any, Literal + +from fastmcp import FastMCP + +from .. import models +from ..database import get_db +from ..services import captures as captures_service +from ..services import profiles as profiles_service +from . import events as mcp_events +from .context import current_client_id +from .resolve import resolve_profile + + +logger = logging.getLogger(__name__) + +# Absolute-path transcribes are bounded to keep a bad client from +# asking us to ingest a 20 GB file. +MAX_TRANSCRIBE_BYTES = 200 * 1024 * 1024 # 200 MB + + +def register_tools(mcp: FastMCP) -> None: + """Attach all Voicebox tools to the given FastMCP instance.""" + + @mcp.tool( + name="voicebox.speak", + description=( + "Speak text in a Voicebox voice profile. Returns a generation id " + "the caller can poll at /generate/{id}/status. Audio plays on the " + "user's speakers and is saved to the Captures / History tab." + ), + ) + async def voicebox_speak( + text: str, + profile: str | None = None, + engine: str | None = None, + intent: Literal["respond", "rewrite", "compose"] | None = None, + language: str | None = None, + ) -> dict[str, Any]: + """Speak ``text`` in a voice profile. + + ``profile`` accepts a voice profile name (e.g. "Morgan") or id. If + omitted, the server looks up the per-client binding for the calling + MCP client, then falls back to the global default voice. + + ``intent`` only matters for profiles that have a personality prompt — + when set, the text is first transformed by the LLM (respond to it, + rewrite it in character, or compose a fresh utterance). Leave unset + for plain TTS. + """ + db = next(get_db()) + try: + vp = resolve_profile(profile, current_client_id.get(), db) + if vp is None: + raise ValueError( + "No voice profile resolved. Pass `profile=` with a " + "voice profile name or id, or set a default voice in " + "Voicebox → Settings → MCP." + ) + + # Persona path if intent requested and personality present. + if intent is not None and vp.personality: + return await _speak_with_persona( + profile_id=vp.id, + profile_name=vp.name, + text=text, + engine=engine, + intent=intent, + language=language, + db=db, + ) + + return await _speak_plain( + profile_id=vp.id, + profile_name=vp.name, + text=text, + engine=engine, + language=language, + db=db, + ) + finally: + db.close() + + @mcp.tool( + name="voicebox.transcribe", + description=( + "Transcribe an audio clip to text using Voicebox's local Whisper. " + "Pass exactly one of `audio_base64` (bytes as base64) or " + "`audio_path` (absolute local file path)." + ), + ) + async def voicebox_transcribe( + audio_base64: str | None = None, + audio_path: str | None = None, + language: str | None = None, + model: str | None = None, + ) -> dict[str, Any]: + if bool(audio_base64) == bool(audio_path): + raise ValueError( + "Pass exactly one of `audio_base64` or `audio_path`." + ) + + # Absolute-path mode: validate and transcribe in place. + if audio_path is not None: + path = Path(audio_path) + if not path.is_absolute(): + raise ValueError("`audio_path` must be absolute.") + if not path.is_file(): + raise ValueError(f"File not found: {audio_path}") + if path.stat().st_size > MAX_TRANSCRIBE_BYTES: + raise ValueError( + f"File exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit." + ) + return await _transcribe_file(path, language, model) + + # Base64 mode: decode into a temp file, transcribe, clean up. + try: + raw = b64.b64decode(audio_base64, validate=True) + except Exception as exc: + raise ValueError(f"Invalid audio_base64: {exc}") from exc + if len(raw) > MAX_TRANSCRIBE_BYTES: + raise ValueError( + f"Audio exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit." + ) + with tempfile.NamedTemporaryFile( + suffix=".wav", delete=False + ) as tmp: + tmp.write(raw) + tmp_path = Path(tmp.name) + try: + return await _transcribe_file(tmp_path, language, model) + finally: + tmp_path.unlink(missing_ok=True) + + @mcp.tool( + name="voicebox.list_captures", + description=( + "List recent voice captures (dictations, recordings, uploads) " + "with their transcripts. Most-recent first." + ), + ) + async def voicebox_list_captures( + limit: int = 20, offset: int = 0 + ) -> dict[str, Any]: + if not (1 <= limit <= 200): + raise ValueError("`limit` must be between 1 and 200.") + if offset < 0: + raise ValueError("`offset` must be >= 0.") + db = next(get_db()) + try: + items, total = captures_service.list_captures( + db, limit=limit, offset=offset + ) + return { + "captures": [ + item.model_dump(mode="json") for item in items + ], + "total": total, + } + finally: + db.close() + + @mcp.tool( + name="voicebox.list_profiles", + description=( + "List available voice profiles (both cloned voices and presets). " + "Use the returned `name` with voicebox.speak(profile=...)." + ), + ) + async def voicebox_list_profiles() -> dict[str, Any]: + db = next(get_db()) + try: + profiles = await profiles_service.list_profiles(db) + return { + "profiles": [ + { + "id": p.id, + "name": p.name, + "voice_type": p.voice_type, + "language": p.language, + "has_personality": bool(getattr(p, "personality", None)), + } + for p in profiles + ] + } + finally: + db.close() + + +# ─── Speak helpers ───────────────────────────────────────────────────────── + + +async def _speak_plain( + *, + profile_id: str, + profile_name: str, + text: str, + engine: str | None, + language: str | None, + db, +) -> dict[str, Any]: + """Plain TTS path — mirrors POST /generate. No LLM transform.""" + from ..routes.generations import generate_speech + + req = models.GenerationRequest( + profile_id=profile_id, + text=text, + language=language or "en", + engine=engine or "qwen", + ) + generation = await generate_speech(req, db) + return _speak_response(generation, profile_name, source="mcp") + + +async def _speak_with_persona( + *, + profile_id: str, + profile_name: str, + text: str, + engine: str | None, + intent: str, + language: str | None, + db, +) -> dict[str, Any]: + """LLM-transformed path — reuses POST /profiles/{id}/speak.""" + from ..routes.profiles import speak_in_character + + req = models.PersonalitySpeakRequest( + text=text, + persist=True, + language=language, + engine=engine, + intent=intent, + ) + generation = await speak_in_character(profile_id, req, db) + return _speak_response(generation, profile_name, source="mcp") + + +def _speak_response( + generation, profile_name: str, *, source: str +) -> dict[str, Any]: + """Normalize a GenerationResponse into the MCP tool's return shape. + + Also fires a speak-start event so the DictateWindow pill surfaces + the agent's speech. Speak-end is fired from run_generation's + completion hook. + """ + payload = generation.model_dump(mode="json") if hasattr( + generation, "model_dump" + ) else dict(generation) + generation_id = payload.get("id") + mcp_events.publish( + "speak-start", + { + "generation_id": generation_id, + "profile_name": profile_name, + "source": source, + "client_id": current_client_id.get(), + }, + ) + return { + "generation_id": generation_id, + "status": payload.get("status"), + "profile": profile_name, + "source": source, + "poll_url": f"/generate/{generation_id}/status" + if generation_id + else None, + } + + +# ─── Transcribe helper ───────────────────────────────────────────────────── + + +async def _transcribe_file( + path: Path, language: str | None, model: str | None +) -> dict[str, Any]: + from ..backends import WHISPER_HF_REPOS + from ..services import transcribe as transcribe_service + from ..utils.audio import load_audio + + whisper = transcribe_service.get_whisper_model() + model_size = model or whisper.model_size + valid = list(WHISPER_HF_REPOS.keys()) + if model_size not in valid: + raise ValueError( + f"Invalid STT model '{model_size}'. Must be one of: {', '.join(valid)}" + ) + + # load_audio is sync; keep the event loop responsive. + audio, sr = await asyncio.to_thread(load_audio, str(path)) + duration = len(audio) / sr + + if ( + not whisper.is_loaded() or whisper.model_size != model_size + ) and not whisper._is_model_cached(model_size): + raise ValueError( + f"Whisper model '{model_size}' is not yet downloaded. Open " + "Voicebox → Settings → Models to download it first." + ) + + text = await whisper.transcribe(str(path), language, model_size) + return { + "text": text, + "duration": duration, + "language": language, + "model": model_size, + } diff --git a/backend/mcp_shim/__init__.py b/backend/mcp_shim/__init__.py new file mode 100644 index 00000000..0ee8555e --- /dev/null +++ b/backend/mcp_shim/__init__.py @@ -0,0 +1,10 @@ +"""Stdio → Streamable HTTP bridge for the Voicebox MCP server. + +Some MCP clients only know how to spawn a subprocess and talk to it over +stdin/stdout (the "stdio" transport). This package is a ~150-line adapter: +the client spawns us as ``voicebox-mcp``; we proxy every JSON-RPC frame +to http://127.0.0.1:17493/mcp/ and stream responses back out. + +All the real work (tools, models, inference) lives in the Voicebox server +process — this package contains no business logic. +""" diff --git a/backend/mcp_shim/__main__.py b/backend/mcp_shim/__main__.py new file mode 100644 index 00000000..dcd0ed1f --- /dev/null +++ b/backend/mcp_shim/__main__.py @@ -0,0 +1,197 @@ +"""voicebox-mcp — stdio ↔ Streamable-HTTP MCP proxy. + +Some MCP clients only speak stdio. They spawn this binary, we pipe each +JSON-RPC message to ``http://127.0.0.1:/mcp/``, and stream the +server's response back. The Voicebox server does all the real work. + +Environment variables: + VOICEBOX_PORT Voicebox server port (default 17493). + VOICEBOX_HOST Host (default 127.0.0.1). + VOICEBOX_CLIENT_ID Forwarded as X-Voicebox-Client-Id on every request. + +Stdout is JSON-RPC only. Diagnostics go to stderr. +Exit 0 on clean EOF, 1 on transport error, 2 if backend never answers. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from typing import Any + +import httpx + + +CLIENT_ID_HEADER = "X-Voicebox-Client-Id" +SESSION_HEADER = "mcp-session-id" +HEALTH_TIMEOUT_S = 30.0 +DEFAULT_PORT = 17493 + + +def _err(msg: str) -> None: + print(f"voicebox-mcp: {msg}", file=sys.stderr, flush=True) + + +def _base_url() -> tuple[str, str]: + host = os.environ.get("VOICEBOX_HOST", "127.0.0.1") + port = int(os.environ.get("VOICEBOX_PORT", str(DEFAULT_PORT))) + return f"http://{host}:{port}/mcp/", f"http://{host}:{port}/health" + + +async def _wait_for_backend(client: httpx.AsyncClient, health_url: str) -> bool: + loop = asyncio.get_running_loop() + deadline = loop.time() + HEALTH_TIMEOUT_S + while loop.time() < deadline: + try: + r = await client.get(health_url, timeout=2.0) + if r.status_code == 200: + return True + except Exception: + pass + await asyncio.sleep(0.5) + return False + + +async def _read_stdin_line() -> str | None: + """Async-read a single line from stdin. Returns None on EOF.""" + loop = asyncio.get_running_loop() + line = await loop.run_in_executor(None, sys.stdin.readline) + if not line: + return None + return line + + +def _write_stdout(obj: Any) -> None: + """Write a JSON object to stdout as one line, flushed.""" + sys.stdout.write(json.dumps(obj, separators=(",", ":"))) + sys.stdout.write("\n") + sys.stdout.flush() + + +async def _handle_request( + client: httpx.AsyncClient, + url: str, + raw: str, + headers: dict[str, str], + session_id: list[str | None], +) -> None: + """Forward one JSON-RPC payload to the server and relay the response.""" + try: + message = json.loads(raw) + except json.JSONDecodeError as exc: + _err(f"invalid JSON on stdin: {exc}") + return + + req_headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + **headers, + } + if session_id[0]: + req_headers[SESSION_HEADER] = session_id[0] + + # Notifications (no "id") don't expect a response body. Server returns + # 202 Accepted and we stay quiet. + is_notification = isinstance(message, dict) and "id" not in message + + async with client.stream( + "POST", url, headers=req_headers, content=raw.encode("utf-8") + ) as response: + # Capture session id on initialize. + if session_id[0] is None: + sid = response.headers.get(SESSION_HEADER) + if sid: + session_id[0] = sid + + if response.status_code == 202: + return # notification acknowledged + if response.status_code >= 400: + body = await response.aread() + _err( + f"server {response.status_code}: " + f"{body.decode('utf-8', errors='replace')[:400]}" + ) + if is_notification: + return + _write_stdout( + { + "jsonrpc": "2.0", + "id": message.get("id"), + "error": { + "code": -32000, + "message": ( + f"Voicebox MCP proxy got HTTP {response.status_code}" + ), + }, + } + ) + return + + ctype = response.headers.get("content-type", "") + if "text/event-stream" in ctype: + # SSE frames: lines prefixed "data: ..." contain the JSON-RPC msg. + async for line in response.aiter_lines(): + if line.startswith("data:"): + payload = line[5:].strip() + if not payload: + continue + try: + _write_stdout(json.loads(payload)) + except json.JSONDecodeError: + _err(f"malformed SSE payload: {payload[:200]}") + else: + body = await response.aread() + try: + _write_stdout(json.loads(body)) + except json.JSONDecodeError: + _err( + f"non-JSON response ({ctype}): " + f"{body.decode('utf-8', errors='replace')[:200]}" + ) + + +async def _run() -> int: + url, health_url = _base_url() + forward_headers: dict[str, str] = {} + client_id = os.environ.get("VOICEBOX_CLIENT_ID") + if client_id: + forward_headers[CLIENT_ID_HEADER] = client_id + + session_id: list[str | None] = [None] + + async with httpx.AsyncClient(timeout=httpx.Timeout(300.0)) as client: + if not await _wait_for_backend(client, health_url): + _err( + f"timed out waiting for Voicebox at {health_url} — is the app open?" + ) + return 2 + + try: + while True: + line = await _read_stdin_line() + if line is None: + return 0 + line = line.strip() + if not line: + continue + await _handle_request( + client, url, line, forward_headers, session_id + ) + except (KeyboardInterrupt, SystemExit): + return 0 + except Exception as exc: + _err(f"proxy failed: {exc!r}") + return 1 + + +def main() -> int: + try: + return asyncio.run(_run()) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/models.py b/backend/models.py index cba3a36f..386a3ede 100644 --- a/backend/models.py +++ b/backend/models.py @@ -248,6 +248,7 @@ class CaptureSettingsResponse(BaseModel): preserve_technical: bool = True allow_auto_paste: bool = True default_playback_voice_id: Optional[str] = None + hotkey_enabled: bool = False chord_push_to_talk_keys: List[str] = Field(default_factory=lambda: ["MetaRight", "AltGr"]) chord_toggle_to_talk_keys: List[str] = Field( default_factory=lambda: ["MetaRight", "AltGr", "Space"] @@ -269,6 +270,7 @@ class CaptureSettingsUpdate(BaseModel): preserve_technical: Optional[bool] = None allow_auto_paste: Optional[bool] = None default_playback_voice_id: Optional[str] = None + hotkey_enabled: Optional[bool] = None chord_push_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6) chord_toggle_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6) @@ -294,6 +296,68 @@ class GenerationSettingsUpdate(BaseModel): autoplay_on_generate: Optional[bool] = None +class MCPClientBindingResponse(BaseModel): + """Per-MCP-client voice binding — what voice / engine / intent the server + should use when a given client_id calls voicebox.speak without args.""" + + client_id: str + label: Optional[str] = None + profile_id: Optional[str] = None + default_engine: Optional[str] = Field( + None, + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + ) + default_intent: Optional[str] = Field( + None, pattern="^(respond|rewrite|compose)$" + ) + last_seen_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + class Config: + from_attributes = True + + +class MCPClientBindingUpsert(BaseModel): + """Create or update a binding. Matched by ``client_id``.""" + + client_id: str = Field(..., min_length=1, max_length=64) + label: Optional[str] = Field(None, max_length=128) + profile_id: Optional[str] = None + default_engine: Optional[str] = Field( + None, + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + ) + default_intent: Optional[str] = Field( + None, pattern="^(respond|rewrite|compose)$" + ) + + +class MCPClientBindingListResponse(BaseModel): + items: List[MCPClientBindingResponse] + + +class SpeakRequest(BaseModel): + """Body for POST /speak — non-MCP REST surface that mirrors voicebox.speak.""" + + text: str = Field(..., min_length=1, max_length=10000) + profile: Optional[str] = Field( + None, + description="Voice profile name or id. Falls back to per-client binding, then default.", + ) + engine: Optional[str] = Field( + None, + pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$", + ) + intent: Optional[str] = Field( + None, pattern="^(respond|rewrite|compose)$" + ) + language: Optional[str] = Field( + None, + pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$", + ) + + class LLMGenerateRequest(BaseModel): """Request model for LLM text generation.""" diff --git a/backend/requirements.txt b/backend/requirements.txt index 9051645a..caafc0e7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -62,6 +62,11 @@ pedalboard>=0.9.0 # HTTP client (for CUDA backend download) httpx>=0.27.0 +# MCP server (Model Context Protocol) — lets local AI agents call +# voicebox.speak / .transcribe / .list_captures / .list_profiles +fastmcp>=3.0,<4.0 +sse-starlette>=2.0 + # Utilities python-multipart>=0.0.6 Pillow>=10.0.0 diff --git a/backend/routes/__init__.py b/backend/routes/__init__.py index 3f46b04c..35563aaa 100644 --- a/backend/routes/__init__.py +++ b/backend/routes/__init__.py @@ -20,6 +20,9 @@ def register_routers(app: FastAPI) -> None: from .settings import router as settings_router from .tasks import router as tasks_router from .cuda import router as cuda_router + from .speak import router as speak_router + from .mcp_bindings import router as mcp_bindings_router + from .events import router as events_router app.include_router(health_router) app.include_router(profiles_router) @@ -36,3 +39,6 @@ def register_routers(app: FastAPI) -> None: app.include_router(settings_router) app.include_router(tasks_router) app.include_router(cuda_router) + app.include_router(speak_router) + app.include_router(mcp_bindings_router) + app.include_router(events_router) diff --git a/backend/routes/events.py b/backend/routes/events.py new file mode 100644 index 00000000..0330eb43 --- /dev/null +++ b/backend/routes/events.py @@ -0,0 +1,46 @@ +"""Server-Sent-Event streams the frontend subscribes to. + +``GET /events/speak`` — broadcasts ``speak-start`` / ``speak-end`` events +whenever an agent-initiated speak (MCP tool or POST /speak) runs. The +DictateWindow uses them to show the floating pill in a `speaking` state. +""" + +import asyncio +import json +import logging + +from fastapi import APIRouter, Request +from sse_starlette.sse import EventSourceResponse + +from ..mcp_server import events as mcp_events + + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/events/speak") +async def speak_events(request: Request): + """SSE stream of speak-start / speak-end events.""" + + async def event_stream(): + queue = mcp_events.subscribe() + try: + # Immediate hello so EventSource knows the connection is live. + yield {"event": "ready", "data": "{}"} + while True: + if await request.is_disconnected(): + return + try: + event = await asyncio.wait_for(queue.get(), timeout=15.0) + except asyncio.TimeoutError: + # Heartbeat so proxies don't reap idle streams. + yield {"event": "ping", "data": "{}"} + continue + kind = event.pop("kind", "message") + yield {"event": kind, "data": json.dumps(event)} + finally: + mcp_events.unsubscribe(queue) + + return EventSourceResponse(event_stream()) diff --git a/backend/routes/mcp_bindings.py b/backend/routes/mcp_bindings.py new file mode 100644 index 00000000..1078f99a --- /dev/null +++ b/backend/routes/mcp_bindings.py @@ -0,0 +1,79 @@ +"""REST endpoints for per-MCP-client voice binding settings. + +The Settings UI uses these to let users configure distinct voices per +agent (Claude Code in Morgan, Cursor in Scarlett, ...). The ``client_id`` +column is the same value the MCP client sends in ``X-Voicebox-Client-Id`` +(or the stdio shim pulls from ``VOICEBOX_CLIENT_ID``). +""" + +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.orm import Session + +from .. import models +from ..database import get_db +from ..database.models import MCPClientBinding + + +router = APIRouter() + + +@router.get( + "/mcp/bindings", + response_model=models.MCPClientBindingListResponse, +) +async def list_mcp_bindings(db: Session = Depends(get_db)): + rows = ( + db.query(MCPClientBinding) + .order_by(MCPClientBinding.client_id) + .all() + ) + return models.MCPClientBindingListResponse( + items=[models.MCPClientBindingResponse.model_validate(r) for r in rows] + ) + + +@router.put( + "/mcp/bindings", + response_model=models.MCPClientBindingResponse, +) +async def upsert_mcp_binding( + data: models.MCPClientBindingUpsert, + db: Session = Depends(get_db), +): + """Create-or-update a binding. Matches by client_id.""" + row = ( + db.query(MCPClientBinding) + .filter(MCPClientBinding.client_id == data.client_id) + .first() + ) + if row is None: + row = MCPClientBinding(client_id=data.client_id) + db.add(row) + + row.label = data.label + row.profile_id = data.profile_id + row.default_engine = data.default_engine + row.default_intent = data.default_intent + row.updated_at = datetime.utcnow() + db.commit() + db.refresh(row) + return models.MCPClientBindingResponse.model_validate(row) + + +@router.delete("/mcp/bindings/{client_id}") +async def delete_mcp_binding( + client_id: str, + db: Session = Depends(get_db), +): + row = ( + db.query(MCPClientBinding) + .filter(MCPClientBinding.client_id == client_id) + .first() + ) + if row is None: + raise HTTPException(status_code=404, detail="Binding not found") + db.delete(row) + db.commit() + return {"deleted": client_id} diff --git a/backend/routes/speak.py b/backend/routes/speak.py new file mode 100644 index 00000000..59cf3d7b --- /dev/null +++ b/backend/routes/speak.py @@ -0,0 +1,93 @@ +"""POST /speak — REST wrapper around voicebox.speak for non-MCP callers. + +Shell scripts, ACP, A2A, or any agent that doesn't speak MCP can hit this +endpoint to play text through a cloned voice. Uses the same profile +resolution and generation pipeline as the MCP tool, so per-client +bindings (via X-Voicebox-Client-Id) work identically. +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy.orm import Session + +from .. import models +from ..database import get_db +from ..mcp_server import events as mcp_events +from ..mcp_server.resolve import resolve_profile + + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.post("/speak", response_model=models.GenerationResponse) +async def speak( + data: models.SpeakRequest, + request: Request, + db: Session = Depends(get_db), +): + """Speak text in a voice profile. Mirrors voicebox.speak (MCP). + + Response shape matches POST /generate — a ``GenerationResponse`` with + ``status="generating"`` and an ``id`` the caller polls at + ``GET /generate/{id}/status``. + """ + client_id = request.headers.get("X-Voicebox-Client-Id") + profile = resolve_profile(data.profile, client_id, db) + if profile is None: + if data.profile: + raise HTTPException( + status_code=404, + detail=f"Voice profile '{data.profile}' not found.", + ) + raise HTTPException( + status_code=400, + detail=( + "No voice profile resolved. Pass `profile` (name or id), " + "or configure a default in Voicebox → Settings → MCP." + ), + ) + + # Persona path if intent requested AND profile has a personality prompt. + if data.intent is not None and profile.personality: + from .profiles import speak_in_character + + generation = await speak_in_character( + profile.id, + models.PersonalitySpeakRequest( + text=data.text, + persist=True, + language=data.language, + engine=data.engine, + intent=data.intent, + ), + db, + ) + else: + # Plain TTS path — matches POST /generate. + from .generations import generate_speech + + generation = await generate_speech( + models.GenerationRequest( + profile_id=profile.id, + text=data.text, + language=data.language or "en", + engine=data.engine or "qwen", + ), + db, + ) + + mcp_events.publish( + "speak-start", + { + "generation_id": getattr(generation, "id", None), + "profile_name": profile.name, + "source": "rest", + "client_id": client_id, + }, + ) + return generation diff --git a/backend/services/generation.py b/backend/services/generation.py index aa5d66c4..ce8fe93c 100644 --- a/backend/services/generation.py +++ b/backend/services/generation.py @@ -134,6 +134,7 @@ async def run_generation( db=bg_db, error="Generation cancelled", ) + _notify_speak_end(generation_id, status="cancelled") except Exception as e: traceback.print_exc() await history.update_generation_status( @@ -142,11 +143,28 @@ async def run_generation( db=bg_db, error=str(e), ) + _notify_speak_end(generation_id, status="failed") + else: + _notify_speak_end(generation_id, status="completed") finally: task_manager.complete_generation(generation_id) bg_db.close() +def _notify_speak_end(generation_id: str, *, status: str) -> None: + """Publish a speak-end event; the frontend ignores unknown ids.""" + try: + from ..mcp_server import events as mcp_events + + mcp_events.publish( + "speak-end", + {"generation_id": generation_id, "status": status}, + ) + except Exception: + # Never let event pub/sub break generation completion. + pass + + def _save_generate( *, generation_id: str, diff --git a/backend/services/profiles.py b/backend/services/profiles.py index 78839504..e6598277 100644 --- a/backend/services/profiles.py +++ b/backend/services/profiles.py @@ -275,6 +275,27 @@ async def get_profile( return _profile_to_response(profile) +def get_profile_orm_by_name_or_id( + name_or_id: str, + db: Session, +) -> DBVoiceProfile | None: + """Resolve a profile from a user-supplied string that may be either id or name. + + Id is tried first (fast path, matches UUIDs). Name fallback is + case-insensitive so agents can say "Morgan" regardless of casing. + """ + if not name_or_id: + return None + row = db.query(DBVoiceProfile).filter(DBVoiceProfile.id == name_or_id).first() + if row is not None: + return row + return ( + db.query(DBVoiceProfile) + .filter(func.lower(DBVoiceProfile.name) == name_or_id.lower()) + .first() + ) + + async def get_profile_samples( profile_id: str, db: Session, diff --git a/backend/voicebox-server.spec b/backend/voicebox-server.spec index ab8566c8..3e5fac49 100644 --- a/backend/voicebox-server.spec +++ b/backend/voicebox-server.spec @@ -46,6 +46,8 @@ tmp_ret = collect_all('espeakng_loader') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('en_core_web_sm') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('unidic_lite') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('mlx') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('mlx_audio') diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index a15701d2..2cbc73a6 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -1,23 +1,30 @@ --- title: "Voicebox Documentation" -description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs." +description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine." --- -Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor. +Voicebox is the **open-source, local-first AI voice studio** — a free +alternative to ElevenLabs and WisprFlow in one app. Clone voices, generate +speech across 7 TTS engines, dictate into any app with a global hotkey, +compose multi-voice projects, and let any MCP-aware agent speak in a voice +you own. Everything runs on your hardware. ![Voicebox App Screenshot](/images/app-screenshot-1.webp) -- **Complete privacy** -- models and voice data stay on your machine -- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro -- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice -- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more -- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters -- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice -- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters -- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives -- **API-first** -- REST API for integrating voice synthesis into your own projects -- **Native performance** -- built with Tauri (Rust), not Electron -- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker +- **Dictation** — hold a chord anywhere on your machine, speak, release; the transcript pastes into the focused field +- **Captures tab** — paired audio + transcript archive, retranscribe / refine / play-as-voice +- **Voice personalities** — per-profile compose / rewrite / respond, powered by a local LLM +- **Agents speak back** — any MCP-aware agent can call Voicebox to speak in one of your cloned voices +- **7 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro +- **Cloning and preset voices** — zero-shot cloning or 50+ curated preset voices +- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili +- **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, filters +- **Expressive speech** — paralinguistic tags (`[laugh]`, `[sigh]`) and natural-language delivery control +- **Unlimited length** — auto-chunking with crossfade for long scripts +- **Stories editor** — multi-track timeline for conversations, podcasts, narratives +- **API-first** — REST + WebSocket API, MCP server for agent integrations +- **Complete privacy** — models, audio, transcripts, LLM output never leave your machine +- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux (ROCm / CPU), Intel Arc, Docker ## Download @@ -32,6 +39,8 @@ Voicebox is a **local-first voice cloning studio** -- a free and open-source alt ## Get Started -- [Installation](/overview/installation) -- download and install Voicebox -- [Quick Start](/overview/quick-start) -- get up and running in 5 minutes -- [API Reference](/api-reference) -- integrate voice synthesis into your apps +- [Installation](/overview/installation) — download and install Voicebox +- [Quick Start](/overview/quick-start) — get up and running in 5 minutes +- [Dictation](/overview/dictation) — start talking to your computer +- [Voice Personalities](/overview/voice-personalities) — compose, rewrite, respond in any profile +- [API Reference](/api-reference) — integrate voice synthesis into your apps diff --git a/docs/content/docs/overview/captures.mdx b/docs/content/docs/overview/captures.mdx new file mode 100644 index 00000000..e00e731e --- /dev/null +++ b/docs/content/docs/overview/captures.mdx @@ -0,0 +1,194 @@ +--- +title: "Captures" +description: "The paired audio + transcript archive — every dictation, recording, and uploaded audio file shows up here, replayable and retranscribable." +--- + +## Overview + +A **capture** is an audio clip paired with its transcript. The Captures tab +is where every dictation, manual recording, and uploaded audio file lands, +with the original audio kept alongside the text so you can replay, re-run +transcription with a different model, refine the transcript, or send the +content somewhere else — including generating it back as speech in any of +your voice profiles. + + + The Captures tab shipped in **0.5.0**, alongside global dictation and the + per-profile personality modes. If you've used earlier versions, note that + the Audio tab moved into **Settings → Audio Channels** to make room for + this one. + + +## Where captures come from + +| Source | How it shows up | Badge | +|---|---|---| +| **Dictation** | Triggered by the global hotkey (see [Dictation](/overview/dictation)). Auto-refined by default. | `dictation` | +| **In-app recording** | Recorded directly in the Captures tab using the built-in mic. | `recording` | +| **File upload** | Any audio file dropped into the Captures tab — `.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`. | `file` | + +All three paths share the same backend pipeline, the same model picker, and +the same refinement flags. The source badge is there so you can visually +scan a long list. + +## List view + +The main Captures view is a chronological list. Each row shows: + +- The transcript (raw or refined — the refined version wins if present) +- Duration + timestamp +- Source badge +- A play button for the original audio +- A meatballs menu with per-row actions + +Filtering and search are a Tier-2 ask — ping if you need them. + +## Detail view + +Clicking into a capture opens the detail view: + +- **Waveform player** for the original audio +- **Transcript editor** — click in and edit. Changes save on blur. +- **Refined vs. raw toggle** if refinement ran on this capture +- **Per-capture action bar** — retranscribe, refine, play as voice, delete +- **Settings snapshot** — STT model used, refinement flags at the time + this capture was processed, and the voice model if any was played + +## Retranscribe + +Runs the capture's original audio through a different Whisper model without +re-uploading or re-refining anything. Useful when: + +- The default model mis-heard something and you want to try a larger model +- You used Base for a noisy clip and want to rerun with Turbo +- A non-English clip needs an explicit language hint + +**Settings → Captures → Transcription** controls the default model and +language lock for new captures. Retranscribe uses those defaults unless you +override them per capture. + +## Refine + +Runs the raw transcript through the local LLM to produce a cleaned-up +version. The flags on the capture are snapshotted when refinement first +runs, so you can re-refine later with different flags without losing the raw +transcript: + +| Flag | Effect | +|---|---| +| **Smart cleanup** | Remove fillers (`um`, `uh`, `like`), tidy punctuation and capitalization. | +| **Remove self-corrections** | Keep the final version when the speaker backtracks ("actually, no, on Tuesday"). | +| **Preserve technical terms** | Leave identifiers (`handleSubmit`, `npm install`) untouched. | + +See the Refinement section of [Dictation](/overview/dictation#refinement) for +how Voicebox strips Whisper loop hallucinations *before* the LLM sees the +transcript — a capture can be re-refined any number of times without +re-introducing "thanks for watching thanks for watching" echoes. + +The refinement model picker (three bundled Qwen3 sizes) lives in +**Settings → Captures → Refinement**. + +## Play as voice + +This is the capability no one else in the dictation category ships: take any +capture and play it back as speech in any of your voice profiles. One +dropdown over every profile, one click, and the capture's text runs through +`/generate` with the selected voice. + +Use cases: + +- Hear your own dictation back in a cloned voice of someone you like +- Send a message you dictated as an audio reply in a specific character +- Quickly prototype a line for a story without retyping + +Playback uses whatever engine the selected profile is bound to — the same +rules as the Generate tab. There's no LLM in this path; the transcript goes +through unchanged. If you want the agent-style "transform the content before +speaking" flow, that's what the +[personality modes](/overview/voice-personalities) do — and the same +primitive is exposed to MCP-aware agents via the +[MCP Server](/overview/mcp-server) so Claude Code, Cursor, or Cline can speak +in one of your voices on their own. + + + The default voice for the Captures tab's Play-as action is set in + **Settings → Captures → Playback → Default voice**. You can still override + it per capture. + + +## Send-to menu + +Each capture has a Send-to menu for moving its content into other parts of +Voicebox: + +- **Copy transcript** — to clipboard +- **Use as voice sample…** — promote this capture to a sample on a voice + profile of your choice. Opens a profile picker (with "+ New voice" for + cold starts) and a reference-text confirm dialog, because cloning needs + the `reference_text` to match the audio verbatim. Edit as needed and + save — the capture stays in the Captures tab untouched; the sample is a + copy, not a move. + +## Storage and retention + +**Settings → Captures → Storage** controls how long captures live on disk: + +| Setting | Effect | +|---|---| +| **Retention: forever** | Never auto-delete. Default. | +| **Retention: 90 days / 30 days / 7 days** | Captures older than the window are pruned on app start. | +| **Clear all captures** | One-click nuke of every capture and its audio on disk. No undo. | + +The original audio is always kept alongside the transcript — archival is on +by default. Every capture's audio file and metadata row can be re-processed +(retranscribe, refine, Play-as) as long as the audio file still exists. + +## Short-recording guard + +Audio clips under **300 ms** are short-circuited client-side and never +uploaded. This prevents a fumbled chord tap from landing an empty capture. +The threshold is tuned to filter accidents without cutting off intentional +short dictations. + +## Keyboard shortcuts + +Inside the Captures tab: + +| Keys | Action | +|---|---| +| `Space` | Play / pause the selected capture | +| `↑` / `↓` | Previous / next capture in the list | +| `Enter` | Open the selected capture in detail view | +| `⌘ / Ctrl` + `C` (in detail view) | Copy the transcript | + +## API surface + +The Captures tab is backed by a small set of REST endpoints: + +| Method | Endpoint | Use | +|---|---|---| +| `POST` | `/captures` | Upload audio + start the pipeline (STT, optional refinement, archival). | +| `GET` | `/captures` | List captures. | +| `GET` | `/captures/{id}` | Fetch one capture. | +| `POST` | `/captures/{id}/retranscribe` | Rerun STT with a chosen model. | +| `POST` | `/captures/{id}/refine` | Rerun refinement with chosen flags. | +| `POST` | `/profiles/{id}/samples/from-capture/{capture_id}` | Promote a capture to a voice profile sample. | + +These endpoints are stable and usable from your own scripts — see +[Remote Mode](/overview/remote-mode) for running Voicebox as a server the rest +of your machine can talk to. + +## Next steps + + + + The global hotkey flow that feeds most captures. + + + Per-profile compose / rewrite / respond modes for captures you want to + transform, not just transcribe. + + + Promote a capture into a voice sample on a profile. + + diff --git a/docs/content/docs/overview/dictation.mdx b/docs/content/docs/overview/dictation.mdx new file mode 100644 index 00000000..897a8355 --- /dev/null +++ b/docs/content/docs/overview/dictation.mdx @@ -0,0 +1,208 @@ +--- +title: "Dictation" +description: "Hold a key anywhere on your machine, speak, release — the transcript lands in whatever text field you had focused." +--- + +## Overview + +Dictation lets you turn speech into clean text anywhere on your computer. Hold +a chord, talk, release — Voicebox transcribes what you said with Whisper, +optionally cleans it up with a local LLM, and pastes the result into the text +field you had focused when you started. + +Everything happens on your hardware. No cloud, no accounts, no audio leaving +the machine. + + + Dictation was introduced in **0.5.0** alongside the Captures tab and the + per-profile personality modes. It's the "input" half of Voicebox's voice I/O + loop — cloning and TTS are still the "output" half. + + +## The flow + + + + Hold the push-to-talk chord anywhere on your machine. A small pill fades + in over your current app. + + + The pill shows `Recording` with a live waveform and an elapsed-time + counter. Speak naturally — you don't have to wait for anything. + + + On release, the pill flips to `Transcribing`, then `Refining` if + auto-refine is on, then disappears. + + + If auto-paste is enabled and Voicebox has Accessibility permission, the + transcript pastes into the text field you had focused when you started + talking — not wherever focus drifted while you were speaking. + + + +Either way, every capture also appears in the **Captures tab** with the +original audio and the transcript paired together. See +[Captures](/overview/captures) for what you can do with them after the fact. + +## Push-to-talk and toggle modes + +Voicebox ships two chord behaviors out of the box: + +| Mode | Default (macOS) | Default (Windows) | Behavior | +|---|---|---|---| +| **Push-to-talk** | Right `⌘` + Right `⌥` | Right `Ctrl` + Right `Shift` | Recording stops when you release the chord. | +| **Toggle-to-talk** | Push-to-talk + `Space` | Push-to-talk + `Space` | Recording keeps going until you tap the chord again. | + +**Holding PTT and tapping `Space` mid-hold upgrades a hold into a toggled +session** without a gap in the audio. This is the single most useful detail of +the chord system — short bursts feel fast, long-form narration feels +hands-free, and there's no decision up front about which mode you wanted. + +## The on-screen pill + +While you're dictating, a floating pill appears over the current app. It walks +through the states of the capture cycle and shows live signals for each: + +| State | What it shows | +|---|---| +| `Recording` | Live waveform + elapsed time. | +| `Transcribing` | Thinking waveform while Whisper runs. | +| `Refining` | Same thinking waveform while the LLM cleans up the transcript (only if auto-refine is on). | +| Error | Red tint. Click the pill to copy the error to your clipboard. Auto-dismisses. | + +The pill is transparent, always-on-top, and pre-created hidden at app start — +so it appears instantly when you hit the chord, with no window flash. + +## Customizing the chord + +Open **Settings → Captures → Dictation** to change either chord. + +- **Left vs right modifier badges.** When you hold keys into the chord + picker, Voicebox records whether each modifier is the left or right variant. + That means you can bind to just the right `⌥` while leaving the left `⌥` + alone — useful if you want dictation on one hand and keep your + other-hand shortcuts intact. +- **Chord defaults are picked to stay out of your way.** On macOS, the + defaults deliberately avoid left-hand `Cmd+Option` chords so + `Cmd+Option+I` (devtools), `Cmd+Option+Esc` (force quit), and + `Cmd+Option+Space` (Spotlight) all remain yours. On Windows, the defaults + route around AltGr collisions on German / French / Spanish layouts where + `Ctrl+Alt` synthesizes AltGr. +- **Live reload.** Changing a chord in Settings takes effect immediately — + no restart, no tab reload. + +## Auto-paste into the focused app + +Once transcription finishes, Voicebox can synthesize a native paste into +whatever text field had focus when you started the chord. Your clipboard is +saved before and restored after, so nothing you had copied goes missing. + +| Platform | Mechanism | +|---|---| +| macOS | `CGEventPost` at the HID tap with a full `⌘V` key sequence, preceded by reactivating the original app via `NSRunningApplication`. | +| Windows | `SendInput` with correct scan codes, plus a `SetForegroundWindow` + `AttachThreadInput` handshake to defeat foreground-lock when pasting into a window that wasn't frontmost at chord-start. | + +**Focus is snapshotted at chord-start.** The paste targets the original field +even if focus drifts during transcribe / refine — that's the "pastes where you +were talking *from*, not where you're looking *now*" behavior. + + + Auto-paste is optional. If Accessibility permission isn't granted (macOS), + or you prefer to keep synthetic input off, dictation still runs — transcripts + land in the Captures tab and you can copy them manually. The setting lives + inline next to the Accessibility prompt in Settings → Captures → Dictation, + not as a global banner. + + +## Refinement + +If auto-refine is on, a local LLM cleans up the raw Whisper transcript +before it's pasted. The goal is to remove verbal clutter without rewriting +what you actually said. + +What refinement typically fixes: + +- Filler words (`um`, `uh`, `like` used as pauses, `you know`) +- Self-corrections — the LLM keeps the final version and drops earlier + attempts (`could you uh run the migration real quick, and then, yeah, + check the logs` → `Could you run the migration, then check the logs?`) +- Basic punctuation and capitalization +- Whisper loop hallucinations — Voicebox strips repeated tokens (six or + more identical tokens in a row, case-insensitive) *before* the LLM + sees the transcript, so a small refinement model can't echo them back + +What refinement deliberately preserves: + +- Technical terms and code identifiers (`npm install`, `handleSubmit`) +- Legitimate repetition (`no, no, no, no, no` has fewer than six identical + tokens, so it survives) +- Your intent — refinement is cleanup, not rewriting + +Flags are snapshotted per capture, so you can re-refine the same raw +transcript later with different flags without losing the original. The +refinement model picker (**Settings → Captures → Refinement**) offers three +bundled Qwen3 sizes: + +| Model | Size | Best for | +|---|---|---| +| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual dictation. | +| Qwen3 1.7B | ~1.1 GB | Sweet spot when transcripts contain code identifiers. | +| Qwen3 4B | ~2.5 GB | Full quality, slowest. | + +This is the same local LLM used by the per-profile personality modes — one +LLM in the app, not two. See [Voice Personalities](/overview/voice-personalities). + +## Platform notes + +### macOS + +- **Accessibility permission** is required for auto-paste. The prompt lives + inline next to the toggle in **Settings → Captures → Dictation**, with a + deep link to **System Settings → Privacy & Security → Accessibility**. +- **TSM crash mitigation.** The global hotkey listener runs on a background + thread with `set_is_main_thread(false)` to sidestep a known + macOS 14+ crash in the `rdev` library. If you hit an unexpected dictation + failure on macOS, check the logs for TSM-related messages. + +### Windows + +- **UAC / UIPI caveat.** Synthetic paste into an *elevated* window from a + non-elevated Voicebox is blocked by Windows itself. Run Voicebox elevated + if you regularly dictate into elevated apps (e.g. an elevated terminal or + Task Manager). +- **Right-hand default chord** (`Ctrl+Shift`) avoids AltGr collisions on + keyboard layouts where `Ctrl+Alt` is the compose key (German, French, + Spanish, some others). + +### Linux + +- **Not yet in this release.** The Rust shim ships the macOS and Windows + paths in 0.5.0. Linux `uinput` / AT-SPI support and the Wayland paste + story are tracked in `docs/plans/VOICE_IO.md`. + +## When auto-paste skips itself + +A few cases where Voicebox deliberately does *not* synthesize a paste: + +- **Focus was inside Voicebox** when the chord started. The transcript goes + to the Captures tab so a dictation-into-Voicebox round-trip doesn't + accidentally paste into the generate box. +- **No text focus detected.** The transcript still lands in the Captures + tab; copy it from there with one click. +- **Accessibility permission not granted** on macOS. Same — Captures tab + only. + +## Next steps + + + + The paired audio + transcript archive every dictation lands in. + + + The same local LLM doubles as per-profile compose / rewrite / respond. + + + Developer-level details on Whisper, Whisper Turbo, and the STT backend. + + diff --git a/docs/content/docs/overview/introduction.mdx b/docs/content/docs/overview/introduction.mdx index 69cca3d5..daf60be1 100644 --- a/docs/content/docs/overview/introduction.mdx +++ b/docs/content/docs/overview/introduction.mdx @@ -1,23 +1,48 @@ --- title: "Introduction" -description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs." +description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine." --- ## What is Voicebox? -Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio or pick from 50+ preset voices, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor. +Voicebox is the **open-source, local-first AI voice studio**. It closes the +voice I/O loop in both directions on one machine, with no cloud and no +accounts: -- **Complete privacy** -- models and voice data stay on your machine -- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro -- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or curated preset voices via Kokoro (50 voices) and Qwen CustomVoice (9 voices) -- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more -- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters -- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice -- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters -- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives -- **API-first** -- REST API for integrating voice synthesis into your own projects -- **Native performance** -- built with Tauri (Rust), not Electron -- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker +- **Humans talk** — hold a chord anywhere on your machine and your + dictation lands as clean text in whatever text field you had focused +- **Agents talk back** — any MCP-aware agent can call Voicebox to speak in + one of your cloned voices +- **Voices speak for themselves** — voice profiles can carry a personality + that compose, rewrite, or respond to text before it's spoken + +It's the free, local alternative to both ElevenLabs (voice cloning and TTS) +and WisprFlow (voice dictation for agents and power users) — covering both +sides of the same loop in one app, with a single model directory and LLM +shared between input and output. + +## What's in the app + +- **Dictation** — global hotkey, push-to-talk and toggle modes, auto-paste + into the focused field on macOS and Windows (see [Dictation](/overview/dictation)) +- **Captures tab** — paired audio + transcript archive, retranscribe, + refine, play-as-voice, promote-to-sample (see [Captures](/overview/captures)) +- **Voice cloning** — 5 cloning engines covering 23 languages. Zero-shot + cloning from a reference sample (see [Voice Cloning](/overview/voice-cloning)) +- **Preset voices** — 50+ curated voices via Kokoro and Qwen CustomVoice + for when you don't want to clone (see [Preset Voices](/overview/preset-voices)) +- **Voice personalities** — optional free-form personality on any profile + plus compose / rewrite / respond modes powered by a local LLM (see + [Voice Personalities](/overview/voice-personalities)) +- **Post-processing effects** — pitch shift, reverb, delay, chorus, + compression, filters (Spotify's Pedalboard) +- **Expressive speech** — paralinguistic tags like `[laugh]` and `[sigh]` + via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice +- **Unlimited length** — auto-chunking with crossfade for long scripts +- **Stories editor** — multi-track timeline for conversations and podcasts +- **API-first** — REST + WebSocket API; MCP server for agent integrations +- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux + (ROCm / CPU), Intel Arc, Docker ## TTS Engines @@ -30,9 +55,21 @@ Seven engines with different strengths, switchable per-generation: | **LuxTTS** | Cloned | English | Lightweight (~1GB VRAM), 48kHz output, 150x realtime on CPU | | **Chatterbox Multilingual** | Cloned | 23 | Broadest language coverage | | **Chatterbox Turbo** | Cloned | English | Fast 350M model with paralinguistic emotion/sound tags | -| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model -- 700s+ coherent audio | +| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model — 700s+ coherent audio | | **Kokoro** | Preset (50 voices) | 9 | 82M parameters, CPU realtime, lowest VRAM of any engine | +## STT and local LLM + +Voicebox also runs a full speech recognition and local LLM stack, shared +between dictation, the Captures tab, and per-profile personality modes: + +| Layer | Models | +|---|---| +| **STT** | Whisper Base / Small / Medium / Large / Turbo (PyTorch or MLX) | +| **LLM** | Qwen3 0.6B / 1.7B / 4B (refinement + per-profile compose / rewrite / respond) | + +No cloud fallback, no bring-your-own-API-key. Local is the product. + ## GPU Support | Platform | Backend | Notes | @@ -46,11 +83,13 @@ Seven engines with different strengths, switchable per-generation: ## Use Cases -- **Game development** -- generate dynamic dialogue for characters -- **Content creation** -- produce podcasts and video voiceovers -- **Accessibility** -- build text-to-speech tools for users who need them -- **Voice assistants** -- create custom voice interfaces -- **Production pipelines** -- automate voiceover workflows via the REST API +- **Dictation for humans and agents** — speak instead of type, in any app +- **Agent voice output** — any MCP-aware agent can speak in a cloned voice +- **Game development** — generate dynamic dialogue for characters +- **Content creation** — podcasts, video voiceovers, audiobooks +- **Accessibility** — speech-to-text for any field, TTS with a voice you own +- **Voice assistants** — custom voice interfaces without a cloud bill +- **Production pipelines** — automate voice workflows via the REST API ## Tech Stack @@ -61,8 +100,9 @@ Seven engines with different strengths, switchable per-generation: | State | Zustand, React Query | | Backend | FastAPI (Python) | | TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro | +| STT | Whisper / Whisper Turbo (PyTorch or MLX) | +| Local LLM | Qwen3 0.6B / 1.7B / 4B (MLX or PyTorch) | | Effects | Pedalboard (Spotify) | -| Transcription | Whisper / Whisper Turbo (PyTorch or MLX) | | Inference | MLX (Apple Silicon) / PyTorch (CUDA/ROCm/XPU/CPU) | | Database | SQLite | | Audio | WaveSurfer.js, librosa | diff --git a/docs/content/docs/overview/mcp-server.mdx b/docs/content/docs/overview/mcp-server.mdx new file mode 100644 index 00000000..1f39e98f --- /dev/null +++ b/docs/content/docs/overview/mcp-server.mdx @@ -0,0 +1,300 @@ +--- +title: "MCP Server" +description: "Let Claude Code, Cursor, Cline, or any MCP-aware agent speak in one of your cloned voices — locally, with no cloud." +--- + +## Overview + +Voicebox ships a built-in **Model Context Protocol** server so local AI +agents can call your Voicebox install directly: speak text in a voice +profile, transcribe audio, and list captures or profiles. The server runs +inside the same process as the rest of Voicebox and is mounted at `/mcp` +over Streamable HTTP. + +Agent asks to speak → Voicebox plays audio on your speakers → an on-screen +pill surfaces the voice name for the whole duration so you always see what's +coming out of your machine. + + + MCP shipped in **0.5.0** alongside [Dictation](/overview/dictation) and + [Voice Personalities](/overview/voice-personalities). The design goal is + "local voice layer for every agent on your machine" — the same app that + captures your voice can generate a response in any voice profile you've + cloned. + + +## Quick install + +### Claude Code + +``` +claude mcp add voicebox \ + --transport http \ + --url http://127.0.0.1:17493/mcp \ + --header "X-Voicebox-Client-Id: claude-code" +``` + +### Cursor / Windsurf / VS Code MCP / any HTTP MCP client + +Drop this into the client's MCP config (usually `.mcp.json` or a Settings UI): + +```json +{ + "mcpServers": { + "voicebox": { + "url": "http://127.0.0.1:17493/mcp", + "headers": { "X-Voicebox-Client-Id": "cursor" } + } + } +} +``` + +Change `cursor` to whatever name you want the binding to show up as in +Voicebox → Settings → MCP. The value is just an identifier for the +per-client voice binding — not a secret, not a credential. + +### Clients that only speak stdio + +A stdio shim binary `voicebox-mcp` is bundled with the desktop app. Point +the client at that binary's absolute path: + + + + ```json + { + "mcpServers": { + "voicebox": { + "command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp", + "env": { "VOICEBOX_CLIENT_ID": "claude-desktop" } + } + } + } + ``` + + + ```json + { + "mcpServers": { + "voicebox": { + "command": "C:\\Program Files\\Voicebox\\voicebox-mcp.exe", + "env": { "VOICEBOX_CLIENT_ID": "claude-desktop" } + } + } + } + ``` + + + ```json + { + "mcpServers": { + "voicebox": { + "command": "/opt/voicebox/voicebox-mcp", + "env": { "VOICEBOX_CLIENT_ID": "claude-desktop" } + } + } + } + ``` + + + +The shim waits up to 30 seconds for the Voicebox backend to come up, then +proxies JSON-RPC from stdio over Streamable HTTP. Voicebox must be running +for the shim to connect. + +## Tools + +| Tool | Use | +|---|---| +| `voicebox.speak` | Speak text in a voice profile. Returns a `generation_id` to poll. | +| `voicebox.transcribe` | Whisper transcription of base64 audio or an absolute local path. | +| `voicebox.list_captures` | Recent captures with transcripts, paginated. | +| `voicebox.list_profiles` | Available voice profiles (cloned + preset). | + +### `voicebox.speak` + +```ts +voicebox.speak({ + text: "Deploy complete.", + profile?: "Morgan", // name or id; falls back to per-client binding, then default + engine?: "qwen", // qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro + intent?: "respond", // respond | rewrite | compose — only if the profile has a personality + language?: "en", +}) +``` + +Returns: + +```json +{ + "generation_id": "…", + "status": "generating", + "profile": "Morgan", + "source": "mcp", + "poll_url": "/generate//status" +} +``` + +- **Plain TTS** — omit `intent`. Text is spoken as-is. +- **Persona mode** — pass `intent` and the profile must have a personality set. + The LLM transforms the text (respond to it, rewrite it in character, or + compose a fresh utterance) before TTS. See [Voice Personalities](/overview/voice-personalities). + +### `voicebox.transcribe` + +```ts +voicebox.transcribe({ + audio_base64?: "", // exactly one of these two + audio_path?: "/absolute/path/to/file.wav", + language?: "en", + model?: "turbo", // base | small | medium | large | turbo +}) +``` + +Returns `{ text, duration, language, model }`. 200 MB ceiling on either path. + +### `voicebox.list_captures` + +`{ limit?: 20, offset?: 0 }` → `{ captures: [...], total }`. `limit` is +clamped to `1..=200`. + +### `voicebox.list_profiles` + +No args → `{ profiles: [{ id, name, voice_type, language, has_personality }] }`. + +## Voice resolution + +Every call to `voicebox.speak` (and `POST /speak`) resolves the voice profile +in this order: + + + + Passed as a name (case-insensitive) or id. If the name/id doesn't match, + the call errors — the server doesn't silently fall back. + + + Looked up by the `X-Voicebox-Client-Id` header. Managed in + **Voicebox → Settings → MCP**. Lets you pin Claude Code to Morgan, + Cursor to Scarlett, etc. + + + `capture_settings.default_playback_voice_id` — same default voice the + Captures tab's "Play as voice" action uses. + + + +If none of the three produce a profile the tool returns a helpful error +pointing at Settings. + +## Per-client bindings + +Voicebox → Settings → MCP shows one row per `client_id` Voicebox has heard +from, plus the config snippets you can copy into each agent. Each row +carries: + +| Field | Purpose | +|---|---| +| `label` | Display name in the Settings UI (e.g. "Claude Code"). | +| `profile_id` | The voice this client uses when `profile` isn't passed. | +| `default_engine` | Override the TTS engine for this client. | +| `default_intent` | Default persona mode (`respond` / `rewrite` / `compose`). | +| `last_seen_at` | Last time the server saw a request from this client. | + +`last_seen_at` is stamped automatically by middleware on every `/mcp/*` +request — useful when you're not sure whether your config took. + +## The speaking pill + +Every agent-initiated speak surfaces the floating pill the same way +[Dictation](/overview/dictation) does, in a new `Speaking` state showing the +profile name and an elapsed timer. The pill is intentionally unmissable — +silent background TTS is a trust hazard, so Voicebox always shows what's +being spoken and in what voice. + +Behind the scenes, the backend broadcasts `speak-start` and `speak-end` +events on `GET /events/speak`, which `DictateWindow` subscribes to via SSE. +The pill overrides the capture session when both would render — you can't +hear two pills at once. + +## Non-MCP REST surface + +`POST /speak` is a thin wrapper on the same code path for callers that +don't speak MCP — shell scripts, ACP, A2A, GitHub Actions, whatever. + +```bash +curl -X POST http://127.0.0.1:17493/speak \ + -H 'Content-Type: application/json' \ + -H 'X-Voicebox-Client-Id: ci' \ + -d '{"text":"Build complete.","profile":"Morgan"}' +``` + +Body fields match the MCP tool: `text`, optional `profile`, `engine`, +`intent`, `language`. Returns a `GenerationResponse` — the same shape as +`POST /generate`. + +## Debugging + +Use the MCP Inspector to poke tools directly without plumbing through an +agent: + +``` +npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp +``` + +Start with `voicebox.list_profiles` to confirm wiring, then +`voicebox.speak` for end-to-end — you should hear audio and see the +generation land in the Captures tab. + + + If an agent can't reach the server, the first thing to check is that + Voicebox is running — the backend only listens while the desktop app is + open. The stdio shim surfaces this as a JSON-RPC error on the client + side after its 30-second health-wait window elapses. + + +## Security + +- **Localhost only.** The server binds to `127.0.0.1`. If you ever point + Voicebox at a non-loopback interface (e.g. remote-mode over a trusted + network), add a bearer token — it's on the roadmap but not in 0.5.0. +- **No auth today.** Any process that can connect to your loopback can + call MCP. That's the same trust boundary as the rest of Voicebox's REST + API and is appropriate for a single-user local tool. +- **`audio_path` reads are unrestricted** against the same trust + boundary. If you're scripting against a shared host, prefer + `audio_base64` so you don't have to think about path sandboxing. +- **Voice cloning consent applies.** See [Voice Cloning](/overview/voice-cloning#limitations) + — an agent being able to call `voicebox.speak` in someone's voice + doesn't change the ethics of whose voices you clone. + +## Implementation notes + +- **Transport:** Streamable HTTP (Nov-2025 MCP spec, post-SSE). Claude + Code, Cursor, Windsurf, and VS Code MCP extensions all support it. +- **Package naming:** the backend package is `backend/mcp_server/`, not + `mcp`, to avoid shadowing the PyPI `mcp` package FastMCP imports + internally. +- **Dependencies:** `fastmcp>=3.0,<4.0`, `sse-starlette>=2.0`. +- **Lifespan:** mounting FastMCP requires the `lifespan=` kwarg on + `FastAPI()` — the startup/shutdown event decorators are incompatible + with FastMCP's Streamable HTTP session manager. The Voicebox app.py + composes both into one async context manager. + +For the full developer-facing tour of the code layout, see +`backend/mcp_server/README.md` in the repo. + +## Next steps + + + + Persona mode (`intent=respond/rewrite/compose`) for agents that should + transform text in-character before speaking. + + + The pill that surfaces agent speech is the same one that surfaces + your dictations — one mental model for both directions of the loop. + + + Every agent-initiated speak lands in the Captures tab with its + generated audio — replay, download, repurpose. + + diff --git a/docs/content/docs/overview/meta.json b/docs/content/docs/overview/meta.json index 90ce7e2e..989670f5 100644 --- a/docs/content/docs/overview/meta.json +++ b/docs/content/docs/overview/meta.json @@ -7,8 +7,12 @@ "docker", "quick-start", "gpu-acceleration", + "dictation", + "captures", "voice-cloning", "preset-voices", + "voice-personalities", + "mcp-server", "stories-editor", "recording-transcription", "generation-history", diff --git a/docs/content/docs/overview/recording-transcription.mdx b/docs/content/docs/overview/recording-transcription.mdx index e86b30b9..61544de4 100644 --- a/docs/content/docs/overview/recording-transcription.mdx +++ b/docs/content/docs/overview/recording-transcription.mdx @@ -1,64 +1,106 @@ --- title: "Recording & Transcription" -description: "Record audio and transcribe speech with Whisper" +description: "A map of the three places you can record and transcribe audio in Voicebox — dictation, captures, and voice-profile samples." --- -## Recording +## Overview -Voicebox includes built-in recording capabilities for creating voice samples and capturing audio. +Voicebox records and transcribes audio in three different contexts, each +feeding a different surface in the app. This page is a map; follow the links +for the detail. -### Features +| Goal | Where | Docs | +|---|---|---| +| Speak and have your words land in another app | Global hotkey → Captures tab + auto-paste | [Dictation](/overview/dictation) | +| Record a thought, a meeting, or a voice memo inside Voicebox | Captures tab | [Captures](/overview/captures) | +| Record a clip to clone a voice from | Voices tab → profile samples | [Creating Voice Profiles](/overview/creating-voice-profiles) | -- **Microphone input** - Record from any audio input device -- **System audio capture** - Record desktop audio (macOS/Windows) -- **Waveform visualization** - See audio levels in real-time -- **Multiple formats** - Export as WAV, MP3, or M4A +All three paths share the same STT backend — it's the surrounding workflow +that differs. -### How to Record +## Dictation - - - Choose your microphone or system audio - - - Click the record button and speak clearly - - - Click stop when finished - - - Use as voice sample or export to file - - +The 0.5.0 headline feature. Hold a chord anywhere on your machine, speak, +release. The transcript lands in whatever text field you had focused, +cleaned up by a local LLM if auto-refine is on. Captures accumulate in the +Captures tab for later replay or re-transcription. -## Transcription +Covered end-to-end in [Dictation](/overview/dictation). -Automatic speech-to-text powered by OpenAI's Whisper model. +## Captures tab -### Features +When you don't need to paste into another app — you just want a clean +transcript of some audio — the Captures tab is the home. Record in-app, +drop in a file (`.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`), or dig +through dictations that already landed there. Every capture keeps its +original audio, can be retranscribed with a different model, and can be +played back through any voice profile you have. -- **High accuracy** - Industry-leading speech recognition -- **Multiple languages** - Supports 50+ languages -- **Automatic detection** - Language auto-detection -- **Timestamps** - Word-level timing information +Covered in [Captures](/overview/captures). -### How to Transcribe +## Voice profile samples - - - Choose a recording or upload an audio file - - - Select language or use auto-detect - - - Click transcribe and wait for processing - - - Review text and export as needed - - +A separate flow, in the Voices tab. When you're creating a profile from an +audio clip, the sample is what the cloning engine actually learns from — +the `reference_text` on a sample must match the audio *verbatim*, which is +why samples are a different data model from captures. + +You can promote a capture to a sample from the Captures tab's Send-to menu +("Use as voice sample…"), which opens a reference-text confirm dialog so +you can correct the last ~10% of transcript accuracy before saving. + +Covered in [Creating Voice Profiles](/overview/creating-voice-profiles). + +## Transcription models + +All three paths share the same Whisper models. Pick a default in +**Settings → Captures → Transcription**; override per capture if you need +to. + +| Model | Size | When to pick it | +|---|---|---| +| Whisper Base | ~300 MB | Fast. Default. Good for clean speech. | +| Whisper Small | ~500 MB | Better quality, still fast. | +| Whisper Medium | ~1.5 GB | High quality. | +| Whisper Large | ~3 GB | Best quality, slow on CPU. | +| Whisper Turbo | ~1.5 GB | Large-tier quality, ~5× faster than Large. | + +On Apple Silicon the model runs through **MLX-Whisper** (~8× faster than +PyTorch). Everywhere else it runs through PyTorch `transformers`. The +backend picks the right one — you don't configure it. - Transcription is useful for creating voice samples from existing audio or generating subtitles. + For noisy clips, prefer **Turbo** or **Large**. Base can hallucinate on + hard inputs — most famously the "thanks for watching" loop. Voicebox + strips those loops deterministically before LLM refinement runs, so a + capture can be cleanly re-refined even if the raw transcript has them. + +## Language + +You can pass a language hint for short clips (under ~5 seconds) where +Whisper's auto-detect is unreliable. Set a default language lock in +**Settings → Captures → Transcription → Language**, or override per capture. + +## Transcription API + +Developer-level detail on the STT backend, model loading, preprocessing, and +the `/transcribe` endpoint lives in the +[Transcription developer guide](/developer/transcription). The Captures +pipeline also exposes `/captures` as a higher-level endpoint that wraps +STT + archival + optional refinement in one call — see +[Captures](/overview/captures#api-surface). + +## Next steps + + + + Hold a chord anywhere on your machine, speak, release. + + + The paired audio + transcript archive. + + + Record or upload samples for voice cloning. + + diff --git a/docs/content/docs/overview/voice-personalities.mdx b/docs/content/docs/overview/voice-personalities.mdx new file mode 100644 index 00000000..b0c4aa63 --- /dev/null +++ b/docs/content/docs/overview/voice-personalities.mdx @@ -0,0 +1,193 @@ +--- +title: "Voice Personalities" +description: "Attach a personality to a voice profile and use Compose, Rewrite, or Respond to generate in-character speech — all powered by a local LLM." +--- + +## Overview + +A **personality** is an optional free-form description attached to a voice +profile — who this voice is, how they speak, what they care about. Set one +and three new actions appear on the profile, each powered by a bundled +Qwen3 LLM running entirely locally: + +- **Compose** — generate a fresh utterance in this character's voice +- **Rewrite** — restate your text in their voice while preserving every idea +- **Respond** — treat your text as a prompt and produce the character's reply + +The LLM produces the text. The voice profile speaks it. No cloud round-trip, +no external API — the whole loop runs on your hardware. + + + Personalities shipped in **0.5.0**. The same local LLM doubles as the + refinement model for [Dictation](/overview/dictation) — one LLM in the app, + not two, sharing one model cache and one GPU-memory footprint. + + +## Setting a personality + +Open a voice profile's edit view. The **Personality** field is free-form text +up to **2,000 characters**. Describe the voice however helps you — past +lines they'd say, speech patterns, tone, boundaries. + +Good descriptions tend to include: + +- A one-line identity (who they are) +- Speech patterns (rhythm, vocabulary, what they avoid) +- Representative phrases — example lines show the LLM the target tone + better than adjectives +- What the character *wouldn't* do (they don't explain, they don't + apologize, they refuse to break character, etc.) + +You can set a personality on any voice profile type — cloned or preset. The +three modes work identically regardless of engine. + +## The three modes + +Each mode is tuned for a specific job and the LLM temperature is adjusted +to match. + +### Compose + +Generate a fresh utterance in the character's voice, with no seed text. +Click again to get a different take. + +- **When to use:** prototyping, sampling a character's voice, brainstorming + a line without typing one first +- **Temperature:** hot — variety is the point +- **Typical output:** a short, punchy line that fits the character's + register + +### Rewrite + +Take your input text and restate it in the character's voice while +preserving every idea. High-fidelity mode — the content doesn't change, only +the voice does. + +- **When to use:** turning a dictated memo into in-character speech; lifting + a plain-English script into a specific voice without editing by hand +- **Temperature:** cold — faithfulness wins +- **Typical output:** same ideas, same order, different phrasing and cadence + +### Respond + +Treat your input as a prompt and produce the character's reply — as if +you'd said it *to* them. + +- **When to use:** spoken-input agents; Q&A with a specific voice; + interactive character experiences +- **Temperature:** balanced — creative but grounded +- **Typical output:** a reply to your prompt, written in-character + +## Speech-only framing + +All three modes enforce **speech-only** output. The LLM is prompted to +produce things a person would actually say out loud — no narration, no +action tags (`*sighs*`, `[laughs]`), no meta-commentary, no markdown +formatting, no stage directions. + +This is deliberate: the output is going straight into TTS, and anything that +isn't speakable ends up either ignored or read literally. The speech-only +framing also makes the output land cleanly inside dialogue, so you can drop +a Respond result straight into a Story. + +## The local LLM + +The bundled LLM is **Qwen3**, available in three sizes: + +| Model | Download size | Best for | +|---|---|---| +| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual use. | +| Qwen3 1.7B | ~1.1 GB | Sweet spot for character personalities with specific phrasing. | +| Qwen3 4B | ~2.5 GB | Full quality. Slowest. Useful for very particular tone. | + +The model runs through the same backend split Voicebox already uses for TTS +— **MLX** (4-bit community quants) on Apple Silicon, **PyTorch** (transformers +`AutoModelForCausalLM`) everywhere else. Downloads go through the same cache +and model-management UI as TTS models. + +Pick a size in **Settings → Captures → Refinement → Refinement model** — the +personality modes reuse it. If you switch models, both refinement and +personality output pick up the change on the next call. + +## Using the modes + +The three actions appear as buttons on the profile when a personality is +set. For each: + + + + Rewrite and Respond need input text. Compose doesn't. + + + The LLM runs, then the result fills the generate box. + + + The TTS engine speaks the LLM output in the profile's voice. The result + lands in generation history as a normal generation. + + + +Each button also has an inline regenerate affordance — click again to +resample. Compose will give you something totally different; Rewrite and +Respond will give you a variation on the same content. + +## Use cases + +- **Agents that speak in a voice you own.** Combine Respond with the + built-in [MCP Server](/overview/mcp-server) so Claude Code, Cursor, + Cline, or any MCP-aware agent can talk back through a profile with a + personality. The agent calls `voicebox.speak({ text, profile, intent: + "respond" })` and Voicebox produces in-character speech in your cloned + voice. +- **Interactive characters.** Games, narrative tools, accessibility + experiences. A character with a personality description plus a cloned + voice becomes a reusable prop. +- **Accessibility.** People who can't speak in their original voice can + keep a personality description of how they used to sound and use Rewrite + to turn typed input into in-character speech. +- **Creative drafting.** Write a plain outline, Rewrite line-by-line into + the character's voice, drop the audio into a Story. + +## API surface + +Personalities and the three modes are accessible via REST: + +| Method | Endpoint | Body | +|---|---|---| +| `PUT` | `/profiles/{id}` | Include a `personality` field up to 2,000 chars to set it. | +| `POST` | `/profiles/{id}/speak` | Runs the LLM + TTS in one shot. Body includes `text`, `intent` (`compose`, `rewrite`, or `respond`), optional `engine`, `language`. | + +The `/profiles/{id}/speak` endpoint is the same primitive MCP's +`voicebox.speak` tool calls when that ships. Scripts and agents can use it +directly today. + +## Limits and gotchas + +- **The personality is a prompt, not a fine-tune.** The LLM will sometimes + drift out of character, especially on Compose at high temperature. Click + again for another take. +- **Long personalities are not always better.** 2,000 chars is a ceiling, + not a goal. A sharp 300-char description with two example lines + typically outperforms a long one. +- **Speech-only framing is enforced, but not bulletproof.** Very large + prompts or unusual inputs can sneak an action tag through. If you see + `[laughs]` in TTS output, it's usually a personality-field hint the + model anchored onto — remove it from the description. +- **Rewrite is stricter than Respond.** If the output is changing your + meaning, you probably want Respond (or a wholesale Compose with context + in the input), not Rewrite. + +## Next steps + + + + Dictate the input for Rewrite or Respond from anywhere on your machine. + + + Captures feed personalities naturally — dictate a memo, rewrite it in + a character voice, generate speech. + + + Add a personality to an existing profile. + + diff --git a/docs/plans/MCP_SERVER.md b/docs/plans/MCP_SERVER.md new file mode 100644 index 00000000..9249c684 --- /dev/null +++ b/docs/plans/MCP_SERVER.md @@ -0,0 +1,344 @@ +# MCP Server — Voicebox Speed Run + +**Status:** v1 shipped — HTTP transport, all 4 tools, per-client bindings, `POST /speak`, stdio shim (binary built, bundled into Tauri sidecar), Settings UI, speak-pill via SSE with Rust-side `dictate:show` handler so agent-initiated speech surfaces the pill on screen. `cargo check` clean, `tsc` clean, full Inspector round-trip verified. +**Last reviewed:** 2026-04-23 + +## Status + +### Shipped (backend) +- **`fastmcp` + `sse-starlette`** pinned in `backend/requirements.txt`. +- **`backend/mcp_server/`** package with `server.py`, `tools.py`, `context.py`, `resolve.py`, `events.py`, `README.md`. Named `mcp_server` (not `mcp`) to sidestep a shadowing conflict with the installed `mcp` PyPI package that FastMCP imports internally. +- **Streamable HTTP mount at `/mcp`** via FastMCP's `http_app(transport='http')`. Sub-app lifespan composed with Voicebox's own startup/shutdown through an `@asynccontextmanager lifespan=` in `backend/app.py` (migrated away from the deprecated `@app.on_event` handlers). +- **Four MCP tools**, dot-named to match the landing and ecosystem convention: + - `voicebox.speak(text, profile?, engine?, intent?, language?)` + - `voicebox.transcribe(audio_base64?, audio_path?, language?, model?)` + - `voicebox.list_captures(limit, offset)` + - `voicebox.list_profiles()` +- **`ClientIdMiddleware`** pulls `X-Voicebox-Client-Id` into a `ContextVar` on every `/mcp*` request; auto-stamps `MCPClientBinding.last_seen_at`, auto-creating the row if the client is new. +- **Profile resolution precedence** `explicit → per-client binding → capture_settings.default_playback_voice_id → error`. `services/profiles.get_profile_orm_by_name_or_id()` lets agents pass a voice by name ("Morgan") instead of UUID. +- **`MCPClientBinding` table** (new) via `Base.metadata.create_all` — no migration needed. +- **Bindings REST:** `GET|PUT /mcp/bindings`, `DELETE /mcp/bindings/{client_id}`. +- **`POST /speak`** REST wrapper for non-MCP callers (shell / ACP / A2A). Same `resolve_profile` precedence, same code path as the MCP tool. +- **Stdio shim** at `backend/mcp_shim/__main__.py` — ~200 lines of `httpx` proxy; reads env (`VOICEBOX_PORT`, `VOICEBOX_HOST`, `VOICEBOX_CLIENT_ID`), waits for `/health`, then streams JSON-RPC ↔ SSE. Rolled our own after the `mcp` SDK's session-management helpers mis-shook-hands. Smoke-tested: `initialize`, `tools/list`, and `tools/call` all round-trip cleanly. +- **Pill SSE:** `GET /events/speak` (`sse-starlette`) emits `speak-start` from the MCP tool and `POST /speak`, `speak-end` from `services/generation.run_generation`'s finally block. +- **PyInstaller:** + - `backend/build_binary.py` `--shim` flag builds a minimal `voicebox-mcp` binary (torch/transformers/mlx/etc. explicitly excluded, target <20 MB). + - The main server spec picks up `fastmcp`, `mcp`, `sse_starlette`, and `backend.mcp_server.*` via `--collect-all` / `--hidden-import`. +- **`backend/mcp_server/README.md`** quickstart (Inspector, `.mcp.json` snippets, tool reference). + +### Shipped (frontend) +- **`Settings → MCP`** page (`app/src/components/ServerTab/MCPPage.tsx`): + - Three copy-paste snippets auto-filled with the detected `serverUrl`: HTTP (recommended), Claude Code CLI one-liner, stdio fallback. + - Default voice picker (bound to `capture_settings.default_playback_voice_id`, shared with Captures-tab "Play as voice"). + - Per-client bindings table with inline profile picker, remove button, and a connection-status indicator that refreshes every 10 s. + - Add-binding form with client_id / label / profile dropdown. +- **`useMCPBindings`** TanStack hook (optimistic delete, invalidate on upsert). +- **`useSpeakEvents`** hook — auto-reconnecting `EventSource('/events/speak')`, tracks the active generation_id, exposes an elapsed-ms timer that ticks so the pill's clock advances. +- **`CapturePill`** has a new `'speaking'` state + "Speaking" label + playing-bars mode. +- **`DictateWindow`** subscribes to speak events and overrides `pillState` when an agent is speaking. Emits `dictate:show` on speak-start so the Rust side can surface the pill window. +- Router + `ServerTab` tab bar wired to `/settings/mcp`. + +### Shipped (native shell) +- **`tauri.conf.json`** — `voicebox-mcp` added to `externalBin` (alongside `voicebox-server`). +- **`dictate:show` listener** in `tauri/src-tauri/src/main.rs` — invokes a new `show_dictate_window(app_handle)` helper that mirrors the hotkey-monitor's position+show logic (undo click-through, reposition to top-center of the current monitor, show). Agent-initiated speech now pops the pill visible on screen. + +### Validated end-to-end (this session, via curl) +- `/mcp/` init → `tools/list` → `tools/call voicebox.speak` → actual audio plays (Jarvis, 1.68 s). +- `POST /speak` with `X-Voicebox-Client-Id: claude-code` resolves to the bound Jarvis profile without passing `profile`. +- `/events/speak` emits `ready`, `speak-start`, `speak-end` in order, generation_id threads through both. +- Stdio shim: `echo {…} | python -m backend.mcp_shim` returns valid JSON-RPC for all 4 methods. +- `last_seen_at` auto-stamps on first call; binding row auto-creates. +- Frontend `tsc --noEmit`: clean. +- `cargo check` on the Tauri crate: clean. + +### Outstanding (must-do before release) +- **CI build for shim on Windows/Linux** — `python backend/build_binary.py --shim` is wired up and built cleanly for `aarch64-apple-darwin` (18 MB, installed at `tauri/src-tauri/binaries/voicebox-mcp-aarch64-apple-darwin`, Tauri `cargo check` green). The Windows and Linux triples (`x86_64-pc-windows-msvc`, `x86_64-unknown-linux-gnu`) need the same build in their respective CI runners and artifacts dropped alongside the macOS binary. +- **Windows/Linux paths in the stdio snippet** — the Settings page hardcodes the macOS path (`/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp`). Needs a per-OS switch (`%LOCALAPPDATA%\Programs\Voicebox\voicebox-mcp.exe`, Linux bundled-path), ideally with the Tauri shell resolving its own app path at runtime and injecting it into the snippet. + +### Nice-to-have (follow-up passes) +- **One-click install buttons** — write/merge into `~/.claude/settings.json`, `~/.cursor/mcp.json`, etc. via a Tauri command. Copy-paste works today; this is pure ergonomics. +- **`.mcpb` desktop extension** for Claude Desktop (single file, double-click to install). Claude Desktop-only, so lower priority than the agent-harness crowd. +- **Refactor the hotkey_monitor.rs show-logic** to call `show_dictate_window()` instead of duplicating the position+show block. Skipped at ship to avoid regressing the well-tested chord path. +- **Source attribution on `Generation.source`** — currently `"manual" | "personality_speak"`; adding `"mcp"` / `"rest"` would let the Captures tab filter by MCP-originated rows. + +## Context + +Voicebox already ships the I/O surface (Captures, Generate, personality-driven `/profiles/{id}/speak`), but local AI agents can't reach any of it. This plan adds a Model Context Protocol server so Claude Code / Cursor / Cline can call `voicebox.speak`, `voicebox.transcribe`, `voicebox.list_captures`, and `voicebox.list_profiles` — turning Voicebox into the local voice layer for every agent on the user's machine (Phase 5 of `docs/plans/VOICE_IO.md`). + +The shortest path to "Claude Code speaks in a cloned voice": mount **FastMCP** inside the existing FastAPI/uvicorn process at `/mcp` (Streamable HTTP), and users install it as a URL (`{"url": "http://127.0.0.1:17493/mcp"}`) — the ecosystem-idiomatic shape for a long-running local service. Per-client voice binding via a new `mcp_client_bindings` table + Settings UI, resolved from an `X-Voicebox-Client-Id` header. A **stdio shim binary** `voicebox-mcp` is bundled as a fallback sidecar for clients that can't speak HTTP MCP. A public `POST /speak` REST wrapper covers non-MCP callers (shell scripts, ACP, A2A). A `speaking` pill state gives agent-initiated audio visibility — trust-critical, non-negotiable. + +## Architecture + +``` +Claude Code / Cursor / Windsurf / VS Code MCP + │ + ├─ HTTP (primary) ────────────────────┐ + │ {"url": ".../mcp"} │ + │ │ + └─ stdio (fallback) ───────────────▶ [voicebox-mcp shim binary] + {"command": "/abs/path/voicebox-mcp"} (absolute path; + │ Settings page + │ copies it for you) + ▼ + uvicorn + FastAPI (port 17493) + ├─ /mcp (FastMCP, Streamable HTTP) + └─ /speak (REST wrapper for non-MCP callers) + └─ tools call existing services +``` + +- **Transport:** Streamable HTTP as primary (Nov-2025 spec, post-SSE). Claude Code, Cursor, Windsurf, and the VS Code MCP extensions all support HTTP — it's the idiomatic shape for a long-running local service, which Voicebox already is. +- **Stdio fallback:** `voicebox-mcp` binary bundled inside the app for clients that can't speak HTTP MCP. The Settings page renders the exact snippet with the detected absolute path — user copies, pastes, done. No PATH manipulation, no custom CLI wrapper. +- **Identity:** HTTP clients set `X-Voicebox-Client-Id` header in their MCP config's `headers` block. Stdio clients set `VOICEBOX_CLIENT_ID` env var, which the shim forwards as the same HTTP header. Server reads it into a `ContextVar`. +- **Profile resolution precedence:** explicit tool arg → per-client `MCPClientBinding.profile_id` → `capture_settings.default_playback_voice_id` → error. +- **Port:** `17493`, matching `tauri/src-tauri/src/main.rs:63` (`SERVER_PORT` constant). Shim default with `VOICEBOX_PORT` env override. +- **Non-MCP access:** `POST /speak` is a thin REST wrapper around the same tool path — one endpoint for shell scripts, ACP, A2A, and anything that isn't MCP-native. + +## Library choice + +- **`fastmcp`** (PyPI — verify on install whether the canonical import is `fastmcp` standalone or `mcp.server.fastmcp` from the consolidated `mcp` package; the API is identical). +- **`sse-starlette`** for the `/events/speak` pill-state broadcast. +- **`httpx` + `anyio`** already present — used by the shim. + +## Data model + +New table, **one row per client_id** (not a singleton — scales to unknown clients, maps 1:1 to the Settings UI list): + +```python +# backend/database/models.py +class MCPClientBinding(Base): + __tablename__ = "mcp_client_bindings" + client_id = Column(String, primary_key=True) # "claude-code", "cursor", ... + label = Column(String, nullable=True) + profile_id = Column(String, ForeignKey("profiles.id"), nullable=True) + default_engine = Column(String, nullable=True) + default_intent = Column(String, nullable=True) # "respond" | "rewrite" | "compose" + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) +``` + +Global default stays in `capture_settings.default_playback_voice_id` — no duplication. Migration: new `_migrate_mcp_client_bindings()` in `backend/database/migrations.py` using `CREATE TABLE IF NOT EXISTS`, mirroring the existing idempotent-add-column pattern. + +## File plan + +### Backend — new + +| File | Purpose | +|---|---| +| `backend/mcp/__init__.py` | Package marker | +| `backend/mcp/server.py` | `build_mcp_server()` + `mount_into(app)`; registers tools, middleware, mount at `/mcp` | +| `backend/mcp/tools.py` | The 4 `@mcp.tool()` functions — thin wrappers over existing services | +| `backend/mcp/context.py` | `current_client_id: ContextVar[str \| None]` + Starlette middleware | +| `backend/mcp/resolve.py` | `resolve_profile(explicit, client_id, db) -> VoiceProfile \| None` | +| `backend/mcp/events.py` | In-memory `asyncio.Queue` pub/sub for speak-start / speak-end | +| `backend/mcp/README.md` | MCP Inspector quickstart + `.mcp.json` snippets | +| `backend/mcp_shim/__init__.py`, `__main__.py` | Stdio ↔ Streamable HTTP proxy (~150 lines) | +| `backend/voicebox-mcp.spec` | PyInstaller spec for the shim (strips torch/transformers from `hiddenimports`) | +| `backend/routes/speak.py` | `POST /speak {text, profile?, engine?, intent?, language?}` — REST wrapper around `resolve_profile()` + `speak_in_character()` for non-MCP agents | + +### Backend — modified + +| File | Change | +|---|---| +| `backend/app.py` | Migrate `@app.on_event("startup"/"shutdown")` (lines 185, 268) to `lifespan=` kwarg on `FastAPI()` using `AsyncExitStack`; call `mount_into(application)` after `register_routers`. Register `ClientIdMiddleware`. | +| `backend/routes/profiles.py` | In `speak_in_character` (line 453): `events.publish("speak-start", {...})` on entry; completion hook publishes `speak-end`. Accept optional `source="mcp"` marker. | +| `backend/services/generation.py` | `run_generation` completion path publishes `speak-end`. | +| `backend/services/profiles.py` | New `async def get_profile_by_name_or_id(name_or_id, db)` — id lookup first, case-insensitive name fallback. | +| `backend/database/models.py` | Add `MCPClientBinding`. | +| `backend/database/migrations.py` | Add `_migrate_mcp_client_bindings`. | +| `backend/models.py` | Add `MCPClientBindingResponse`, `MCPClientBindingUpdate`. | +| `backend/routes/__init__.py` | Register `mcp_bindings_router`, `speak_router`, `events_router`. | +| `backend/routes/mcp_bindings.py` (new) | REST CRUD for bindings (list, upsert, delete). | +| `backend/routes/events.py` (new) | `GET /events/speak` — `EventSourceResponse` subscribed to the events queue. | +| `backend/requirements.txt` | `+ fastmcp` (or `mcp>=1.0`), `+ sse-starlette` | +| `backend/voicebox-server.spec` | `hiddenimports += ['mcp', 'mcp.server', 'fastmcp']` | +| `backend/build_binary.py` | Second PyInstaller invocation for `voicebox-mcp.spec`; copy to `tauri/src-tauri/binaries/` with target-triple suffix | + +### Frontend — new + +| File | Purpose | +|---|---| +| `app/src/components/ServerSettings/MCPBindings.tsx` | Settings section — default voice + per-client binding rows + `.mcp.json` copy-paste cheatsheet | +| `app/src/lib/hooks/useMCPBindings.ts` | TanStack Query mirror of `useCaptureSettings` | +| `app/src/lib/api/mcp.ts` | `listMCPBindings` / `upsertMCPBinding` / `deleteMCPBinding` | + +### Frontend — modified + +| File | Change | +|---|---| +| `app/src/components/DictateWindow/DictateWindow.tsx` | Open `EventSource('/events/speak')`; on `speak-start` set pill to `speaking` with profile name; dismiss on `speak-end`. | +| `app/src/components/CapturePill/CapturePill.tsx` | Add `speaking` branch — reuse the active waveform, swap status label to profile name. | +| `app/src/lib/hooks/useCaptureRecordingSession.ts` | Union a `speaking` injection into the derived pill state. | +| `app/src/lib/api/types.ts` | `MCPClientBinding`, `MCPClientBindingUpdate` types. | +| `app/src/components/ServerSettings/index.tsx` | Register the new MCP section in the tab aggregator. | + +### Tauri + +| File | Change | +|---|---| +| `tauri/src-tauri/tauri.conf.json` | `"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"]` | +| `tauri/src-tauri/binaries/voicebox-mcp-` | Build artifact from PyInstaller | + +## Tool signatures + +All tools read `current_client_id.get()` (from middleware). Return JSON-serializable dicts. + +Tools are registered with **dotted names** (`voicebox.speak`, etc.) to match the landing page and the industry convention (`filesystem.read_file`, `github.create_issue`). Python function names stay snake_case; the dot goes in the `name=` kwarg. + +```python +# backend/mcp/tools.py + +@mcp.tool(name="voicebox.speak") +async def speak(text: str, + profile: str | None = None, # name OR id + engine: str | None = None, + intent: str = "respond", # "respond" | "rewrite" | "compose" + language: str | None = None) -> dict: + """Speak text in a voice profile. Returns {generation_id, status, profile, poll}.""" + # resolve profile via precedence, call speak_in_character (profiles.py:453) + # with persist=True so it lands in history. + +@mcp.tool(name="voicebox.transcribe") +async def transcribe(audio_base64: str | None = None, + audio_path: str | None = None, # absolute local path + language: str | None = None, + model: str | None = None) -> dict: + """Transcribe audio. Exactly one of audio_base64/audio_path. Returns {text, duration, language}.""" + # validate path readable, size < 200 MB, then call services.transcribe.transcribe_bytes + +@mcp.tool(name="voicebox.list_captures") +async def list_captures(limit: int = 20, offset: int = 0) -> dict: + """Recent captures with transcripts. Returns {captures: [...]}""" + +@mcp.tool(name="voicebox.list_profiles") +async def list_profiles() -> dict: + """Available voice profiles. Returns {profiles: [{id, name, voice_type, has_personality}]}""" +``` + +### `POST /speak` (non-MCP REST wrapper) + +```python +# backend/routes/speak.py +@router.post("/speak", response_model=GenerationResponse) +async def speak(data: SpeakRequest, request: Request, db: Session = Depends(get_db)): + """Same behavior as the MCP tool — for shell scripts, ACP, A2A, or anything non-MCP.""" + client_id = request.headers.get("X-Voicebox-Client-Id") + profile = resolve_profile(data.profile, client_id, db) + if profile is None: raise HTTPException(400, "No voice profile resolved.") + persist_req = PersonalitySpeakRequest(text=data.text, persist=True, language=data.language, + engine=data.engine, intent=data.intent or "respond") + return await speak_in_character(profile.id, persist_req, db) +``` + +`SpeakRequest`: `{ text: str, profile: str | None, engine: str | None, intent: str | None, language: str | None }`. Accepts name OR id for `profile` (via `resolve_profile`), and resolves via the same precedence as the MCP tool so the two surfaces behave identically. + +## Mount point (`backend/app.py`) + +```python +# After register_routers(application): +from .mcp.server import mount_into +mount_into(application) +``` + +`mount_into` installs `ClientIdMiddleware` and calls `app.mount("/mcp", mcp.streamable_http_app())`. + +**Lifespan migration is load-bearing** — FastMCP's session manager requires the `lifespan=` kwarg, not `@app.on_event`. Wrap the existing startup/shutdown bodies in an `@asynccontextmanager` using `contextlib.AsyncExitStack` so both Voicebox's init and FastMCP's session manager run. Verify dev + packaged build after the migration. + +## Stdio shim (`backend/mcp_shim/__main__.py`) + +1. Port: `int(os.environ.get("VOICEBOX_PORT", "17493"))`. +2. Client id: `os.environ.get("VOICEBOX_CLIENT_ID", "unknown")`. +3. Health probe `GET /health` with 30 s tolerance (torch imports slowly). On failure, emit JSON-RPC error on stdout, exit 1. +4. Connect Streamable HTTP MCP client to `http://127.0.0.1:{port}/mcp` with `X-Voicebox-Client-Id: {client_id}` header. +5. Proxy JSON-RPC bidirectionally — stdin → HTTP, SSE → stdout. Use `mcp` SDK's built-in stdio↔HTTP bridge if available; otherwise ~40 lines of asyncio. +6. Stdout = JSON-RPC only. All logs to stderr. + +PyInstaller spec keeps only `mcp`, `httpx`, `anyio`, `click` — target binary <20 MB. + +## Pill `speaking` state + +- `backend/mcp/events.py`: module-level `_subscribers: list[asyncio.Queue]` + `publish(kind, payload)` + `subscribe() -> Queue`. +- `speak_in_character` publishes `speak-start` with `{generation_id, profile_id, profile_name, source}` immediately after `task_manager.start_generation`; `run_generation`'s completion path publishes `speak-end`. +- `/events/speak` → `EventSourceResponse`. +- `DictateWindow` opens `EventSource` next to existing `dictate:*` listeners, maps `speak-start/end` → pill `speaking` mode with profile name. +- Optional filter: only show pill when `source === "mcp"` (avoids pill churn during manual speak flows). Settings toggle later. + +## Settings UI (`MCPBindings.tsx`) + +- **Global default voice** picker bound to `capture_settings.default_playback_voice_id` (reuses `useCaptureSettings`). +- **Per-client table** — add/edit/remove rows of `{client_id, label, profile_id, default_engine, default_intent}`. Uses `useMCPBindings`. +- **Connection cheatsheet** — two tabs, HTTP (default) and Stdio (fallback), with copy-to-clipboard snippets per known client: + + HTTP form (primary): + ```json + {"mcpServers": {"voicebox": { + "url": "http://127.0.0.1:17493/mcp", + "headers": {"X-Voicebox-Client-Id": "claude-code"} + }}} + ``` + + Stdio form (fallback, absolute path auto-filled from detected app location): + ```json + {"mcpServers": {"voicebox": { + "command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp", + "env": {"VOICEBOX_CLIENT_ID": "claude-code"} + }}} + ``` + + Plus the Claude-Code-specific one-liner: + ``` + claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code" + ``` +- **One-click install buttons** for known clients (v1: Claude Code via `claude mcp add` invocation, and a config-file writer for Cursor/Windsurf whose config locations are known). Each has a matching "Remove" button. Hide buttons for clients not detected on disk. +- **Connection status** — small indicator next to each binding showing the last time that `client_id` actually called the server (rolling timestamp recorded by middleware), so users can tell their install worked. + +## Ordered task list (shortest path first) + +1. `fastmcp` + `sse-starlette` → `backend/requirements.txt`; install. +2. Add `backend/mcp/{server,tools,context,resolve}.py` with the 4 tools registered as `voicebox.speak` etc. (no middleware yet — global default profile only). +3. Migrate `app.py` to `lifespan=`; mount FastMCP at `/mcp`. +4. **Milestone:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp` — call `voicebox.speak`, hear audio. +5. Add `get_profile_by_name_or_id`; wire the tool's `profile` arg. +6. `MCPClientBinding` model + migration; middleware; full `resolve_profile` precedence. +7. `backend/routes/speak.py` — `POST /speak` REST wrapper, reusing `resolve_profile` + `speak_in_character`. +8. `/mcp/bindings` REST + `MCPBindings.tsx` UI with HTTP and stdio copy-snippets, one-click install for detected clients, and connection-status indicators. **Users can install Voicebox as an MCP server after this step.** +9. `backend/mcp_shim/__main__.py` + PyInstaller spec + `build_binary.py` second pass; register `voicebox-mcp` as a Tauri sidecar. (Fallback path goes live.) +10. Events queue + `/events/speak` SSE + `DictateWindow` `speaking` pill state. +11. `backend/mcp/README.md` quickstart. + +Claude Code can call `voicebox.speak` after step 4 (direct HTTP, manual config). Step 8 makes that a one-click experience. Step 9 adds the stdio fallback for clients that don't speak HTTP MCP. + +## Verification + +- **Step 4 smoke:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp`. Call `voicebox.list_profiles`, then `voicebox.speak(text="hello from mcp")`. Audio plays; generation appears in History with `source="personality_speak"` (or new `source="mcp"` if we add one). +- **REST wrapper:** `curl -X POST http://127.0.0.1:17493/speak -d '{"text":"hi","profile":"Morgan"}'` — same behavior, same pill surface. +- **Per-client:** open two Inspector sessions with different `X-Voicebox-Client-Id` headers, bind each to a different profile in Settings, verify distinct voices without `profile` arg. +- **Claude Code end-to-end (HTTP):** `claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code"`, then ask Claude Code to speak. Pill shows `speaking: `, audio plays, capture appears in history. +- **Stdio fallback:** manually paste the stdio snippet from Settings into a client's config, verify same behavior. `VOICEBOX_CLIENT_ID=claude-code python -m backend.mcp_shim` while backend is up; pipe a tools/list JSON-RPC in, verify response over stdout. +- **Transcribe:** point at `/tmp/test.wav`; diff against `POST /transcribe` response. +- **Failure modes:** kill backend mid-speak — shim must surface a JSON-RPC error, not deadlock. When backend isn't running, HTTP clients should get a clear connection-refused surfaced by the client. + +## Risks / open decisions + +- **`fastmcp` vs `mcp` package name** — confirm on `pip install`; APIs are near-identical, adjust imports. +- **Lifespan migration** touches critical path (DB init, task queue, watchdog). Dev + packaged build both need a smoke after. +- **Shim binary size** — if `mcp` pulls in enough dep weight that PyInstaller output is awkward, fall back to a Rust shim (Tauri shell is already Rust; JSON-RPC framing is trivial). +- **Source attribution** — consider `source="mcp"` on the `Generation` model, or a dedicated `originator_client` column, if the Captures tab should filter MCP-originated generations. +- **`audio_path` in `voicebox_transcribe`** — local-only today, but if the server ever binds beyond 127.0.0.1 we need to restrict reads to `data_dir` + user-whitelist. +- **Auth** — none for now (127.0.0.1 only). If we bind outside, bearer token via `~/.voicebox/secret` + plumb through shim. +- **HTTP MCP client support** — the plan leads with direct HTTP. Claude Code, Cursor, Windsurf, and VS Code MCP extensions all support it as of 2026, but if we discover an important client is stdio-only we still have the shim fallback ready. +- **`.mcpb` desktop extension for Claude Desktop** (v2 polish) — Claude Desktop supports a double-clickable extension bundle format. Worth revisiting after v1 ships for an even cleaner install; skipped for now since Claude Desktop isn't the primary user (Claude Code + IDE users are). + +## Critical files + +- `backend/app.py` +- `backend/routes/profiles.py` +- `backend/routes/speak.py` (new) +- `backend/database/models.py` +- `backend/database/migrations.py` +- `backend/services/generation.py` +- `backend/build_binary.py` +- `tauri/src-tauri/tauri.conf.json` +- `tauri/src-tauri/src/main.rs` (port constant — no change, just reference) +- `app/src/components/DictateWindow/DictateWindow.tsx` +- `app/src/components/CapturePill/CapturePill.tsx` +- `app/src/components/ServerSettings/` diff --git a/landing/src/components/AgentIntegration.tsx b/landing/src/components/AgentIntegration.tsx index ecf8d0aa..305cea3f 100644 --- a/landing/src/components/AgentIntegration.tsx +++ b/landing/src/components/AgentIntegration.tsx @@ -60,18 +60,7 @@ const TONE_CLASSES: Record = { // ─── Console mockup ───────────────────────────────────────────────────────── -function AgentConsole() { - const [idx, setIdx] = useState(0); - - useEffect(() => { - const iv = window.setInterval(() => { - setIdx((i) => (i + 1) % SCENARIOS.length); - }, 4200); - return () => window.clearInterval(iv); - }, []); - - const scenario = SCENARIOS[idx]; - +function AgentConsole({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) { return (
{/* Titlebar */} @@ -88,12 +77,11 @@ function AgentConsole() {
{/* Body */} -
- {/* Log lines */} -
+
+
{scenario.log.map((line, i) => ( - {/* The pill in speaking state — the payoff */} + {/* Idle cursor so the terminal doesn't feel empty */} +
+ $ + +
+
+
+ ); +} + +// ─── Desktop-floating pill stage ──────────────────────────────────────────── + +function AgentSpeakStage({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) { + return ( +
+ {/* Caption in the corner — "this is on the desktop, not in a terminal" */} +
+ On your desktop +
+ + {/* Voice-tinted glow behind the pill */} +
+
+ + {/* Pill + utterance caption */} +
+
- + Speaking · {scenario.voice} -
+
{[0, 1, 2, 3, 4, 5].map((i) => ( @@ -145,13 +177,12 @@ function AgentConsole() {
- {/* The utterance — what the agent said */} “{scenario.utterance}” @@ -165,8 +196,7 @@ function AgentConsole() { const MCP_CONFIG = `{ "mcpServers": { "voicebox": { - "command": "voicebox", - "args": ["mcp"] + "url": "http://127.0.0.1:17493/mcp" } } }`; @@ -246,13 +276,24 @@ const BULLETS = [ // ─── Section ──────────────────────────────────────────────────────────────── export function AgentIntegration() { + const [idx, setIdx] = useState(0); + + useEffect(() => { + const iv = window.setInterval(() => { + setIdx((i) => (i + 1) % SCENARIOS.length); + }, 4200); + return () => window.clearInterval(iv); + }, []); + + const scenario = SCENARIOS[idx]; + return ( -
+
{/* Header */}
- Agents + MCP

Every agent gets a voice. @@ -265,10 +306,13 @@ export function AgentIntegration() {

- {/* Code + console split */} -
+ {/* Code (left) + console with pill stage stacked underneath (right) */} +
- +
+ + +
{/* Bullets */} diff --git a/landing/src/components/Footer.tsx b/landing/src/components/Footer.tsx index da7b6b75..172889eb 100644 --- a/landing/src/components/Footer.tsx +++ b/landing/src/components/Footer.tsx @@ -40,8 +40,28 @@ export function Footer() {

Product

diff --git a/landing/src/components/Navbar.tsx b/landing/src/components/Navbar.tsx index a3158c5e..565937d2 100644 --- a/landing/src/components/Navbar.tsx +++ b/landing/src/components/Navbar.tsx @@ -51,7 +51,7 @@ export function Navbar() { href="/#features" className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground" > - Features + Clone - Agents + MCP Bindings { - #[cfg(target_os = "macos")] - let (m1, m2) = (Key::MetaRight, Key::AltGr); - #[cfg(not(target_os = "macos"))] - let (m1, m2) = (Key::ControlRight, Key::ShiftRight); - - let mut b = Bindings::new(); - b.insert(ChordAction::PushToTalk, { - let mut s = HashSet::new(); - s.insert(m1); - s.insert(m2); - s - }); - b.insert(ChordAction::ToggleToTalk, { - let mut s = HashSet::new(); - s.insert(m1); - s.insert(m2); - s.insert(Key::Space); - s - }); - b -} - pub struct HotkeyMonitor { chord: Arc>, } impl HotkeyMonitor { pub fn spawn(app: AppHandle, bindings: Bindings) -> Self { + eprintln!("[HotkeyMonitor] spawn() called with {} bindings", bindings.len()); let chord = Arc::new(Mutex::new(Chord::new(bindings))); let chord_for_thread = chord.clone(); let app_for_thread = app.clone(); @@ -219,7 +184,9 @@ impl HotkeyMonitor { #[cfg(target_os = "macos")] rdev::set_is_main_thread(false); + eprintln!("[HotkeyMonitor] background thread entering rdev::listen"); let result = listen(move |event| { + eprintln!("[HotkeyMonitor] rdev event: {:?}", event.event_type); let input = match event.event_type { EventType::KeyPress(k) => KeyEvent::Down(k), EventType::KeyRelease(k) => KeyEvent::Up(k), @@ -231,11 +198,17 @@ impl HotkeyMonitor { Err(_) => return, }; + if !effects.is_empty() { + eprintln!("[HotkeyMonitor] chord matched, effects: {:?}", effects); + } + for effect in effects { apply_effect(&app_for_thread, effect); } }); + // listen() blocks forever on success; reaching here means it errored. + eprintln!("[HotkeyMonitor] rdev::listen returned (this only happens on error): {:?}", result); if let Err(err) = result { eprintln!( "HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.", diff --git a/tauri/src-tauri/src/input_monitoring.rs b/tauri/src-tauri/src/input_monitoring.rs new file mode 100644 index 00000000..eed15daa --- /dev/null +++ b/tauri/src-tauri/src/input_monitoring.rs @@ -0,0 +1,82 @@ +//! Platform permission gate for the global keyboard tap. +//! +//! On macOS 10.15+, creating a CGEventTap that observes keyboard events +//! requires the host process to be listed under System Settings → Privacy & +//! Security → Input Monitoring. Without that trust, `rdev::listen` returns +//! immediately and no key events ever flow through the chord engine. +//! +//! The relevant TCC pair lives in IOKit, mirroring `AXIsProcessTrusted` / +//! `AXIsProcessTrustedWithOptions` on the Accessibility side: +//! +//! - `IOHIDCheckAccess(kIOHIDRequestTypeListenEvent)` — read the current +//! grant without prompting. We call this from the Captures settings UI +//! so the row can show "granted" / "missing" without surprising the user. +//! - `IOHIDRequestAccess(kIOHIDRequestTypeListenEvent)` — fire the +//! "Voicebox would like to receive keystrokes from any application" +//! dialog and add Voicebox to the Input Monitoring pane (toggle off). +//! Returns true when access is already granted; otherwise returns false +//! and queues the prompt. The user still has to flip the toggle on; this +//! just gets us into the list. +//! +//! `enable_hotkey` calls `request` on first invocation so the prompt fires +//! from a deterministic, user-initiated point (the Captures toggle) instead +//! of as a side-effect of `rdev::listen` creating its CGEventTap. +//! +//! Windows / Linux don't gate keyboard taps behind a TCC-style permission, +//! so those branches return `true`. + +#[cfg(target_os = "macos")] +mod ffi { + use std::os::raw::c_uint; + + /// `kIOHIDRequestTypeListenEvent` from `` — + /// the request-type discriminator for "I want to read keyboard / mouse + /// events created by other processes." + pub const REQUEST_TYPE_LISTEN_EVENT: c_uint = 1; + + /// `kIOHIDAccessTypeGranted` from `IOHIDLib.h`. The other values are + /// `Denied = 1` and `Unknown = 2`; we only ever care about the granted + /// case so they don't get their own constants. + pub const ACCESS_TYPE_GRANTED: c_uint = 0; + + #[link(name = "IOKit", kind = "framework")] + extern "C" { + /// Returns the current access state as an `IOHIDAccessType` enum + /// (Granted=0, Denied=1, Unknown=2). No prompt side-effect. + /// + /// Declared as `c_uint` rather than `bool`: the C signature returns + /// the full enum, and reading a 3-valued enum into Rust's 1-bit + /// `bool` is undefined behaviour that silently inverts our gate. + pub fn IOHIDCheckAccess(request_type: c_uint) -> c_uint; + + /// Returns true when access is already granted; otherwise queues + /// the system prompt and returns false synchronously. Safe to call + /// repeatedly — once the entry exists in the Input Monitoring pane + /// macOS won't re-prompt. Real `Boolean` (UInt8) return on the C + /// side, so `bool` here is correct. + pub fn IOHIDRequestAccess(request_type: c_uint) -> bool; + } +} + +#[cfg(target_os = "macos")] +pub fn is_trusted() -> bool { + unsafe { ffi::IOHIDCheckAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) == ffi::ACCESS_TYPE_GRANTED } +} + +/// Fire the Input Monitoring prompt if not already granted. Returns the +/// current grant state; a `false` here means the prompt was queued and the +/// user needs to flip the toggle in System Settings before key events flow. +#[cfg(target_os = "macos")] +pub fn request() -> bool { + unsafe { ffi::IOHIDRequestAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) } +} + +#[cfg(not(target_os = "macos"))] +pub fn is_trusted() -> bool { + true +} + +#[cfg(not(target_os = "macos"))] +pub fn request() -> bool { + true +} diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index 6994ef0d..7bfb08e2 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -8,6 +8,7 @@ mod clipboard; mod focus_capture; #[cfg(desktop)] mod hotkey_monitor; +mod input_monitoring; #[cfg(desktop)] mod key_codes; mod synthetic_keys; @@ -57,6 +58,39 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result bool { accessibility::is_trusted() } -/// Push a new chord configuration into the running `HotkeyMonitor`. The -/// frontend calls this both at startup (replaying the saved chord from -/// capture_settings) and any time the user edits the chord in the picker — -/// no app restart needed because the engine swap is atomic under the -/// monitor's mutex. -/// -/// Returns an error when a key name doesn't map to an `rdev::Key`, so the -/// picker UI can surface "this key isn't supported" instead of silently -/// dropping it from the chord. -#[cfg(desktop)] +/// Reports whether the process can observe global keyboard events. Read by +/// the Captures settings UI to surface a "missing — open Settings" hint +/// beside the hotkey toggle. No prompt side-effect. #[command] -fn update_chord_bindings( - monitor: State<'_, hotkey_monitor::HotkeyMonitor>, - push_to_talk: Vec, - toggle_to_talk: Vec, -) -> Result<(), String> { +fn check_input_monitoring_permission() -> bool { + input_monitoring::is_trusted() +} + +/// Holds the lazily-spawned global hotkey monitor. The monitor is `None` +/// until the user opts in via the Captures settings toggle — that opt-in is +/// what triggers the macOS Input Monitoring TCC prompt, so a fresh-install +/// user who never enables the hotkey never sees the prompt. +/// +/// Once spawned, the monitor stays alive for the rest of the process: rdev's +/// `listen` blocks forever and offers no stop signal. "Disable" therefore +/// swaps the chord engine to empty bindings (matches nothing, fires nothing) +/// rather than tearing down the CGEventTap. +#[cfg(desktop)] +#[derive(Default)] +pub struct HotkeyState { + monitor: Mutex>, +} + +#[cfg(desktop)] +fn build_chord_bindings( + push_to_talk: &[String], + toggle_to_talk: &[String], +) -> Result { use hotkey_monitor::{Bindings, ChordAction}; use rdev::Key; use std::collections::HashSet; @@ -824,14 +870,103 @@ fn update_chord_bindings( Ok(chord) } - let push_chord = build_chord("push-to-talk", &push_to_talk)?; - let toggle_chord = build_chord("toggle-to-talk", &toggle_to_talk)?; + let push_chord = build_chord("push-to-talk", push_to_talk)?; + let toggle_chord = build_chord("toggle-to-talk", toggle_to_talk)?; let mut bindings = Bindings::new(); bindings.insert(ChordAction::PushToTalk, push_chord); bindings.insert(ChordAction::ToggleToTalk, toggle_chord); + Ok(bindings) +} - monitor.update_bindings(bindings); +/// Spawn the global hotkey monitor on first call; subsequent calls just push +/// the new bindings into the existing monitor. Idempotent on purpose — the +/// frontend invokes this both at startup (when `capture_settings.hotkey_enabled` +/// is true) and from the settings toggle. +/// +/// On macOS this is the call that triggers the "Voicebox would like to receive +/// keystrokes from any application" TCC prompt, since `rdev::listen` creates +/// the CGEventTap inside `HotkeyMonitor::spawn`. +#[cfg(desktop)] +#[command] +fn enable_hotkey( + app: tauri::AppHandle, + state: State<'_, HotkeyState>, + push_to_talk: Vec, + toggle_to_talk: Vec, +) -> Result<(), String> { + eprintln!("[enable_hotkey] called: push={:?}, toggle={:?}", push_to_talk, toggle_to_talk); + let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?; + + // Fire the Input Monitoring TCC prompt explicitly from the user's + // toggle click, before rdev::listen would do it implicitly via + // CGEventTap creation. Two reasons: (1) the prompt timing becomes + // deterministic — it appears in response to a click instead of as a + // mysterious side-effect of "the app started"; (2) on subsequent + // launches we can short-circuit the spawn entirely if the user + // revoked the grant, instead of leaning on rdev silently failing. + // The call returns the current grant state; we ignore it because + // rdev::listen will surface its own error via stderr, and the + // settings UI polls `check_input_monitoring_permission` separately. + let granted = input_monitoring::request(); + eprintln!("[enable_hotkey] IOHIDRequestAccess returned granted={}", granted); + eprintln!("[enable_hotkey] IOHIDCheckAccess says trusted={}", input_monitoring::is_trusted()); + + // The dictate pill webview must exist before the first chord fires so it + // can subscribe to `dictate:start`. Build it here (idempotent — Tauri + // returns the existing window when one with this label already exists). + if app.get_webview_window(DICTATE_WINDOW_LABEL).is_none() { + if let Err(e) = build_dictate_window(&app) { + eprintln!("Failed to build dictate window: {}", e); + } + } + + let mut slot = state.monitor.lock().map_err(|e| e.to_string())?; + match slot.as_ref() { + Some(monitor) => monitor.update_bindings(bindings), + None => { + *slot = Some(hotkey_monitor::HotkeyMonitor::spawn(app, bindings)); + } + } + Ok(()) +} + +/// Quiet the global hotkey by swapping the chord engine to empty bindings. +/// The CGEventTap stays alive (rdev::listen has no stop) but the chord state +/// machine matches nothing, so no `dictate:*` events fire and the dictate +/// pill never shows. A subsequent `enable_hotkey` call re-arms it without +/// re-prompting for permission. +#[cfg(desktop)] +#[command] +fn disable_hotkey(state: State<'_, HotkeyState>) -> Result<(), String> { + let slot = state.monitor.lock().map_err(|e| e.to_string())?; + if let Some(monitor) = slot.as_ref() { + monitor.update_bindings(hotkey_monitor::Bindings::new()); + } + Ok(()) +} + +/// Push a new chord configuration into the running `HotkeyMonitor`. Called +/// by the chord-picker UI when the user edits the chord. No-ops when the +/// monitor isn't spawned — the picker is gated behind the enable toggle, so +/// this can only happen if the frontend races; the next `enable_hotkey` will +/// pick up the saved chords. +/// +/// Returns an error when a key name doesn't map to an `rdev::Key`, so the +/// picker UI can surface "this key isn't supported" instead of silently +/// dropping it from the chord. +#[cfg(desktop)] +#[command] +fn update_chord_bindings( + state: State<'_, HotkeyState>, + push_to_talk: Vec, + toggle_to_talk: Vec, +) -> Result<(), String> { + let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?; + let slot = state.monitor.lock().map_err(|e| e.to_string())?; + if let Some(monitor) = slot.as_ref() { + monitor.update_bindings(bindings); + } Ok(()) } @@ -855,6 +990,26 @@ fn open_accessibility_settings(app: tauri::AppHandle) -> Result<(), String> { } } +/// Open the Privacy & Security → Input Monitoring pane in System Settings. +/// Used by the Captures settings UI when the toggle is on but the grant +/// is missing, so the user can flip the system toggle without hunting. +#[command] +fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + let url = "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent"; + app.shell() + .open(url, None) + .map_err(|e| format!("Failed to open Input Monitoring settings: {e}"))?; + Ok(()) + } + #[cfg(not(target_os = "macos"))] + { + let _ = app; + Err("Input Monitoring settings pane is only implemented on macOS".into()) + } +} + /// Deliver `text` into the UI that had focus when the chord fired. /// /// Pipeline: activate the captured PID → settle → save the user's @@ -1044,18 +1199,12 @@ pub fn run() { app.handle().plugin(tauri_plugin_updater::Builder::new().build())?; app.handle().plugin(tauri_plugin_process::init())?; - if let Err(e) = build_dictate_window(app.handle()) { - eprintln!("Failed to pre-create dictate window: {}", e); - } - - let monitor = hotkey_monitor::HotkeyMonitor::spawn( - app.handle().clone(), - hotkey_monitor::default_bindings(), - ); - // Stored as state so the chord-picker UI can call - // `update_chord_bindings` to live-swap the engine's chords - // without restarting the listener thread. - app.manage(monitor); + // HotkeyMonitor is spawned lazily via the `enable_hotkey` + // command — see HotkeyState. The dictate pill webview is + // built in the same lazy path so we don't pay setup cost + // (and don't trigger the macOS Input Monitoring TCC prompt) + // for users who never enable the global hotkey. + app.manage(HotkeyState::default()); // The frontend emits `dictate:hide` whenever the pill cycle // finishes (rest-fade → hidden). `hide()` alone has been @@ -1073,6 +1222,16 @@ pub fn run() { let _ = window.hide(); } }); + + // Agent-initiated speech (voicebox.speak over MCP or POST /speak) + // pops the pill up so the user can see what's coming out of their + // machine. The DictateWindow subscribes to /events/speak via SSE + // and emits `dictate:show` on speak-start; we repeat the same + // position+show dance the hotkey path uses. + let handle_for_show = app.handle().clone(); + app.handle().listen("dictate:show", move |_event| { + show_dictate_window(&handle_for_show); + }); } // Hide title bar icon on Windows @@ -1148,8 +1307,12 @@ pub fn run() { debug_capture_focus, debug_focus_roundtrip, check_accessibility_permission, + check_input_monitoring_permission, open_accessibility_settings, + open_input_monitoring_settings, paste_final_text, + enable_hotkey, + disable_hotkey, update_chord_bindings ]) .on_window_event({ diff --git a/tauri/src-tauri/tauri.conf.json b/tauri/src-tauri/tauri.conf.json index 5e940f52..9c055848 100644 --- a/tauri/src-tauri/tauri.conf.json +++ b/tauri/src-tauri/tauri.conf.json @@ -13,7 +13,7 @@ "active": true, "targets": "all", "createUpdaterArtifacts": "v1Compatible", - "externalBin": ["binaries/voicebox-server"], + "externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"], "icon": [ "icons/32x32.png", "icons/128x128.png",