mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-10-03 09:05:17 -07:00
feat(mcp): local MCP server exposes voicebox.* tools to AI agents
Mounts FastMCP at /mcp (Streamable HTTP) so Claude Code, Cursor, Windsurf, and the VS Code MCP extensions can call voicebox.speak, voicebox.transcribe, voicebox.list_captures, and voicebox.list_profiles against the running Voicebox server. Backend - new backend/mcp_server package (tools, middleware, profile resolve, pub/sub events); named mcp_server to avoid shadowing the installed mcp PyPI package FastMCP imports internally - app.py migrated from @app.on_event to lifespan= so FastMCP's session manager cohabits with Voicebox's startup/shutdown - new MCPClientBinding table + /mcp/bindings CRUD; ClientIdMiddleware reads X-Voicebox-Client-Id into a ContextVar and stamps last_seen_at - profile resolution precedence: explicit -> per-client binding -> capture_settings.default_playback_voice_id - POST /speak REST wrapper for non-MCP callers (shell, ACP, A2A) - GET /events/speak SSE broadcasts speak-start / speak-end so the pill surfaces agent-initiated speech - backend/mcp_shim proxy (plain httpx) for stdio-only MCP clients - PyInstaller spec updates + new --shim build target (~18 MB) Frontend - Settings -> MCP page with HTTP / stdio / claude-mcp-add copy snippets, default voice picker, per-client bindings table, connection status - useMCPBindings, useSpeakEvents hooks - CapturePill gains 'speaking' state; DictateWindow subscribes to SSE and emits dictate:show so the Rust side surfaces the pill window Native - tauri.conf.json externalBin now includes voicebox-mcp - show_dictate_window helper + dictate:show listener in main.rs - (also in this commit: InputMonitoringGate UX, hotkey_monitor tweaks, landing footer/navbar updates, new overview docs for captures / dictation / mcp-server / voice-personalities) Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
87c582ad54
commit
0cef2c9fe1
@@ -10,6 +10,7 @@ export type PillState =
|
|||||||
| 'recording'
|
| 'recording'
|
||||||
| 'transcribing'
|
| 'transcribing'
|
||||||
| 'refining'
|
| 'refining'
|
||||||
|
| 'speaking'
|
||||||
| 'completed'
|
| 'completed'
|
||||||
| 'rest'
|
| 'rest'
|
||||||
| 'error';
|
| 'error';
|
||||||
@@ -18,13 +19,14 @@ const PILL_LABELS: Record<Exclude<PillState, 'rest' | 'error'>, string> = {
|
|||||||
recording: 'Recording',
|
recording: 'Recording',
|
||||||
transcribing: 'Transcribing',
|
transcribing: 'Transcribing',
|
||||||
refining: 'Refining',
|
refining: 'Refining',
|
||||||
|
speaking: 'Speaking',
|
||||||
completed: 'Done',
|
completed: 'Done',
|
||||||
};
|
};
|
||||||
|
|
||||||
function barModeFor(
|
function barModeFor(
|
||||||
state: Exclude<PillState, 'error'>,
|
state: Exclude<PillState, 'error'>,
|
||||||
): 'generating' | 'playing' | 'idle' {
|
): 'generating' | 'playing' | 'idle' {
|
||||||
if (state === 'recording') return 'playing';
|
if (state === 'recording' || state === 'speaking') return 'playing';
|
||||||
if (state === 'completed' || state === 'rest') return 'idle';
|
if (state === 'completed' || state === 'rest') return 'idle';
|
||||||
return 'generating';
|
return 'generating';
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui';
|
|||||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||||
import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
||||||
import { cn } from '@/lib/utils/cn';
|
import { cn } from '@/lib/utils/cn';
|
||||||
|
import { displayLabelForKey, modifierSideHint } from '@/lib/utils/keyCodes';
|
||||||
import { usePlayerStore } from '@/stores/playerStore';
|
import { usePlayerStore } from '@/stores/playerStore';
|
||||||
|
|
||||||
const CAPTURE_AUDIO_MIME = 'audio/*,.wav,.mp3,.m4a,.flac,.ogg,.webm';
|
const CAPTURE_AUDIO_MIME = 'audio/*,.wav,.mp3,.m4a,.flac,.ogg,.webm';
|
||||||
@@ -86,6 +87,30 @@ function snippetOf(capture: CaptureResponse): string {
|
|||||||
return source.trim() || '(no transcript)';
|
return source.trim() || '(no transcript)';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ChordKeys({ keys }: { keys: string[] }) {
|
||||||
|
if (keys.length === 0) return null;
|
||||||
|
return (
|
||||||
|
<div className="flex items-center gap-1">
|
||||||
|
{keys.map((k) => {
|
||||||
|
const side = modifierSideHint(k);
|
||||||
|
return (
|
||||||
|
<span
|
||||||
|
key={k}
|
||||||
|
className="relative inline-flex items-center justify-center h-6 min-w-[1.5rem] px-1.5 rounded-md border border-border bg-muted/60 font-mono text-[11px] font-medium shadow-sm text-foreground"
|
||||||
|
>
|
||||||
|
{displayLabelForKey(k)}
|
||||||
|
{side ? (
|
||||||
|
<span className="absolute -top-1 -right-1 h-3 min-w-[0.75rem] px-0.5 rounded-sm bg-accent text-[7px] font-bold leading-none flex items-center justify-center text-accent-foreground">
|
||||||
|
{side}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function SourceBadge({ source }: { source: CaptureSource }) {
|
function SourceBadge({ source }: { source: CaptureSource }) {
|
||||||
const Icon = source === 'dictation' ? Mic : source === 'recording' ? CircleDot : FileAudio;
|
const Icon = source === 'dictation' ? Mic : source === 'recording' ? CircleDot : FileAudio;
|
||||||
const label = source === 'dictation' ? 'Dictation' : source === 'recording' ? 'Recording' : 'File';
|
const label = source === 'dictation' ? 'Dictation' : source === 'recording' ? 'Recording' : 'File';
|
||||||
@@ -160,6 +185,9 @@ export function CapturesTab() {
|
|||||||
const { settings: captureSettings } = useCaptureSettings();
|
const { settings: captureSettings } = useCaptureSettings();
|
||||||
const sttModel = captureSettings?.stt_model ?? 'turbo';
|
const sttModel = captureSettings?.stt_model ?? 'turbo';
|
||||||
const llmModel = captureSettings?.llm_model ?? '0.6B';
|
const llmModel = captureSettings?.llm_model ?? '0.6B';
|
||||||
|
const hotkeyEnabled = captureSettings?.hotkey_enabled ?? false;
|
||||||
|
const pushToTalkKeys = captureSettings?.chord_push_to_talk_keys ?? [];
|
||||||
|
const toggleToTalkKeys = captureSettings?.chord_toggle_to_talk_keys ?? [];
|
||||||
|
|
||||||
const session = useCaptureRecordingSession({
|
const session = useCaptureRecordingSession({
|
||||||
onCaptureCreated: (capture) => setSelectedId(capture.id),
|
onCaptureCreated: (capture) => setSelectedId(capture.id),
|
||||||
@@ -394,17 +422,11 @@ export function CapturesTab() {
|
|||||||
<Loader2 className="h-4 w-4 animate-spin" />
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
</div>
|
</div>
|
||||||
) : filtered.length === 0 ? (
|
) : filtered.length === 0 ? (
|
||||||
<div className="px-4 py-12 text-center text-sm text-muted-foreground space-y-3">
|
<div className="px-4 py-12 text-center text-sm text-muted-foreground">
|
||||||
{search ? (
|
{search ? (
|
||||||
<p>No captures match "{search}"</p>
|
<p>No captures match "{search}"</p>
|
||||||
) : (
|
) : (
|
||||||
<>
|
|
||||||
<p>No captures yet.</p>
|
<p>No captures yet.</p>
|
||||||
<Button variant="outline" size="sm" onClick={handleUploadClick}>
|
|
||||||
<Upload className="h-3.5 w-3.5 mr-1.5" />
|
|
||||||
Import audio
|
|
||||||
</Button>
|
|
||||||
</>
|
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
@@ -744,22 +766,53 @@ export function CapturesTab() {
|
|||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<div className="flex-1 flex items-center justify-center text-muted-foreground pt-20">
|
<div className="flex-1 flex items-center justify-center text-muted-foreground pt-20">
|
||||||
|
{capturesLoading ? (
|
||||||
<div className="text-center space-y-3">
|
<div className="text-center space-y-3">
|
||||||
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||||
{capturesLoading ? (
|
|
||||||
<p className="text-sm">Loading captures…</p>
|
<p className="text-sm">Loading captures…</p>
|
||||||
) : captures.length ? (
|
|
||||||
<p className="text-sm">Pick a capture to see the transcript.</p>
|
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
<p className="text-sm">No captures yet.</p>
|
|
||||||
<Button variant="outline" size="sm" onClick={handleUploadClick}>
|
|
||||||
<Upload className="h-3.5 w-3.5 mr-1.5" />
|
|
||||||
Import audio
|
|
||||||
</Button>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
|
) : captures.length ? (
|
||||||
|
<div className="text-center space-y-3">
|
||||||
|
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||||
|
<p className="text-sm">Pick a capture to see the transcript.</p>
|
||||||
|
</div>
|
||||||
|
) : hotkeyEnabled && (pushToTalkKeys.length || toggleToTalkKeys.length) ? (
|
||||||
|
<div className="max-w-sm mx-auto text-center space-y-5">
|
||||||
|
<div className="space-y-2">
|
||||||
|
{pushToTalkKeys.length ? (
|
||||||
|
<div className="flex items-center justify-center gap-3">
|
||||||
|
<ChordKeys keys={pushToTalkKeys} />
|
||||||
|
<span className="text-[11px] uppercase tracking-wider text-muted-foreground">
|
||||||
|
Hold to record
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{toggleToTalkKeys.length ? (
|
||||||
|
<div className="flex items-center justify-center gap-3">
|
||||||
|
<ChordKeys keys={toggleToTalkKeys} />
|
||||||
|
<span className="text-[11px] uppercase tracking-wider text-muted-foreground">
|
||||||
|
Toggle hands-free
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
<p className="text-sm">
|
||||||
|
Press the shortcut anywhere on your machine to start your first capture.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="max-w-sm mx-auto text-center space-y-3">
|
||||||
|
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||||
|
<p className="text-sm">No captures yet.</p>
|
||||||
|
<p className="text-xs text-muted-foreground leading-relaxed">
|
||||||
|
Turn on the global shortcut to dictate from anywhere — or click
|
||||||
|
Dictate above for an in-app capture.
|
||||||
|
</p>
|
||||||
|
<Button asChild variant="outline" size="sm">
|
||||||
|
<Link to="/settings/captures">Open Captures settings</Link>
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { useEffect, useRef } from 'react';
|
|||||||
import { CapturePill } from '@/components/CapturePill/CapturePill';
|
import { CapturePill } from '@/components/CapturePill/CapturePill';
|
||||||
import type { FocusSnapshot } from '@/lib/api/types';
|
import type { FocusSnapshot } from '@/lib/api/types';
|
||||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||||
|
import { useSpeakEvents } from '@/lib/hooks/useSpeakEvents';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Floating dictate surface shown in a separate transparent Tauri window.
|
* Floating dictate surface shown in a separate transparent Tauri window.
|
||||||
@@ -82,27 +83,47 @@ export function DictateWindow() {
|
|||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// When the pill cycle ends, tell Rust to tuck the window away. The Rust
|
// Subscribe to agent-initiated speak events so the pill surfaces while
|
||||||
// side is responsible for the hide + park-off-screen + click-through
|
// voicebox.speak (MCP) or POST /speak is producing audio. We ask Rust
|
||||||
// combo because calling hide() directly from JS has been unreliable for
|
// to show the pill window by emitting `dictate:show` — the existing
|
||||||
// transparent always-on-top windows on macOS. Showing is the reverse —
|
// `dictate:hide` happens on cycle end below.
|
||||||
// the HotkeyMonitor restores position, clicks, and visibility when a
|
const speaking = useSpeakEvents();
|
||||||
// chord next fires.
|
const prevSpeakingIdRef = useRef<string | null>(null);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (session.pillState === 'hidden') {
|
const id = speaking?.generationId ?? null;
|
||||||
|
if (id && id !== prevSpeakingIdRef.current) {
|
||||||
|
emit('dictate:show').catch(() => {});
|
||||||
|
}
|
||||||
|
prevSpeakingIdRef.current = id;
|
||||||
|
}, [speaking?.generationId]);
|
||||||
|
|
||||||
|
// Compose the effective pill state: speak events override the capture
|
||||||
|
// session when both would render, because agent speech is always
|
||||||
|
// category-mattering (the user can't hear two pills). Elapsed time
|
||||||
|
// restarts for speaking so the timer reflects playback length.
|
||||||
|
const isSpeaking = Boolean(speaking);
|
||||||
|
const effectiveState = isSpeaking ? 'speaking' : session.pillState;
|
||||||
|
const effectiveElapsed = isSpeaking ? speaking!.elapsedMs : session.pillElapsedMs;
|
||||||
|
|
||||||
|
// When the pill cycle ends (no capture AND no speak), tell Rust to tuck
|
||||||
|
// the window away. Rust owns the hide + park-off-screen + click-through
|
||||||
|
// combo because calling hide() directly from JS has been unreliable for
|
||||||
|
// transparent always-on-top windows on macOS.
|
||||||
|
useEffect(() => {
|
||||||
|
if (effectiveState === 'hidden') {
|
||||||
emit('dictate:hide').catch(() => {});
|
emit('dictate:hide').catch(() => {});
|
||||||
}
|
}
|
||||||
}, [session.pillState]);
|
}, [effectiveState]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
className="h-screen w-screen flex items-center justify-center px-3"
|
className="h-screen w-screen flex items-center justify-center px-3"
|
||||||
style={{ background: 'transparent' }}
|
style={{ background: 'transparent' }}
|
||||||
>
|
>
|
||||||
{session.pillState !== 'hidden' ? (
|
{effectiveState !== 'hidden' ? (
|
||||||
<CapturePill
|
<CapturePill
|
||||||
state={session.pillState}
|
state={effectiveState}
|
||||||
elapsedMs={session.pillElapsedMs}
|
elapsedMs={effectiveElapsed}
|
||||||
errorMessage={session.errorMessage}
|
errorMessage={session.errorMessage}
|
||||||
onDismiss={session.dismissError}
|
onDismiss={session.dismissError}
|
||||||
onStop={session.isRecording ? session.stopRecording : undefined}
|
onStop={session.isRecording ? session.stopRecording : undefined}
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import { invoke } from '@tauri-apps/api/core';
|
||||||
|
import { AlertTriangle, ExternalLink } from 'lucide-react';
|
||||||
|
import { useCallback, useEffect, useState } from 'react';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { usePlatform } from '@/platform/PlatformContext';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tracks macOS Input Monitoring permission state. Without it, `rdev::listen`
|
||||||
|
* sees no key events and the chord engine never fires — but neither does
|
||||||
|
* anything error-out visibly, so we surface an inline prompt next to the
|
||||||
|
* hotkey toggle instead of leaving the user wondering why the shortcut is
|
||||||
|
* dead.
|
||||||
|
*
|
||||||
|
* Re-checked on mount and on window focus (cheap way to pick up the user
|
||||||
|
* flipping the toggle in System Settings and alt-tabbing back).
|
||||||
|
*/
|
||||||
|
export function useInputMonitoringPermission() {
|
||||||
|
const platform = usePlatform();
|
||||||
|
const [needsPermission, setNeedsPermission] = useState(false);
|
||||||
|
const [checking, setChecking] = useState(false);
|
||||||
|
|
||||||
|
const recheck = useCallback(async (): Promise<boolean> => {
|
||||||
|
if (!platform.metadata.isTauri) return true;
|
||||||
|
setChecking(true);
|
||||||
|
try {
|
||||||
|
const trusted = await invoke<boolean>('check_input_monitoring_permission');
|
||||||
|
setNeedsPermission(!trusted);
|
||||||
|
return trusted;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[input-monitoring] check failed:', err);
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
setChecking(false);
|
||||||
|
}
|
||||||
|
}, [platform.metadata.isTauri]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!platform.metadata.isTauri) return;
|
||||||
|
recheck();
|
||||||
|
const onFocus = () => {
|
||||||
|
recheck();
|
||||||
|
};
|
||||||
|
window.addEventListener('focus', onFocus);
|
||||||
|
return () => window.removeEventListener('focus', onFocus);
|
||||||
|
}, [platform.metadata.isTauri, recheck]);
|
||||||
|
|
||||||
|
const openSettings = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
await invoke('open_input_monitoring_settings');
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[input-monitoring] open settings failed:', err);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return { needsPermission, checking, recheck, openSettings };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inline notice rendered under the global-shortcut toggle when the user has
|
||||||
|
* opted in but macOS Input Monitoring is not granted. Returns null when the
|
||||||
|
* permission is present (or when the toggle is off and the notice would just
|
||||||
|
* be noise).
|
||||||
|
*/
|
||||||
|
export function InputMonitoringNotice({ enabled }: { enabled: boolean }) {
|
||||||
|
const { needsPermission, checking, recheck, openSettings } =
|
||||||
|
useInputMonitoringPermission();
|
||||||
|
const [stillMissing, setStillMissing] = useState(false);
|
||||||
|
|
||||||
|
const handleRecheck = useCallback(async () => {
|
||||||
|
setStillMissing(false);
|
||||||
|
const trusted = await recheck();
|
||||||
|
if (!trusted) setStillMissing(true);
|
||||||
|
}, [recheck]);
|
||||||
|
|
||||||
|
if (!enabled || !needsPermission) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="mt-3 rounded-lg border border-amber-500/30 bg-amber-500/10 px-3.5 py-3">
|
||||||
|
<div className="flex items-start gap-3">
|
||||||
|
<AlertTriangle className="h-4 w-4 shrink-0 mt-0.5 text-amber-500" />
|
||||||
|
<div className="flex-1 min-w-0 space-y-1">
|
||||||
|
<p className="text-sm font-medium text-foreground">
|
||||||
|
Grant Input Monitoring to enable the global shortcut
|
||||||
|
</p>
|
||||||
|
<p className="text-sm text-muted-foreground leading-relaxed">
|
||||||
|
Voicebox needs System Settings → Privacy & Security → Input
|
||||||
|
Monitoring to detect your dictation chord. The toggle is on, but
|
||||||
|
macOS is blocking key events until you allow it.
|
||||||
|
</p>
|
||||||
|
<div className="flex items-center gap-2 pt-1.5">
|
||||||
|
<Button size="sm" onClick={openSettings} className="gap-1.5">
|
||||||
|
<ExternalLink className="h-3.5 w-3.5" />
|
||||||
|
Open Settings
|
||||||
|
</Button>
|
||||||
|
<Button variant="outline" size="sm" onClick={handleRecheck} disabled={checking}>
|
||||||
|
{checking ? 'Checking…' : "I've enabled it"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
{stillMissing && !checking && (
|
||||||
|
<p className="text-xs text-amber-600 dark:text-amber-400 pt-1">
|
||||||
|
Still not detected. macOS usually requires quitting and reopening
|
||||||
|
Voicebox after toggling the permission.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { Check, ChevronDown, Keyboard, Laptop, Lock, Trash2, Volume2 } from 'lucide-react';
|
import { Check, ChevronDown, Keyboard, Laptop, Lock, Trash2, Volume2 } from 'lucide-react';
|
||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
import { AccessibilityNotice } from '@/components/AccessibilityGate/AccessibilityGate';
|
import { AccessibilityNotice } from '@/components/AccessibilityGate/AccessibilityGate';
|
||||||
|
import { InputMonitoringNotice } from '@/components/InputMonitoringGate/InputMonitoringGate';
|
||||||
import { CapturePill, type PillState } from '@/components/CapturePill/CapturePill';
|
import { CapturePill, type PillState } from '@/components/CapturePill/CapturePill';
|
||||||
import { ChordPicker } from '@/components/ChordPicker/ChordPicker';
|
import { ChordPicker } from '@/components/ChordPicker/ChordPicker';
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
@@ -140,13 +141,13 @@ export function CapturesPage() {
|
|||||||
const preserveTechnical = settings?.preserve_technical ?? true;
|
const preserveTechnical = settings?.preserve_technical ?? true;
|
||||||
const allowAutoPaste = settings?.allow_auto_paste ?? true;
|
const allowAutoPaste = settings?.allow_auto_paste ?? true;
|
||||||
const defaultVoiceId = settings?.default_playback_voice_id ?? null;
|
const defaultVoiceId = settings?.default_playback_voice_id ?? null;
|
||||||
|
const hotkeyEnabled = settings?.hotkey_enabled ?? false;
|
||||||
const pushToTalkKeys = settings?.chord_push_to_talk_keys ?? ['MetaRight', 'AltGr'];
|
const pushToTalkKeys = settings?.chord_push_to_talk_keys ?? ['MetaRight', 'AltGr'];
|
||||||
const toggleToTalkKeys = settings?.chord_toggle_to_talk_keys ?? ['MetaRight', 'AltGr', 'Space'];
|
const toggleToTalkKeys = settings?.chord_toggle_to_talk_keys ?? ['MetaRight', 'AltGr', 'Space'];
|
||||||
|
|
||||||
// Mock-only settings — not yet wired to a backend. Keep local so the UI
|
// Mock-only settings — not yet wired to a backend. Keep local so the UI
|
||||||
// still responds while Phase 7 (hotkey / clipboard / paste) catches up.
|
// still responds while Phase 7 (hotkey / clipboard / paste) catches up.
|
||||||
const [archiveAudio, setArchiveAudio] = useState(true);
|
const [archiveAudio, setArchiveAudio] = useState(true);
|
||||||
const [hotkeyEnabled, setHotkeyEnabled] = useState(true);
|
|
||||||
const [copyToClipboard, setCopyToClipboard] = useState(true);
|
const [copyToClipboard, setCopyToClipboard] = useState(true);
|
||||||
const [retention, setRetention] = useState('forever');
|
const [retention, setRetention] = useState('forever');
|
||||||
const [chordEditor, setChordEditor] = useState<'push' | 'toggle' | null>(null);
|
const [chordEditor, setChordEditor] = useState<'push' | 'toggle' | null>(null);
|
||||||
@@ -162,14 +163,21 @@ export function CapturesPage() {
|
|||||||
title="Dictation"
|
title="Dictation"
|
||||||
description="Capture from anywhere on your machine with a global shortcut."
|
description="Capture from anywhere on your machine with a global shortcut."
|
||||||
>
|
>
|
||||||
|
<div>
|
||||||
<SettingRow
|
<SettingRow
|
||||||
title="Global shortcut"
|
title="Global shortcut"
|
||||||
description="Hold the shortcut to record. Release to transcribe. Requires an Accessibility permission the first time you enable it."
|
description="Hold the shortcut to record from anywhere on your machine. Release to transcribe. macOS will ask for Input Monitoring permission the first time you turn this on."
|
||||||
htmlFor="hotkeyEnabled"
|
htmlFor="hotkeyEnabled"
|
||||||
action={
|
action={
|
||||||
<Toggle id="hotkeyEnabled" checked={hotkeyEnabled} onCheckedChange={setHotkeyEnabled} />
|
<Toggle
|
||||||
|
id="hotkeyEnabled"
|
||||||
|
checked={hotkeyEnabled}
|
||||||
|
onCheckedChange={(v) => update({ hotkey_enabled: v })}
|
||||||
|
/>
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
|
<InputMonitoringNotice enabled={hotkeyEnabled} />
|
||||||
|
</div>
|
||||||
|
|
||||||
<SettingRow
|
<SettingRow
|
||||||
title="Push-to-talk shortcut"
|
title="Push-to-talk shortcut"
|
||||||
|
|||||||
@@ -0,0 +1,324 @@
|
|||||||
|
import { Check, Copy, Plug, Trash2, Waypoints } from 'lucide-react';
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { useMCPBindings } from '@/lib/hooks/useMCPBindings';
|
||||||
|
import { useProfiles } from '@/lib/hooks/useProfiles';
|
||||||
|
import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
||||||
|
import { useServerStore } from '@/stores/serverStore';
|
||||||
|
import { SettingRow, SettingSection } from './SettingRow';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Settings → MCP — configure per-agent voice binding and show copy-paste
|
||||||
|
* install snippets for major MCP clients. Backend runs at /mcp on the
|
||||||
|
* existing Voicebox server; this page is the agent-onboarding surface.
|
||||||
|
*/
|
||||||
|
export function MCPPage() {
|
||||||
|
const serverUrl = useServerStore((s) => s.serverUrl);
|
||||||
|
const { bindings, upsertAsync, remove } = useMCPBindings();
|
||||||
|
const { data: profiles } = useProfiles();
|
||||||
|
const { settings: captureSettings, update: updateCapture } = useCaptureSettings();
|
||||||
|
|
||||||
|
const defaultProfileId = captureSettings?.default_playback_voice_id ?? '';
|
||||||
|
const mcpUrl = `${serverUrl}/mcp`;
|
||||||
|
|
||||||
|
const [newClientId, setNewClientId] = useState('');
|
||||||
|
const [newLabel, setNewLabel] = useState('');
|
||||||
|
const [newProfileId, setNewProfileId] = useState('');
|
||||||
|
const [adding, setAdding] = useState(false);
|
||||||
|
|
||||||
|
const handleAdd = async () => {
|
||||||
|
if (!newClientId.trim()) return;
|
||||||
|
setAdding(true);
|
||||||
|
try {
|
||||||
|
await upsertAsync({
|
||||||
|
client_id: newClientId.trim(),
|
||||||
|
label: newLabel.trim() || null,
|
||||||
|
profile_id: newProfileId || null,
|
||||||
|
});
|
||||||
|
setNewClientId('');
|
||||||
|
setNewLabel('');
|
||||||
|
setNewProfileId('');
|
||||||
|
} finally {
|
||||||
|
setAdding(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex gap-8 items-start max-w-5xl">
|
||||||
|
<div className="flex-1 min-w-0 max-w-2xl space-y-8">
|
||||||
|
<SettingSection
|
||||||
|
title="Install into your agent"
|
||||||
|
description="Voicebox exposes a local MCP server whenever the app is open. Paste one of these snippets into your agent's MCP config."
|
||||||
|
>
|
||||||
|
<SnippetRow
|
||||||
|
title="HTTP (recommended)"
|
||||||
|
description="For clients that speak HTTP MCP — Claude Code, Cursor, Windsurf, VS Code."
|
||||||
|
snippet={JSON.stringify(
|
||||||
|
{
|
||||||
|
mcpServers: {
|
||||||
|
voicebox: {
|
||||||
|
url: mcpUrl,
|
||||||
|
headers: { 'X-Voicebox-Client-Id': 'claude-code' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
<SnippetRow
|
||||||
|
title="Claude Code one-liner"
|
||||||
|
description="Registers via the Claude Code CLI."
|
||||||
|
snippet={`claude mcp add voicebox --transport http --url ${mcpUrl} --header "X-Voicebox-Client-Id: claude-code"`}
|
||||||
|
/>
|
||||||
|
<SnippetRow
|
||||||
|
title="Stdio (fallback)"
|
||||||
|
description="For clients that only spawn stdio processes. The shim binary ships with the app."
|
||||||
|
snippet={JSON.stringify(
|
||||||
|
{
|
||||||
|
mcpServers: {
|
||||||
|
voicebox: {
|
||||||
|
command:
|
||||||
|
'/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp',
|
||||||
|
env: { VOICEBOX_CLIENT_ID: 'claude-code' },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
</SettingSection>
|
||||||
|
|
||||||
|
<SettingSection
|
||||||
|
title="Default voice"
|
||||||
|
description="Used when an agent calls voicebox.speak without a specific profile and has no per-client binding."
|
||||||
|
>
|
||||||
|
<SettingRow
|
||||||
|
title="Default playback voice"
|
||||||
|
description="Shared with the Captures-tab 'Play as voice' dropdown — one default voice for passive playback."
|
||||||
|
action={
|
||||||
|
<select
|
||||||
|
value={defaultProfileId}
|
||||||
|
onChange={(e) =>
|
||||||
|
updateCapture({
|
||||||
|
default_playback_voice_id: e.target.value || null,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="h-8 px-2 rounded-md border bg-background text-sm min-w-[180px]"
|
||||||
|
>
|
||||||
|
<option value="">(none)</option>
|
||||||
|
{(profiles ?? []).map((p) => (
|
||||||
|
<option key={p.id} value={p.id}>
|
||||||
|
{p.name}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</SettingSection>
|
||||||
|
|
||||||
|
<SettingSection
|
||||||
|
title="Per-agent voice"
|
||||||
|
description="Bind specific agents to specific voices so you can tell who's speaking without looking. The agent identifies itself by the X-Voicebox-Client-Id header (or VOICEBOX_CLIENT_ID env for stdio)."
|
||||||
|
>
|
||||||
|
{bindings.length === 0 ? (
|
||||||
|
<p className="text-sm text-muted-foreground py-4 italic">
|
||||||
|
No bindings yet. Add one below, then configure your MCP client to
|
||||||
|
send the matching <code>X-Voicebox-Client-Id</code>.
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<div className="divide-y divide-border/60">
|
||||||
|
{bindings.map((b) => (
|
||||||
|
<div
|
||||||
|
key={b.client_id}
|
||||||
|
className="py-3 grid grid-cols-[1fr_auto_auto] gap-4 items-center"
|
||||||
|
>
|
||||||
|
<div className="min-w-0">
|
||||||
|
<div className="font-medium text-sm truncate">
|
||||||
|
{b.label || b.client_id}
|
||||||
|
</div>
|
||||||
|
<div className="text-xs text-muted-foreground truncate">
|
||||||
|
<code className="text-[11px]">{b.client_id}</code>
|
||||||
|
{' · '}
|
||||||
|
{b.last_seen_at ? (
|
||||||
|
<span title={`Last seen ${b.last_seen_at}`}>
|
||||||
|
<Plug className="inline h-3 w-3 text-emerald-500" />{' '}
|
||||||
|
last seen {formatRelative(b.last_seen_at)}
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<span>never connected</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<select
|
||||||
|
value={b.profile_id ?? ''}
|
||||||
|
onChange={(e) =>
|
||||||
|
upsertAsync({
|
||||||
|
client_id: b.client_id,
|
||||||
|
label: b.label,
|
||||||
|
profile_id: e.target.value || null,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
className="h-8 px-2 rounded-md border bg-background text-sm min-w-[160px]"
|
||||||
|
>
|
||||||
|
<option value="">(default)</option>
|
||||||
|
{(profiles ?? []).map((p) => (
|
||||||
|
<option key={p.id} value={p.id}>
|
||||||
|
{p.name}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<Button
|
||||||
|
size="icon"
|
||||||
|
variant="ghost"
|
||||||
|
onClick={() => remove(b.client_id)}
|
||||||
|
aria-label={`Remove binding for ${b.client_id}`}
|
||||||
|
>
|
||||||
|
<Trash2 className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="pt-4 space-y-2">
|
||||||
|
<div className="text-sm font-medium">Add a binding</div>
|
||||||
|
<div className="grid grid-cols-[1fr_1fr_auto] gap-2">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="client id (e.g. claude-code)"
|
||||||
|
value={newClientId}
|
||||||
|
onChange={(e) => setNewClientId(e.target.value)}
|
||||||
|
className="h-9 px-3 rounded-md border bg-background text-sm"
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
placeholder="label (optional)"
|
||||||
|
value={newLabel}
|
||||||
|
onChange={(e) => setNewLabel(e.target.value)}
|
||||||
|
className="h-9 px-3 rounded-md border bg-background text-sm"
|
||||||
|
/>
|
||||||
|
<select
|
||||||
|
value={newProfileId}
|
||||||
|
onChange={(e) => setNewProfileId(e.target.value)}
|
||||||
|
className="h-9 px-2 rounded-md border bg-background text-sm min-w-[140px]"
|
||||||
|
>
|
||||||
|
<option value="">(default)</option>
|
||||||
|
{(profiles ?? []).map((p) => (
|
||||||
|
<option key={p.id} value={p.id}>
|
||||||
|
{p.name}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
onClick={handleAdd}
|
||||||
|
disabled={!newClientId.trim() || adding}
|
||||||
|
>
|
||||||
|
Add binding
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</SettingSection>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<aside className="hidden lg:block w-[280px] shrink-0 space-y-6 sticky top-0">
|
||||||
|
<div className="space-y-2">
|
||||||
|
<h3 className="text-sm font-semibold">About MCP</h3>
|
||||||
|
<p className="text-sm text-muted-foreground leading-relaxed">
|
||||||
|
Model Context Protocol lets your AI coding agent — Claude Code,
|
||||||
|
Cursor, Windsurf — call Voicebox tools. Speak in a cloned voice,
|
||||||
|
transcribe audio, browse captures.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="space-y-2">
|
||||||
|
<h3 className="text-sm font-semibold">Available tools</h3>
|
||||||
|
<ul className="text-sm text-muted-foreground space-y-1.5 leading-relaxed">
|
||||||
|
<li>
|
||||||
|
<code className="text-accent">voicebox.speak</code>
|
||||||
|
<div>Speak text in a voice profile.</div>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<code className="text-accent">voicebox.transcribe</code>
|
||||||
|
<div>Whisper STT on a clip.</div>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<code className="text-accent">voicebox.list_captures</code>
|
||||||
|
<div>Recent dictations / recordings.</div>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<code className="text-accent">voicebox.list_profiles</code>
|
||||||
|
<div>Available voice profiles.</div>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||||
|
<Waypoints className="h-3.5 w-3.5 text-accent" />
|
||||||
|
<span>
|
||||||
|
Also exposed as <code>POST /speak</code> for shell scripts, ACP,
|
||||||
|
A2A.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</aside>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function SnippetRow({
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
snippet,
|
||||||
|
}: {
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
snippet: string;
|
||||||
|
}) {
|
||||||
|
const [copied, setCopied] = useState(false);
|
||||||
|
const copy = async () => {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(snippet);
|
||||||
|
setCopied(true);
|
||||||
|
setTimeout(() => setCopied(false), 1500);
|
||||||
|
} catch {
|
||||||
|
// ignore; user can still select-and-copy the pre content
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="py-3 space-y-2">
|
||||||
|
<div className="flex items-center justify-between gap-4">
|
||||||
|
<div>
|
||||||
|
<div className="text-sm font-medium">{title}</div>
|
||||||
|
<div className="text-xs text-muted-foreground">{description}</div>
|
||||||
|
</div>
|
||||||
|
<Button size="sm" variant="outline" onClick={copy}>
|
||||||
|
{copied ? (
|
||||||
|
<>
|
||||||
|
<Check className="h-3.5 w-3.5 mr-1.5" />
|
||||||
|
Copied
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Copy className="h-3.5 w-3.5 mr-1.5" />
|
||||||
|
Copy
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
<pre className="text-[11px] font-mono p-3 rounded-md bg-muted/50 overflow-x-auto whitespace-pre-wrap break-all">
|
||||||
|
{snippet}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRelative(iso: string): string {
|
||||||
|
const then = new Date(iso).getTime();
|
||||||
|
const now = Date.now();
|
||||||
|
const diff = Math.max(0, now - then);
|
||||||
|
if (diff < 60_000) return 'just now';
|
||||||
|
if (diff < 3600_000) return `${Math.floor(diff / 60_000)} min ago`;
|
||||||
|
if (diff < 86400_000) return `${Math.floor(diff / 3600_000)} h ago`;
|
||||||
|
return `${Math.floor(diff / 86400_000)} d ago`;
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ interface SettingsTab {
|
|||||||
| '/settings'
|
| '/settings'
|
||||||
| '/settings/generation'
|
| '/settings/generation'
|
||||||
| '/settings/captures'
|
| '/settings/captures'
|
||||||
|
| '/settings/mcp'
|
||||||
| '/settings/gpu'
|
| '/settings/gpu'
|
||||||
| '/settings/logs'
|
| '/settings/logs'
|
||||||
| '/settings/changelog'
|
| '/settings/changelog'
|
||||||
@@ -23,6 +24,7 @@ const tabs: SettingsTab[] = [
|
|||||||
{ labelKey: 'settings.tabs.general', path: '/settings' },
|
{ labelKey: 'settings.tabs.general', path: '/settings' },
|
||||||
{ labelKey: 'settings.tabs.generation', path: '/settings/generation' },
|
{ labelKey: 'settings.tabs.generation', path: '/settings/generation' },
|
||||||
{ label: 'Captures', path: '/settings/captures' },
|
{ label: 'Captures', path: '/settings/captures' },
|
||||||
|
{ label: 'MCP', path: '/settings/mcp' },
|
||||||
{ labelKey: 'settings.tabs.gpu', path: '/settings/gpu', tauriOnly: true },
|
{ labelKey: 'settings.tabs.gpu', path: '/settings/gpu', tauriOnly: true },
|
||||||
{ labelKey: 'settings.tabs.logs', path: '/settings/logs', tauriOnly: true },
|
{ labelKey: 'settings.tabs.logs', path: '/settings/logs', tauriOnly: true },
|
||||||
{ labelKey: 'settings.tabs.changelog', path: '/settings/changelog' },
|
{ labelKey: 'settings.tabs.changelog', path: '/settings/changelog' },
|
||||||
|
|||||||
@@ -45,6 +45,9 @@ import type {
|
|||||||
CaptureSource,
|
CaptureSource,
|
||||||
GenerationSettings,
|
GenerationSettings,
|
||||||
GenerationSettingsUpdate,
|
GenerationSettingsUpdate,
|
||||||
|
MCPClientBinding,
|
||||||
|
MCPClientBindingListResponse,
|
||||||
|
MCPClientBindingUpsert,
|
||||||
} from './types';
|
} from './types';
|
||||||
|
|
||||||
function formatErrorDetail(detail: unknown, fallback: string): string {
|
function formatErrorDetail(detail: unknown, fallback: string): string {
|
||||||
@@ -503,6 +506,27 @@ class ApiClient {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MCP bindings — per-MCP-client voice/engine/intent mapping.
|
||||||
|
async listMCPBindings(): Promise<MCPClientBindingListResponse> {
|
||||||
|
return this.request<MCPClientBindingListResponse>('/mcp/bindings');
|
||||||
|
}
|
||||||
|
|
||||||
|
async upsertMCPBinding(
|
||||||
|
data: MCPClientBindingUpsert,
|
||||||
|
): Promise<MCPClientBinding> {
|
||||||
|
return this.request<MCPClientBinding>('/mcp/bindings', {
|
||||||
|
method: 'PUT',
|
||||||
|
body: JSON.stringify(data),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteMCPBinding(clientId: string): Promise<{ deleted: string }> {
|
||||||
|
return this.request<{ deleted: string }>(
|
||||||
|
`/mcp/bindings/${encodeURIComponent(clientId)}`,
|
||||||
|
{ method: 'DELETE' },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Model Management
|
// Model Management
|
||||||
async getModelStatus(): Promise<ModelStatusListResponse> {
|
async getModelStatus(): Promise<ModelStatusListResponse> {
|
||||||
return this.request<ModelStatusListResponse>('/models/status');
|
return this.request<ModelStatusListResponse>('/models/status');
|
||||||
|
|||||||
@@ -208,6 +208,9 @@ export interface CaptureSettings {
|
|||||||
preserve_technical: boolean;
|
preserve_technical: boolean;
|
||||||
allow_auto_paste: boolean;
|
allow_auto_paste: boolean;
|
||||||
default_playback_voice_id: string | null;
|
default_playback_voice_id: string | null;
|
||||||
|
/** Whether the global keyboard hotkey is armed. Off by default — turning
|
||||||
|
* this on triggers the macOS Input Monitoring TCC prompt. */
|
||||||
|
hotkey_enabled: boolean;
|
||||||
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr"]. */
|
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr"]. */
|
||||||
chord_push_to_talk_keys: string[];
|
chord_push_to_talk_keys: string[];
|
||||||
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr","Space"]. */
|
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr","Space"]. */
|
||||||
@@ -465,3 +468,28 @@ export interface ApplyEffectsRequest {
|
|||||||
label?: string;
|
label?: string;
|
||||||
set_as_default?: boolean;
|
set_as_default?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ─── MCP ─────────────────────────────────────────────────────────────── */
|
||||||
|
|
||||||
|
export interface MCPClientBinding {
|
||||||
|
client_id: string;
|
||||||
|
label: string | null;
|
||||||
|
profile_id: string | null;
|
||||||
|
default_engine: string | null;
|
||||||
|
default_intent: 'respond' | 'rewrite' | 'compose' | null;
|
||||||
|
last_seen_at: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MCPClientBindingUpsert {
|
||||||
|
client_id: string;
|
||||||
|
label?: string | null;
|
||||||
|
profile_id?: string | null;
|
||||||
|
default_engine?: string | null;
|
||||||
|
default_intent?: 'respond' | 'rewrite' | 'compose' | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MCPClientBindingListResponse {
|
||||||
|
items: MCPClientBinding[];
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,32 +4,41 @@ import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
|||||||
import { usePlatform } from '@/platform/PlatformContext';
|
import { usePlatform } from '@/platform/PlatformContext';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push the user's saved chord into the running Rust `HotkeyMonitor`.
|
* Spawn (or quiet) the global hotkey monitor based on the saved
|
||||||
* The monitor boots with hard-coded right-hand defaults; this hook
|
* `capture_settings.hotkey_enabled` flag, and keep its bindings in sync with
|
||||||
* replaces them as soon as capture_settings resolves and re-applies on
|
* the user's chord choices.
|
||||||
* every subsequent change so chord edits land without a restart.
|
|
||||||
*
|
*
|
||||||
* Call once from the main app shell — multiple call sites would just
|
* Boot sequence:
|
||||||
* fire redundant invokes, since the chord engine swap is the same value
|
* - hotkey_enabled = false → call `disable_hotkey` (no-op if monitor was
|
||||||
* either way.
|
* never spawned). Crucially, we do *not* call `enable_hotkey`, so the
|
||||||
|
* macOS Input Monitoring TCC prompt is never triggered for users who
|
||||||
|
* haven't opted in.
|
||||||
|
* - hotkey_enabled = true → call `enable_hotkey` with the saved chords.
|
||||||
|
* This is the call that creates the CGEventTap and triggers the TCC
|
||||||
|
* prompt on first opt-in.
|
||||||
|
*
|
||||||
|
* Call once from the main app shell.
|
||||||
*/
|
*/
|
||||||
export function useChordSync() {
|
export function useChordSync() {
|
||||||
const platform = usePlatform();
|
const platform = usePlatform();
|
||||||
const { settings } = useCaptureSettings();
|
const { settings } = useCaptureSettings();
|
||||||
|
const enabled = settings?.hotkey_enabled;
|
||||||
const pushKeys = settings?.chord_push_to_talk_keys;
|
const pushKeys = settings?.chord_push_to_talk_keys;
|
||||||
const toggleKeys = settings?.chord_toggle_to_talk_keys;
|
const toggleKeys = settings?.chord_toggle_to_talk_keys;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!platform.metadata.isTauri) return;
|
if (!platform.metadata.isTauri) return;
|
||||||
if (!pushKeys || !toggleKeys) return;
|
if (enabled === undefined || !pushKeys || !toggleKeys) return;
|
||||||
invoke('update_chord_bindings', {
|
const command = enabled ? 'enable_hotkey' : 'disable_hotkey';
|
||||||
pushToTalk: pushKeys,
|
const args = enabled
|
||||||
toggleToTalk: toggleKeys,
|
? { pushToTalk: pushKeys, toggleToTalk: toggleKeys }
|
||||||
}).catch((err) => {
|
: {};
|
||||||
console.warn('[chord-sync] failed to update bindings:', err);
|
invoke(command, args).catch((err) => {
|
||||||
|
console.warn(`[chord-sync] ${command} failed:`, err);
|
||||||
});
|
});
|
||||||
}, [
|
}, [
|
||||||
platform.metadata.isTauri,
|
platform.metadata.isTauri,
|
||||||
|
enabled,
|
||||||
// Stringify so a referentially-new array with the same content
|
// Stringify so a referentially-new array with the same content
|
||||||
// doesn't fire a redundant invoke on every settings refetch.
|
// doesn't fire a redundant invoke on every settings refetch.
|
||||||
pushKeys?.join(','),
|
pushKeys?.join(','),
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||||
|
import { apiClient } from '@/lib/api/client';
|
||||||
|
import type {
|
||||||
|
MCPClientBindingListResponse,
|
||||||
|
MCPClientBindingUpsert,
|
||||||
|
} from '@/lib/api/types';
|
||||||
|
|
||||||
|
const MCP_BINDINGS_KEY = ['settings', 'mcp', 'bindings'] as const;
|
||||||
|
|
||||||
|
/** Manage per-MCP-client voice bindings (Claude Code → Morgan, etc.). */
|
||||||
|
export function useMCPBindings() {
|
||||||
|
const queryClient = useQueryClient();
|
||||||
|
|
||||||
|
const query = useQuery({
|
||||||
|
queryKey: MCP_BINDINGS_KEY,
|
||||||
|
queryFn: () => apiClient.listMCPBindings(),
|
||||||
|
// Keep fresh while the Settings page is open — the ``last_seen_at``
|
||||||
|
// timestamp is useful for confirming an install works, and we want it
|
||||||
|
// to tick forward when a client connects.
|
||||||
|
refetchInterval: 10_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const upsertMutation = useMutation({
|
||||||
|
mutationFn: (data: MCPClientBindingUpsert) =>
|
||||||
|
apiClient.upsertMCPBinding(data),
|
||||||
|
onSuccess: () => {
|
||||||
|
queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const deleteMutation = useMutation({
|
||||||
|
mutationFn: (clientId: string) => apiClient.deleteMCPBinding(clientId),
|
||||||
|
onMutate: async (clientId) => {
|
||||||
|
await queryClient.cancelQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||||
|
const prev =
|
||||||
|
queryClient.getQueryData<MCPClientBindingListResponse>(MCP_BINDINGS_KEY);
|
||||||
|
if (prev) {
|
||||||
|
queryClient.setQueryData<MCPClientBindingListResponse>(
|
||||||
|
MCP_BINDINGS_KEY,
|
||||||
|
{ items: prev.items.filter((b) => b.client_id !== clientId) },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { prev };
|
||||||
|
},
|
||||||
|
onError: (_err, _id, ctx) => {
|
||||||
|
if (ctx?.prev) queryClient.setQueryData(MCP_BINDINGS_KEY, ctx.prev);
|
||||||
|
},
|
||||||
|
onSettled: () => {
|
||||||
|
queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
bindings: query.data?.items ?? [],
|
||||||
|
isLoading: query.isLoading,
|
||||||
|
upsert: upsertMutation.mutate,
|
||||||
|
upsertAsync: upsertMutation.mutateAsync,
|
||||||
|
remove: deleteMutation.mutate,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { useEffect, useRef, useState } from 'react';
|
||||||
|
import { useServerStore } from '@/stores/serverStore';
|
||||||
|
|
||||||
|
/** Payload for a speak-start SSE event broadcast by the backend. */
|
||||||
|
export interface ActiveSpeak {
|
||||||
|
generationId: string;
|
||||||
|
profileName: string;
|
||||||
|
source: 'mcp' | 'rest' | string;
|
||||||
|
clientId: string | null;
|
||||||
|
startedAt: number;
|
||||||
|
elapsedMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Subscribes to `/events/speak` and reports whichever agent-initiated
|
||||||
|
* speak is currently producing audio. Returns ``null`` when nothing is
|
||||||
|
* speaking.
|
||||||
|
*
|
||||||
|
* Multiple concurrent speaks are rare (the model can only really do one
|
||||||
|
* at a time) and we don't bother stacking them — newest wins, the old
|
||||||
|
* one's speak-end will clear when it fires.
|
||||||
|
*/
|
||||||
|
export function useSpeakEvents(): ActiveSpeak | null {
|
||||||
|
const [active, setActive] = useState<ActiveSpeak | null>(null);
|
||||||
|
const activeRef = useRef<ActiveSpeak | null>(null);
|
||||||
|
activeRef.current = active;
|
||||||
|
|
||||||
|
// Keep a live timer so the pill's elapsed label advances smoothly
|
||||||
|
// without re-opening the SSE stream.
|
||||||
|
const [, setTick] = useState(0);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!active) return;
|
||||||
|
const iv = window.setInterval(() => setTick((t) => t + 1), 250);
|
||||||
|
return () => window.clearInterval(iv);
|
||||||
|
}, [active]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const baseUrl = useServerStore.getState().serverUrl;
|
||||||
|
if (!baseUrl) return;
|
||||||
|
|
||||||
|
let cancelled = false;
|
||||||
|
let source: EventSource | null = null;
|
||||||
|
|
||||||
|
const connect = () => {
|
||||||
|
if (cancelled) return;
|
||||||
|
source = new EventSource(`${baseUrl}/events/speak`);
|
||||||
|
|
||||||
|
source.addEventListener('speak-start', (event: MessageEvent) => {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(event.data);
|
||||||
|
const now = Date.now();
|
||||||
|
setActive({
|
||||||
|
generationId: String(data.generation_id ?? ''),
|
||||||
|
profileName: String(data.profile_name ?? ''),
|
||||||
|
source: String(data.source ?? 'mcp'),
|
||||||
|
clientId: data.client_id ?? null,
|
||||||
|
startedAt: now,
|
||||||
|
elapsedMs: 0,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// malformed payload — ignore, don't crash the stream
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
source.addEventListener('speak-end', (event: MessageEvent) => {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(event.data);
|
||||||
|
const endedId = String(data.generation_id ?? '');
|
||||||
|
// Only clear if this end matches the currently-active id; late
|
||||||
|
// ends from previous sessions are ignored.
|
||||||
|
if (activeRef.current?.generationId === endedId) setActive(null);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
source.onerror = () => {
|
||||||
|
// EventSource auto-reconnects, but if the browser gives up we
|
||||||
|
// manually retry with backoff.
|
||||||
|
source?.close();
|
||||||
|
if (!cancelled) window.setTimeout(connect, 2000);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
connect();
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
source?.close();
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!active) return null;
|
||||||
|
return { ...active, elapsedMs: Date.now() - active.startedAt };
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ import { GeneralPage } from '@/components/ServerTab/GeneralPage';
|
|||||||
import { GenerationPage } from '@/components/ServerTab/GenerationPage';
|
import { GenerationPage } from '@/components/ServerTab/GenerationPage';
|
||||||
import { GpuPage } from '@/components/ServerTab/GpuPage';
|
import { GpuPage } from '@/components/ServerTab/GpuPage';
|
||||||
import { LogsPage } from '@/components/ServerTab/LogsPage';
|
import { LogsPage } from '@/components/ServerTab/LogsPage';
|
||||||
|
import { MCPPage } from '@/components/ServerTab/MCPPage';
|
||||||
import { SettingsLayout } from '@/components/ServerTab/ServerTab';
|
import { SettingsLayout } from '@/components/ServerTab/ServerTab';
|
||||||
import { Sidebar } from '@/components/Sidebar';
|
import { Sidebar } from '@/components/Sidebar';
|
||||||
import { StoriesTab } from '@/components/StoriesTab/StoriesTab';
|
import { StoriesTab } from '@/components/StoriesTab/StoriesTab';
|
||||||
@@ -159,6 +160,12 @@ const settingsCapturesRoute = createRoute({
|
|||||||
component: CapturesPage,
|
component: CapturesPage,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const settingsMCPRoute = createRoute({
|
||||||
|
getParentRoute: () => settingsRoute,
|
||||||
|
path: '/mcp',
|
||||||
|
component: MCPPage,
|
||||||
|
});
|
||||||
|
|
||||||
const settingsGpuRoute = createRoute({
|
const settingsGpuRoute = createRoute({
|
||||||
getParentRoute: () => settingsRoute,
|
getParentRoute: () => settingsRoute,
|
||||||
path: '/gpu',
|
path: '/gpu',
|
||||||
@@ -204,6 +211,7 @@ const routeTree = rootRoute.addChildren([
|
|||||||
settingsGeneralRoute,
|
settingsGeneralRoute,
|
||||||
settingsGenerationRoute,
|
settingsGenerationRoute,
|
||||||
settingsCapturesRoute,
|
settingsCapturesRoute,
|
||||||
|
settingsMCPRoute,
|
||||||
settingsGpuRoute,
|
settingsGpuRoute,
|
||||||
settingsLogsRoute,
|
settingsLogsRoute,
|
||||||
settingsChangelogRoute,
|
settingsChangelogRoute,
|
||||||
|
|||||||
+28
-9
@@ -4,6 +4,7 @@ import asyncio
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@@ -68,15 +69,36 @@ def safe_content_disposition(disposition_type: str, filename: str) -> str:
|
|||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
"""Create and configure the FastAPI application."""
|
"""Create and configure the FastAPI application."""
|
||||||
|
from .mcp_server.server import build_mcp_server
|
||||||
|
from .mcp_server.context import ClientIdMiddleware
|
||||||
|
|
||||||
|
# Build the MCP app up-front so we can wire its lifespan into FastAPI's —
|
||||||
|
# FastMCP's Streamable HTTP transport only works if its session manager
|
||||||
|
# runs inside the parent ASGI lifespan.
|
||||||
|
mcp = build_mcp_server()
|
||||||
|
mcp_app = mcp.http_app(path="/", transport="http")
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
await _run_startup(app)
|
||||||
|
async with mcp_app.router.lifespan_context(app):
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
await _run_shutdown()
|
||||||
|
|
||||||
application = FastAPI(
|
application = FastAPI(
|
||||||
title="voicebox API",
|
title="voicebox API",
|
||||||
description="Production-quality Qwen3-TTS voice cloning API",
|
description="Production-quality Qwen3-TTS voice cloning API",
|
||||||
version=__version__,
|
version=__version__,
|
||||||
|
lifespan=lifespan,
|
||||||
)
|
)
|
||||||
|
|
||||||
_configure_cors(application)
|
_configure_cors(application)
|
||||||
|
application.add_middleware(ClientIdMiddleware)
|
||||||
register_routers(application)
|
register_routers(application)
|
||||||
_register_lifecycle(application)
|
application.mount("/mcp", mcp_app)
|
||||||
|
logger.info("MCP: mounted at /mcp")
|
||||||
_mount_frontend(application)
|
_mount_frontend(application)
|
||||||
|
|
||||||
return application
|
return application
|
||||||
@@ -179,11 +201,8 @@ def _get_gpu_status() -> str:
|
|||||||
return "None (CPU only)"
|
return "None (CPU only)"
|
||||||
|
|
||||||
|
|
||||||
def _register_lifecycle(application: FastAPI) -> None:
|
async def _run_startup(application: FastAPI) -> None:
|
||||||
"""Attach startup and shutdown event handlers."""
|
"""Database init, warnings, model-cache prep. Runs on lifespan entry."""
|
||||||
|
|
||||||
@application.on_event("startup")
|
|
||||||
async def startup_event():
|
|
||||||
import platform
|
import platform
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@@ -237,7 +256,6 @@ def _register_lifecycle(application: FastAPI) -> None:
|
|||||||
logger.info("Backend: %s", backend_type.upper())
|
logger.info("Backend: %s", backend_type.upper())
|
||||||
logger.info("GPU: %s", _get_gpu_status())
|
logger.info("GPU: %s", _get_gpu_status())
|
||||||
|
|
||||||
# Warn if GPU architecture is not supported by this PyTorch build
|
|
||||||
from .backends.base import check_cuda_compatibility
|
from .backends.base import check_cuda_compatibility
|
||||||
|
|
||||||
_compatible, _cuda_warning = check_cuda_compatibility()
|
_compatible, _cuda_warning = check_cuda_compatibility()
|
||||||
@@ -265,8 +283,9 @@ def _register_lifecycle(application: FastAPI) -> None:
|
|||||||
|
|
||||||
logger.info("Ready")
|
logger.info("Ready")
|
||||||
|
|
||||||
@application.on_event("shutdown")
|
|
||||||
async def shutdown_event():
|
async def _run_shutdown() -> None:
|
||||||
|
"""Unload models on lifespan exit."""
|
||||||
logger.info("Voicebox server shutting down...")
|
logger.info("Voicebox server shutting down...")
|
||||||
try:
|
try:
|
||||||
tts.unload_tts_model()
|
tts.unload_tts_model()
|
||||||
|
|||||||
+119
-1
@@ -295,6 +295,28 @@ def build_server(cuda=False):
|
|||||||
"unidic_lite",
|
"unidic_lite",
|
||||||
"--hidden-import",
|
"--hidden-import",
|
||||||
"loguru",
|
"loguru",
|
||||||
|
# MCP server — Streamable-HTTP endpoint and the 4 voicebox.* tools.
|
||||||
|
# FastMCP pulls in a chain of deps (mcp, cyclopts, openapi-pydantic,
|
||||||
|
# etc.) that don't auto-discover cleanly under PyInstaller, so we
|
||||||
|
# collect them whole. Small compared to torch.
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server.server",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server.tools",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server.context",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server.resolve",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_server.events",
|
||||||
|
"--collect-all",
|
||||||
|
"fastmcp",
|
||||||
|
"--collect-all",
|
||||||
|
"mcp",
|
||||||
|
"--hidden-import",
|
||||||
|
"sse_starlette",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -447,12 +469,108 @@ def build_server(cuda=False):
|
|||||||
logger.info("Binary built in %s", backend_dir / "dist" / binary_name)
|
logger.info("Binary built in %s", backend_dir / "dist" / binary_name)
|
||||||
|
|
||||||
|
|
||||||
|
def build_shim():
|
||||||
|
"""Build the voicebox-mcp stdio shim as a tiny standalone binary.
|
||||||
|
|
||||||
|
This is the bridge for MCP clients that only speak stdio — it proxies
|
||||||
|
JSON-RPC to the main voicebox-server's /mcp endpoint. Keep it small: no
|
||||||
|
torch, no ML deps, just httpx + asyncio.
|
||||||
|
"""
|
||||||
|
backend_dir = Path(__file__).parent
|
||||||
|
|
||||||
|
args = [
|
||||||
|
"mcp_shim/__main__.py",
|
||||||
|
"--onefile",
|
||||||
|
"--name",
|
||||||
|
"voicebox-mcp",
|
||||||
|
# Stdio-only — no console hiding needed on Windows since the parent
|
||||||
|
# MCP client is spawning this as a child process and wants stdio.
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_shim",
|
||||||
|
"--hidden-import",
|
||||||
|
"backend.mcp_shim.__main__",
|
||||||
|
"--hidden-import",
|
||||||
|
"httpx",
|
||||||
|
"--hidden-import",
|
||||||
|
"httpx._transports.default",
|
||||||
|
"--hidden-import",
|
||||||
|
"anyio",
|
||||||
|
# Exclude everything heavy that httpx/asyncio don't actually need so
|
||||||
|
# the binary stays tiny (~15 MB instead of ~400 MB).
|
||||||
|
"--exclude-module",
|
||||||
|
"torch",
|
||||||
|
"--exclude-module",
|
||||||
|
"transformers",
|
||||||
|
"--exclude-module",
|
||||||
|
"mlx",
|
||||||
|
"--exclude-module",
|
||||||
|
"mlx_audio",
|
||||||
|
"--exclude-module",
|
||||||
|
"qwen_tts",
|
||||||
|
"--exclude-module",
|
||||||
|
"chatterbox",
|
||||||
|
"--exclude-module",
|
||||||
|
"zipvoice",
|
||||||
|
"--exclude-module",
|
||||||
|
"tada",
|
||||||
|
"--exclude-module",
|
||||||
|
"kokoro",
|
||||||
|
"--exclude-module",
|
||||||
|
"misaki",
|
||||||
|
"--exclude-module",
|
||||||
|
"spacy",
|
||||||
|
"--exclude-module",
|
||||||
|
"librosa",
|
||||||
|
"--exclude-module",
|
||||||
|
"numba",
|
||||||
|
"--exclude-module",
|
||||||
|
"numpy",
|
||||||
|
"--exclude-module",
|
||||||
|
"pedalboard",
|
||||||
|
"--exclude-module",
|
||||||
|
"fastapi",
|
||||||
|
"--exclude-module",
|
||||||
|
"uvicorn",
|
||||||
|
"--exclude-module",
|
||||||
|
"sqlalchemy",
|
||||||
|
"--exclude-module",
|
||||||
|
"fastmcp",
|
||||||
|
"--exclude-module",
|
||||||
|
"mcp",
|
||||||
|
]
|
||||||
|
|
||||||
|
dist_dir = str(backend_dir / "dist")
|
||||||
|
build_dir = str(backend_dir / "build")
|
||||||
|
args.extend(
|
||||||
|
[
|
||||||
|
"--distpath",
|
||||||
|
dist_dir,
|
||||||
|
"--workpath",
|
||||||
|
build_dir,
|
||||||
|
"--noconfirm",
|
||||||
|
"--clean",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
os.chdir(backend_dir)
|
||||||
|
PyInstaller.__main__.run(args)
|
||||||
|
logger.info("Shim built: %s", backend_dir / "dist" / "voicebox-mcp")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Build voicebox-server binary")
|
parser = argparse.ArgumentParser(description="Build voicebox binaries")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--cuda",
|
"--cuda",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Build CUDA-enabled binary (voicebox-server-cuda)",
|
help="Build CUDA-enabled binary (voicebox-server-cuda)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--shim",
|
||||||
|
action="store_true",
|
||||||
|
help="Build the voicebox-mcp stdio shim binary instead of the server",
|
||||||
|
)
|
||||||
cli_args = parser.parse_args()
|
cli_args = parser.parse_args()
|
||||||
|
if cli_args.shim:
|
||||||
|
build_shim()
|
||||||
|
else:
|
||||||
build_server(cuda=cli_args.cuda)
|
build_server(cuda=cli_args.cuda)
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from .models import (
|
|||||||
Generation,
|
Generation,
|
||||||
GenerationSettings,
|
GenerationSettings,
|
||||||
GenerationVersion,
|
GenerationVersion,
|
||||||
|
MCPClientBinding,
|
||||||
ProfileChannelMapping,
|
ProfileChannelMapping,
|
||||||
ProfileSample,
|
ProfileSample,
|
||||||
Project,
|
Project,
|
||||||
@@ -35,6 +36,7 @@ __all__ = [
|
|||||||
"Generation",
|
"Generation",
|
||||||
"GenerationSettings",
|
"GenerationSettings",
|
||||||
"GenerationVersion",
|
"GenerationVersion",
|
||||||
|
"MCPClientBinding",
|
||||||
"ProfileChannelMapping",
|
"ProfileChannelMapping",
|
||||||
"ProfileSample",
|
"ProfileSample",
|
||||||
"Project",
|
"Project",
|
||||||
|
|||||||
@@ -224,6 +224,13 @@ def _migrate_capture_settings(engine, inspector, tables: set[str]) -> None:
|
|||||||
"chord_toggle_to_talk_keys TEXT NOT NULL DEFAULT '[\"MetaRight\",\"AltGr\",\"Space\"]'",
|
"chord_toggle_to_talk_keys TEXT NOT NULL DEFAULT '[\"MetaRight\",\"AltGr\",\"Space\"]'",
|
||||||
"chord_toggle_to_talk_keys",
|
"chord_toggle_to_talk_keys",
|
||||||
)
|
)
|
||||||
|
if "hotkey_enabled" not in columns:
|
||||||
|
_add_column(
|
||||||
|
engine,
|
||||||
|
"capture_settings",
|
||||||
|
"hotkey_enabled BOOLEAN NOT NULL DEFAULT 0",
|
||||||
|
"hotkey_enabled",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _normalize_storage_paths(engine, tables: set[str]) -> None:
|
def _normalize_storage_paths(engine, tables: set[str]) -> None:
|
||||||
|
|||||||
@@ -196,6 +196,12 @@ class CaptureSettings(Base):
|
|||||||
preserve_technical = Column(Boolean, nullable=False, default=True)
|
preserve_technical = Column(Boolean, nullable=False, default=True)
|
||||||
allow_auto_paste = Column(Boolean, nullable=False, default=True)
|
allow_auto_paste = Column(Boolean, nullable=False, default=True)
|
||||||
default_playback_voice_id = Column(String, nullable=True)
|
default_playback_voice_id = Column(String, nullable=True)
|
||||||
|
# Default OFF — opting in is what triggers the macOS Input Monitoring TCC
|
||||||
|
# prompt. We deliberately don't spawn the global keyboard tap until the
|
||||||
|
# user flips this on so a fresh-install user doesn't see a scary
|
||||||
|
# "Voicebox would like to receive keystrokes from any application" dialog
|
||||||
|
# before they've even opened the Captures tab.
|
||||||
|
hotkey_enabled = Column(Boolean, nullable=False, default=False)
|
||||||
# Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand
|
# Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand
|
||||||
# modifiers by default so they don't collide with left-hand system
|
# modifiers by default so they don't collide with left-hand system
|
||||||
# shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit).
|
# shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit).
|
||||||
@@ -221,6 +227,29 @@ class GenerationSettings(Base):
|
|||||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
|
class MCPClientBinding(Base):
|
||||||
|
"""Per-MCP-client settings (voice profile, engine, intent).
|
||||||
|
|
||||||
|
Lets users bind distinct voices to distinct agents — e.g. Claude Code
|
||||||
|
speaks in "Morgan," Cursor in "Scarlett." The MCP client identifies
|
||||||
|
itself via the ``X-Voicebox-Client-Id`` HTTP header; direct-HTTP
|
||||||
|
clients set it in their MCP config's ``headers`` block, the stdio
|
||||||
|
shim forwards it from the ``VOICEBOX_CLIENT_ID`` env var.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "mcp_client_bindings"
|
||||||
|
|
||||||
|
client_id = Column(String, primary_key=True)
|
||||||
|
label = Column(String, nullable=True) # display name
|
||||||
|
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
|
||||||
|
default_engine = Column(String, nullable=True)
|
||||||
|
# "respond" | "rewrite" | "compose" — null means plain TTS (no LLM transform).
|
||||||
|
default_intent = Column(String, nullable=True)
|
||||||
|
last_seen_at = Column(DateTime, nullable=True)
|
||||||
|
created_at = Column(DateTime, default=datetime.utcnow)
|
||||||
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
class Capture(Base):
|
class Capture(Base):
|
||||||
"""A single voice input capture (dictation, recording, or uploaded file).
|
"""A single voice input capture (dictation, recording, or uploaded file).
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
# Voicebox MCP server
|
||||||
|
|
||||||
|
Local **Model Context Protocol** server — lets any MCP-aware agent
|
||||||
|
(Claude Code, Cursor, Windsurf, VS Code MCP extensions, etc.) speak text
|
||||||
|
in your cloned voices, transcribe audio, and browse captures.
|
||||||
|
|
||||||
|
The server runs inside the same `uvicorn` process as the rest of Voicebox
|
||||||
|
and is mounted at `/mcp` (Streamable HTTP transport).
|
||||||
|
|
||||||
|
## Install into your agent
|
||||||
|
|
||||||
|
Preferred — direct HTTP:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"url": "http://127.0.0.1:17493/mcp",
|
||||||
|
"headers": { "X-Voicebox-Client-Id": "claude-code" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Fallback — stdio shim (when the client doesn't speak HTTP MCP). The
|
||||||
|
`voicebox-mcp` binary ships inside the Voicebox.app bundle:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||||
|
"env": { "VOICEBOX_CLIENT_ID": "claude-code" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Claude Code one-liner:
|
||||||
|
|
||||||
|
```
|
||||||
|
claude mcp add voicebox \
|
||||||
|
--transport http \
|
||||||
|
--url http://127.0.0.1:17493/mcp \
|
||||||
|
--header "X-Voicebox-Client-Id: claude-code"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| Name | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. |
|
||||||
|
| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. |
|
||||||
|
| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. |
|
||||||
|
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||||
|
|
||||||
|
All tools resolve voice profiles in this precedence:
|
||||||
|
|
||||||
|
1. Explicit `profile` arg (name or id — case-insensitive)
|
||||||
|
2. Per-client binding keyed by `X-Voicebox-Client-Id`
|
||||||
|
3. `capture_settings.default_playback_voice_id` (global default)
|
||||||
|
|
||||||
|
Bindings are managed via `GET|PUT /mcp/bindings` or in the app under
|
||||||
|
Settings → MCP.
|
||||||
|
|
||||||
|
## Debug with MCP Inspector
|
||||||
|
|
||||||
|
```
|
||||||
|
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Point it at the URL, hit "List tools," call `voicebox.list_profiles`
|
||||||
|
first to confirm wiring, then `voicebox.speak` for end-to-end.
|
||||||
|
|
||||||
|
## Non-MCP REST surface
|
||||||
|
|
||||||
|
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||||
|
don't speak MCP (shell scripts, ACP, A2A):
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -X POST http://127.0.0.1:17493/speak \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-H 'X-Voicebox-Client-Id: claude-code' \
|
||||||
|
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Code layout
|
||||||
|
|
||||||
|
```
|
||||||
|
backend/mcp_server/
|
||||||
|
├── __init__.py # re-export mount_into
|
||||||
|
├── server.py # build_mcp_server() + mount_into(app)
|
||||||
|
├── tools.py # @mcp.tool() implementations
|
||||||
|
├── context.py # ClientIdMiddleware + current_client_id ContextVar
|
||||||
|
├── resolve.py # profile resolution precedence
|
||||||
|
├── events.py # pub/sub queue for /events/speak pill SSE
|
||||||
|
└── README.md # you are here
|
||||||
|
|
||||||
|
backend/mcp_shim/ # stdio ↔ Streamable-HTTP proxy (see its README)
|
||||||
|
```
|
||||||
|
|
||||||
|
The package is **`mcp_server`**, not `mcp`, to avoid shadowing the
|
||||||
|
installed `mcp` PyPI package that FastMCP imports internally.
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""Model Context Protocol server — exposes Voicebox tools to local AI agents.
|
||||||
|
|
||||||
|
Mounts a FastMCP instance at /mcp on the main FastAPI app (Streamable HTTP).
|
||||||
|
A bundled stdio shim (backend/mcp_shim) forwards JSON-RPC into the same
|
||||||
|
endpoint for MCP clients that only speak stdio.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .server import mount_into
|
||||||
|
|
||||||
|
__all__ = ["mount_into"]
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Per-request client identity for MCP calls.
|
||||||
|
|
||||||
|
MCP clients identify themselves via an ``X-Voicebox-Client-Id`` HTTP header
|
||||||
|
(direct-HTTP clients set it in their MCP config; the stdio shim forwards it
|
||||||
|
from the ``VOICEBOX_CLIENT_ID`` env var). Middleware copies the value into a
|
||||||
|
ContextVar so tool implementations can read it without plumbing the request
|
||||||
|
object through every service call.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from contextvars import ContextVar
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
from starlette.requests import Request
|
||||||
|
from starlette.responses import Response
|
||||||
|
from starlette.types import ASGIApp
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||||
|
|
||||||
|
# Tool handlers read this to apply per-client voice bindings.
|
||||||
|
current_client_id: ContextVar[str | None] = ContextVar(
|
||||||
|
"current_client_id", default=None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ClientIdMiddleware(BaseHTTPMiddleware):
|
||||||
|
"""Copy X-Voicebox-Client-Id into a ContextVar and stamp last_seen_at.
|
||||||
|
|
||||||
|
Only stamps on MCP-endpoint requests (anything under ``/mcp``) so
|
||||||
|
unrelated REST traffic with the header set won't advance the
|
||||||
|
last-seen timestamp — the Settings UI uses that to show when each
|
||||||
|
client was last heard from.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, app: ASGIApp) -> None:
|
||||||
|
super().__init__(app)
|
||||||
|
|
||||||
|
async def dispatch(self, request: Request, call_next) -> Response:
|
||||||
|
client_id = request.headers.get(CLIENT_ID_HEADER)
|
||||||
|
token = current_client_id.set(client_id)
|
||||||
|
try:
|
||||||
|
response = await call_next(request)
|
||||||
|
finally:
|
||||||
|
current_client_id.reset(token)
|
||||||
|
|
||||||
|
if client_id and request.url.path.startswith("/mcp"):
|
||||||
|
_stamp_last_seen(client_id)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def _stamp_last_seen(client_id: str) -> None:
|
||||||
|
"""Update or create the MCPClientBinding row for this client_id."""
|
||||||
|
try:
|
||||||
|
from ..database import get_db
|
||||||
|
from ..database.models import MCPClientBinding
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
db = next(get_db())
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
row = (
|
||||||
|
db.query(MCPClientBinding)
|
||||||
|
.filter(MCPClientBinding.client_id == client_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
row = MCPClientBinding(client_id=client_id)
|
||||||
|
db.add(row)
|
||||||
|
row.last_seen_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
except Exception:
|
||||||
|
logger.debug(
|
||||||
|
"Could not stamp last_seen_at for %s", client_id, exc_info=True
|
||||||
|
)
|
||||||
|
db.rollback()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""In-memory pub/sub for speaking-pill SSE broadcasts.
|
||||||
|
|
||||||
|
MCP ``voicebox.speak`` calls and the REST ``POST /speak`` route publish
|
||||||
|
start/end events that DictateWindow subscribes to via /events/speak, so the
|
||||||
|
floating pill surfaces whenever an agent is speaking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
# Each subscriber gets its own queue. Bounded to drop oldest if a client lags.
|
||||||
|
_subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def subscribe() -> asyncio.Queue[dict[str, Any]]:
|
||||||
|
"""Register a new subscriber; caller must call unsubscribe() when done."""
|
||||||
|
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=64)
|
||||||
|
_subscribers.add(queue)
|
||||||
|
return queue
|
||||||
|
|
||||||
|
|
||||||
|
def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||||
|
_subscribers.discard(queue)
|
||||||
|
|
||||||
|
|
||||||
|
def publish(kind: str, payload: dict[str, Any]) -> None:
|
||||||
|
"""Fan out to all current subscribers. Non-blocking; drops on full queue."""
|
||||||
|
event = {"kind": kind, **payload}
|
||||||
|
for queue in list(_subscribers):
|
||||||
|
try:
|
||||||
|
queue.put_nowait(event)
|
||||||
|
except asyncio.QueueFull:
|
||||||
|
# Slow subscriber — skip rather than block publishers.
|
||||||
|
pass
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""Voice profile resolution for MCP tool calls.
|
||||||
|
|
||||||
|
Precedence:
|
||||||
|
1. Explicit tool arg (profile name or id)
|
||||||
|
2. Per-client MCPClientBinding.profile_id
|
||||||
|
3. CaptureSettings.default_playback_voice_id (global default)
|
||||||
|
4. None — caller raises a helpful error
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..database import VoiceProfile as DBVoiceProfile, get_db
|
||||||
|
from ..database.models import CaptureSettings
|
||||||
|
from ..services.profiles import get_profile_orm_by_name_or_id as _lookup_profile
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_profile(
|
||||||
|
explicit: str | None,
|
||||||
|
client_id: str | None,
|
||||||
|
db: Session,
|
||||||
|
) -> DBVoiceProfile | None:
|
||||||
|
"""Apply the full precedence chain and return the profile ORM row (or None)."""
|
||||||
|
if explicit:
|
||||||
|
profile = _lookup_profile(explicit, db)
|
||||||
|
if profile is not None:
|
||||||
|
return profile
|
||||||
|
# Explicit but not found — return None so the caller can report it.
|
||||||
|
return None
|
||||||
|
|
||||||
|
if client_id:
|
||||||
|
# Per-client binding. Imported lazily so this module stays importable
|
||||||
|
# even before the migration adds the table on first boot.
|
||||||
|
from ..database.models import MCPClientBinding # noqa: WPS433
|
||||||
|
|
||||||
|
binding = (
|
||||||
|
db.query(MCPClientBinding)
|
||||||
|
.filter(MCPClientBinding.client_id == client_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if binding and binding.profile_id:
|
||||||
|
profile = _lookup_profile(binding.profile_id, db)
|
||||||
|
if profile is not None:
|
||||||
|
return profile
|
||||||
|
|
||||||
|
# Global default from capture settings.
|
||||||
|
settings = db.query(CaptureSettings).filter(CaptureSettings.id == 1).first()
|
||||||
|
if settings and settings.default_playback_voice_id:
|
||||||
|
profile = _lookup_profile(settings.default_playback_voice_id, db)
|
||||||
|
if profile is not None:
|
||||||
|
return profile
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def with_db() -> Session:
|
||||||
|
"""Utility for tool handlers that aren't managed by FastAPI's Depends."""
|
||||||
|
return next(get_db())
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Construct the FastMCP server and mount it on the FastAPI app.
|
||||||
|
|
||||||
|
The MCP endpoint lives at ``/mcp`` (Streamable HTTP transport). Modern MCP
|
||||||
|
clients (Claude Code, Cursor, Windsurf, VS Code MCP extensions) connect
|
||||||
|
directly via URL; older stdio-only clients use the ``voicebox-mcp`` shim
|
||||||
|
binary bundled with the desktop app.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from contextlib import AsyncExitStack, asynccontextmanager
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
|
from .context import ClientIdMiddleware
|
||||||
|
from .tools import register_tools
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def build_mcp_server() -> FastMCP:
|
||||||
|
"""Create the FastMCP instance with Voicebox tools registered."""
|
||||||
|
mcp = FastMCP(
|
||||||
|
name="voicebox",
|
||||||
|
instructions=(
|
||||||
|
"Voicebox is a local voice I/O layer. Use `voicebox.speak` to "
|
||||||
|
"play text in a voice profile, `voicebox.transcribe` for "
|
||||||
|
"audio→text, and the `list_*` tools to discover profiles and "
|
||||||
|
"captures."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
register_tools(mcp)
|
||||||
|
return mcp
|
||||||
|
|
||||||
|
|
||||||
|
def mount_into(
|
||||||
|
app: FastAPI,
|
||||||
|
*,
|
||||||
|
extra_startup: Callable[[], None] | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Attach the MCP app to ``app`` at ``/mcp`` and install the client-id middleware.
|
||||||
|
|
||||||
|
``extra_startup`` — if provided, runs during the FastAPI lifespan. This
|
||||||
|
is the hook that lets ``app.py`` keep its existing startup/shutdown
|
||||||
|
bodies while also driving FastMCP's session manager.
|
||||||
|
"""
|
||||||
|
mcp = build_mcp_server()
|
||||||
|
mcp_app = mcp.http_app(path="/", transport="http")
|
||||||
|
|
||||||
|
# ClientIdMiddleware must run before FastMCP so the ContextVar is set
|
||||||
|
# by the time tool handlers execute. Starlette composes middlewares
|
||||||
|
# outermost-first, so adding here on the parent app is correct.
|
||||||
|
app.add_middleware(ClientIdMiddleware)
|
||||||
|
app.mount("/mcp", mcp_app)
|
||||||
|
app.state.mcp_lifespan = mcp_app.router.lifespan_context
|
||||||
|
logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", ""))
|
||||||
|
|
||||||
|
|
||||||
|
def compose_lifespan(*lifespans):
|
||||||
|
"""Combine multiple async context managers into a single FastAPI lifespan.
|
||||||
|
|
||||||
|
Used by ``create_app`` to run the existing Voicebox startup/shutdown
|
||||||
|
together with FastMCP's session manager (which MUST run in the
|
||||||
|
ASGI lifespan for Streamable HTTP to work).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _combined(app):
|
||||||
|
async with AsyncExitStack() as stack:
|
||||||
|
for cm_factory in lifespans:
|
||||||
|
cm = cm_factory(app) if callable(cm_factory) else cm_factory
|
||||||
|
await stack.enter_async_context(cm)
|
||||||
|
yield
|
||||||
|
|
||||||
|
return _combined
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
"""Voicebox MCP tool implementations.
|
||||||
|
|
||||||
|
Thin wrappers over existing services/routes. Tools are registered with dotted
|
||||||
|
names (``voicebox.speak`` etc.) so they look natural in agent logs —
|
||||||
|
the Python function name stays snake_case.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import base64 as b64
|
||||||
|
import logging
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
|
from .. import models
|
||||||
|
from ..database import get_db
|
||||||
|
from ..services import captures as captures_service
|
||||||
|
from ..services import profiles as profiles_service
|
||||||
|
from . import events as mcp_events
|
||||||
|
from .context import current_client_id
|
||||||
|
from .resolve import resolve_profile
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Absolute-path transcribes are bounded to keep a bad client from
|
||||||
|
# asking us to ingest a 20 GB file.
|
||||||
|
MAX_TRANSCRIBE_BYTES = 200 * 1024 * 1024 # 200 MB
|
||||||
|
|
||||||
|
|
||||||
|
def register_tools(mcp: FastMCP) -> None:
|
||||||
|
"""Attach all Voicebox tools to the given FastMCP instance."""
|
||||||
|
|
||||||
|
@mcp.tool(
|
||||||
|
name="voicebox.speak",
|
||||||
|
description=(
|
||||||
|
"Speak text in a Voicebox voice profile. Returns a generation id "
|
||||||
|
"the caller can poll at /generate/{id}/status. Audio plays on the "
|
||||||
|
"user's speakers and is saved to the Captures / History tab."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
async def voicebox_speak(
|
||||||
|
text: str,
|
||||||
|
profile: str | None = None,
|
||||||
|
engine: str | None = None,
|
||||||
|
intent: Literal["respond", "rewrite", "compose"] | None = None,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Speak ``text`` in a voice profile.
|
||||||
|
|
||||||
|
``profile`` accepts a voice profile name (e.g. "Morgan") or id. If
|
||||||
|
omitted, the server looks up the per-client binding for the calling
|
||||||
|
MCP client, then falls back to the global default voice.
|
||||||
|
|
||||||
|
``intent`` only matters for profiles that have a personality prompt —
|
||||||
|
when set, the text is first transformed by the LLM (respond to it,
|
||||||
|
rewrite it in character, or compose a fresh utterance). Leave unset
|
||||||
|
for plain TTS.
|
||||||
|
"""
|
||||||
|
db = next(get_db())
|
||||||
|
try:
|
||||||
|
vp = resolve_profile(profile, current_client_id.get(), db)
|
||||||
|
if vp is None:
|
||||||
|
raise ValueError(
|
||||||
|
"No voice profile resolved. Pass `profile=` with a "
|
||||||
|
"voice profile name or id, or set a default voice in "
|
||||||
|
"Voicebox → Settings → MCP."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Persona path if intent requested and personality present.
|
||||||
|
if intent is not None and vp.personality:
|
||||||
|
return await _speak_with_persona(
|
||||||
|
profile_id=vp.id,
|
||||||
|
profile_name=vp.name,
|
||||||
|
text=text,
|
||||||
|
engine=engine,
|
||||||
|
intent=intent,
|
||||||
|
language=language,
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
|
||||||
|
return await _speak_plain(
|
||||||
|
profile_id=vp.id,
|
||||||
|
profile_name=vp.name,
|
||||||
|
text=text,
|
||||||
|
engine=engine,
|
||||||
|
language=language,
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
@mcp.tool(
|
||||||
|
name="voicebox.transcribe",
|
||||||
|
description=(
|
||||||
|
"Transcribe an audio clip to text using Voicebox's local Whisper. "
|
||||||
|
"Pass exactly one of `audio_base64` (bytes as base64) or "
|
||||||
|
"`audio_path` (absolute local file path)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
async def voicebox_transcribe(
|
||||||
|
audio_base64: str | None = None,
|
||||||
|
audio_path: str | None = None,
|
||||||
|
language: str | None = None,
|
||||||
|
model: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if bool(audio_base64) == bool(audio_path):
|
||||||
|
raise ValueError(
|
||||||
|
"Pass exactly one of `audio_base64` or `audio_path`."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Absolute-path mode: validate and transcribe in place.
|
||||||
|
if audio_path is not None:
|
||||||
|
path = Path(audio_path)
|
||||||
|
if not path.is_absolute():
|
||||||
|
raise ValueError("`audio_path` must be absolute.")
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"File not found: {audio_path}")
|
||||||
|
if path.stat().st_size > MAX_TRANSCRIBE_BYTES:
|
||||||
|
raise ValueError(
|
||||||
|
f"File exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||||
|
)
|
||||||
|
return await _transcribe_file(path, language, model)
|
||||||
|
|
||||||
|
# Base64 mode: decode into a temp file, transcribe, clean up.
|
||||||
|
try:
|
||||||
|
raw = b64.b64decode(audio_base64, validate=True)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ValueError(f"Invalid audio_base64: {exc}") from exc
|
||||||
|
if len(raw) > MAX_TRANSCRIBE_BYTES:
|
||||||
|
raise ValueError(
|
||||||
|
f"Audio exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||||
|
)
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
suffix=".wav", delete=False
|
||||||
|
) as tmp:
|
||||||
|
tmp.write(raw)
|
||||||
|
tmp_path = Path(tmp.name)
|
||||||
|
try:
|
||||||
|
return await _transcribe_file(tmp_path, language, model)
|
||||||
|
finally:
|
||||||
|
tmp_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
@mcp.tool(
|
||||||
|
name="voicebox.list_captures",
|
||||||
|
description=(
|
||||||
|
"List recent voice captures (dictations, recordings, uploads) "
|
||||||
|
"with their transcripts. Most-recent first."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
async def voicebox_list_captures(
|
||||||
|
limit: int = 20, offset: int = 0
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not (1 <= limit <= 200):
|
||||||
|
raise ValueError("`limit` must be between 1 and 200.")
|
||||||
|
if offset < 0:
|
||||||
|
raise ValueError("`offset` must be >= 0.")
|
||||||
|
db = next(get_db())
|
||||||
|
try:
|
||||||
|
items, total = captures_service.list_captures(
|
||||||
|
db, limit=limit, offset=offset
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"captures": [
|
||||||
|
item.model_dump(mode="json") for item in items
|
||||||
|
],
|
||||||
|
"total": total,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
@mcp.tool(
|
||||||
|
name="voicebox.list_profiles",
|
||||||
|
description=(
|
||||||
|
"List available voice profiles (both cloned voices and presets). "
|
||||||
|
"Use the returned `name` with voicebox.speak(profile=...)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
async def voicebox_list_profiles() -> dict[str, Any]:
|
||||||
|
db = next(get_db())
|
||||||
|
try:
|
||||||
|
profiles = await profiles_service.list_profiles(db)
|
||||||
|
return {
|
||||||
|
"profiles": [
|
||||||
|
{
|
||||||
|
"id": p.id,
|
||||||
|
"name": p.name,
|
||||||
|
"voice_type": p.voice_type,
|
||||||
|
"language": p.language,
|
||||||
|
"has_personality": bool(getattr(p, "personality", None)),
|
||||||
|
}
|
||||||
|
for p in profiles
|
||||||
|
]
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Speak helpers ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
async def _speak_plain(
|
||||||
|
*,
|
||||||
|
profile_id: str,
|
||||||
|
profile_name: str,
|
||||||
|
text: str,
|
||||||
|
engine: str | None,
|
||||||
|
language: str | None,
|
||||||
|
db,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Plain TTS path — mirrors POST /generate. No LLM transform."""
|
||||||
|
from ..routes.generations import generate_speech
|
||||||
|
|
||||||
|
req = models.GenerationRequest(
|
||||||
|
profile_id=profile_id,
|
||||||
|
text=text,
|
||||||
|
language=language or "en",
|
||||||
|
engine=engine or "qwen",
|
||||||
|
)
|
||||||
|
generation = await generate_speech(req, db)
|
||||||
|
return _speak_response(generation, profile_name, source="mcp")
|
||||||
|
|
||||||
|
|
||||||
|
async def _speak_with_persona(
|
||||||
|
*,
|
||||||
|
profile_id: str,
|
||||||
|
profile_name: str,
|
||||||
|
text: str,
|
||||||
|
engine: str | None,
|
||||||
|
intent: str,
|
||||||
|
language: str | None,
|
||||||
|
db,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""LLM-transformed path — reuses POST /profiles/{id}/speak."""
|
||||||
|
from ..routes.profiles import speak_in_character
|
||||||
|
|
||||||
|
req = models.PersonalitySpeakRequest(
|
||||||
|
text=text,
|
||||||
|
persist=True,
|
||||||
|
language=language,
|
||||||
|
engine=engine,
|
||||||
|
intent=intent,
|
||||||
|
)
|
||||||
|
generation = await speak_in_character(profile_id, req, db)
|
||||||
|
return _speak_response(generation, profile_name, source="mcp")
|
||||||
|
|
||||||
|
|
||||||
|
def _speak_response(
|
||||||
|
generation, profile_name: str, *, source: str
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Normalize a GenerationResponse into the MCP tool's return shape.
|
||||||
|
|
||||||
|
Also fires a speak-start event so the DictateWindow pill surfaces
|
||||||
|
the agent's speech. Speak-end is fired from run_generation's
|
||||||
|
completion hook.
|
||||||
|
"""
|
||||||
|
payload = generation.model_dump(mode="json") if hasattr(
|
||||||
|
generation, "model_dump"
|
||||||
|
) else dict(generation)
|
||||||
|
generation_id = payload.get("id")
|
||||||
|
mcp_events.publish(
|
||||||
|
"speak-start",
|
||||||
|
{
|
||||||
|
"generation_id": generation_id,
|
||||||
|
"profile_name": profile_name,
|
||||||
|
"source": source,
|
||||||
|
"client_id": current_client_id.get(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"generation_id": generation_id,
|
||||||
|
"status": payload.get("status"),
|
||||||
|
"profile": profile_name,
|
||||||
|
"source": source,
|
||||||
|
"poll_url": f"/generate/{generation_id}/status"
|
||||||
|
if generation_id
|
||||||
|
else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ─── Transcribe helper ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
async def _transcribe_file(
|
||||||
|
path: Path, language: str | None, model: str | None
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
from ..backends import WHISPER_HF_REPOS
|
||||||
|
from ..services import transcribe as transcribe_service
|
||||||
|
from ..utils.audio import load_audio
|
||||||
|
|
||||||
|
whisper = transcribe_service.get_whisper_model()
|
||||||
|
model_size = model or whisper.model_size
|
||||||
|
valid = list(WHISPER_HF_REPOS.keys())
|
||||||
|
if model_size not in valid:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid STT model '{model_size}'. Must be one of: {', '.join(valid)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# load_audio is sync; keep the event loop responsive.
|
||||||
|
audio, sr = await asyncio.to_thread(load_audio, str(path))
|
||||||
|
duration = len(audio) / sr
|
||||||
|
|
||||||
|
if (
|
||||||
|
not whisper.is_loaded() or whisper.model_size != model_size
|
||||||
|
) and not whisper._is_model_cached(model_size):
|
||||||
|
raise ValueError(
|
||||||
|
f"Whisper model '{model_size}' is not yet downloaded. Open "
|
||||||
|
"Voicebox → Settings → Models to download it first."
|
||||||
|
)
|
||||||
|
|
||||||
|
text = await whisper.transcribe(str(path), language, model_size)
|
||||||
|
return {
|
||||||
|
"text": text,
|
||||||
|
"duration": duration,
|
||||||
|
"language": language,
|
||||||
|
"model": model_size,
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""Stdio → Streamable HTTP bridge for the Voicebox MCP server.
|
||||||
|
|
||||||
|
Some MCP clients only know how to spawn a subprocess and talk to it over
|
||||||
|
stdin/stdout (the "stdio" transport). This package is a ~150-line adapter:
|
||||||
|
the client spawns us as ``voicebox-mcp``; we proxy every JSON-RPC frame
|
||||||
|
to http://127.0.0.1:17493/mcp/ and stream responses back out.
|
||||||
|
|
||||||
|
All the real work (tools, models, inference) lives in the Voicebox server
|
||||||
|
process — this package contains no business logic.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
"""voicebox-mcp — stdio ↔ Streamable-HTTP MCP proxy.
|
||||||
|
|
||||||
|
Some MCP clients only speak stdio. They spawn this binary, we pipe each
|
||||||
|
JSON-RPC message to ``http://127.0.0.1:<port>/mcp/``, and stream the
|
||||||
|
server's response back. The Voicebox server does all the real work.
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
VOICEBOX_PORT Voicebox server port (default 17493).
|
||||||
|
VOICEBOX_HOST Host (default 127.0.0.1).
|
||||||
|
VOICEBOX_CLIENT_ID Forwarded as X-Voicebox-Client-Id on every request.
|
||||||
|
|
||||||
|
Stdout is JSON-RPC only. Diagnostics go to stderr.
|
||||||
|
Exit 0 on clean EOF, 1 on transport error, 2 if backend never answers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
|
||||||
|
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||||
|
SESSION_HEADER = "mcp-session-id"
|
||||||
|
HEALTH_TIMEOUT_S = 30.0
|
||||||
|
DEFAULT_PORT = 17493
|
||||||
|
|
||||||
|
|
||||||
|
def _err(msg: str) -> None:
|
||||||
|
print(f"voicebox-mcp: {msg}", file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _base_url() -> tuple[str, str]:
|
||||||
|
host = os.environ.get("VOICEBOX_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("VOICEBOX_PORT", str(DEFAULT_PORT)))
|
||||||
|
return f"http://{host}:{port}/mcp/", f"http://{host}:{port}/health"
|
||||||
|
|
||||||
|
|
||||||
|
async def _wait_for_backend(client: httpx.AsyncClient, health_url: str) -> bool:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
deadline = loop.time() + HEALTH_TIMEOUT_S
|
||||||
|
while loop.time() < deadline:
|
||||||
|
try:
|
||||||
|
r = await client.get(health_url, timeout=2.0)
|
||||||
|
if r.status_code == 200:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def _read_stdin_line() -> str | None:
|
||||||
|
"""Async-read a single line from stdin. Returns None on EOF."""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
line = await loop.run_in_executor(None, sys.stdin.readline)
|
||||||
|
if not line:
|
||||||
|
return None
|
||||||
|
return line
|
||||||
|
|
||||||
|
|
||||||
|
def _write_stdout(obj: Any) -> None:
|
||||||
|
"""Write a JSON object to stdout as one line, flushed."""
|
||||||
|
sys.stdout.write(json.dumps(obj, separators=(",", ":")))
|
||||||
|
sys.stdout.write("\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
async def _handle_request(
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
url: str,
|
||||||
|
raw: str,
|
||||||
|
headers: dict[str, str],
|
||||||
|
session_id: list[str | None],
|
||||||
|
) -> None:
|
||||||
|
"""Forward one JSON-RPC payload to the server and relay the response."""
|
||||||
|
try:
|
||||||
|
message = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
_err(f"invalid JSON on stdin: {exc}")
|
||||||
|
return
|
||||||
|
|
||||||
|
req_headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json, text/event-stream",
|
||||||
|
**headers,
|
||||||
|
}
|
||||||
|
if session_id[0]:
|
||||||
|
req_headers[SESSION_HEADER] = session_id[0]
|
||||||
|
|
||||||
|
# Notifications (no "id") don't expect a response body. Server returns
|
||||||
|
# 202 Accepted and we stay quiet.
|
||||||
|
is_notification = isinstance(message, dict) and "id" not in message
|
||||||
|
|
||||||
|
async with client.stream(
|
||||||
|
"POST", url, headers=req_headers, content=raw.encode("utf-8")
|
||||||
|
) as response:
|
||||||
|
# Capture session id on initialize.
|
||||||
|
if session_id[0] is None:
|
||||||
|
sid = response.headers.get(SESSION_HEADER)
|
||||||
|
if sid:
|
||||||
|
session_id[0] = sid
|
||||||
|
|
||||||
|
if response.status_code == 202:
|
||||||
|
return # notification acknowledged
|
||||||
|
if response.status_code >= 400:
|
||||||
|
body = await response.aread()
|
||||||
|
_err(
|
||||||
|
f"server {response.status_code}: "
|
||||||
|
f"{body.decode('utf-8', errors='replace')[:400]}"
|
||||||
|
)
|
||||||
|
if is_notification:
|
||||||
|
return
|
||||||
|
_write_stdout(
|
||||||
|
{
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": message.get("id"),
|
||||||
|
"error": {
|
||||||
|
"code": -32000,
|
||||||
|
"message": (
|
||||||
|
f"Voicebox MCP proxy got HTTP {response.status_code}"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
ctype = response.headers.get("content-type", "")
|
||||||
|
if "text/event-stream" in ctype:
|
||||||
|
# SSE frames: lines prefixed "data: ..." contain the JSON-RPC msg.
|
||||||
|
async for line in response.aiter_lines():
|
||||||
|
if line.startswith("data:"):
|
||||||
|
payload = line[5:].strip()
|
||||||
|
if not payload:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
_write_stdout(json.loads(payload))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
_err(f"malformed SSE payload: {payload[:200]}")
|
||||||
|
else:
|
||||||
|
body = await response.aread()
|
||||||
|
try:
|
||||||
|
_write_stdout(json.loads(body))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
_err(
|
||||||
|
f"non-JSON response ({ctype}): "
|
||||||
|
f"{body.decode('utf-8', errors='replace')[:200]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run() -> int:
|
||||||
|
url, health_url = _base_url()
|
||||||
|
forward_headers: dict[str, str] = {}
|
||||||
|
client_id = os.environ.get("VOICEBOX_CLIENT_ID")
|
||||||
|
if client_id:
|
||||||
|
forward_headers[CLIENT_ID_HEADER] = client_id
|
||||||
|
|
||||||
|
session_id: list[str | None] = [None]
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=httpx.Timeout(300.0)) as client:
|
||||||
|
if not await _wait_for_backend(client, health_url):
|
||||||
|
_err(
|
||||||
|
f"timed out waiting for Voicebox at {health_url} — is the app open?"
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
line = await _read_stdin_line()
|
||||||
|
if line is None:
|
||||||
|
return 0
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
await _handle_request(
|
||||||
|
client, url, line, forward_headers, session_id
|
||||||
|
)
|
||||||
|
except (KeyboardInterrupt, SystemExit):
|
||||||
|
return 0
|
||||||
|
except Exception as exc:
|
||||||
|
_err(f"proxy failed: {exc!r}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
return asyncio.run(_run())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -248,6 +248,7 @@ class CaptureSettingsResponse(BaseModel):
|
|||||||
preserve_technical: bool = True
|
preserve_technical: bool = True
|
||||||
allow_auto_paste: bool = True
|
allow_auto_paste: bool = True
|
||||||
default_playback_voice_id: Optional[str] = None
|
default_playback_voice_id: Optional[str] = None
|
||||||
|
hotkey_enabled: bool = False
|
||||||
chord_push_to_talk_keys: List[str] = Field(default_factory=lambda: ["MetaRight", "AltGr"])
|
chord_push_to_talk_keys: List[str] = Field(default_factory=lambda: ["MetaRight", "AltGr"])
|
||||||
chord_toggle_to_talk_keys: List[str] = Field(
|
chord_toggle_to_talk_keys: List[str] = Field(
|
||||||
default_factory=lambda: ["MetaRight", "AltGr", "Space"]
|
default_factory=lambda: ["MetaRight", "AltGr", "Space"]
|
||||||
@@ -269,6 +270,7 @@ class CaptureSettingsUpdate(BaseModel):
|
|||||||
preserve_technical: Optional[bool] = None
|
preserve_technical: Optional[bool] = None
|
||||||
allow_auto_paste: Optional[bool] = None
|
allow_auto_paste: Optional[bool] = None
|
||||||
default_playback_voice_id: Optional[str] = None
|
default_playback_voice_id: Optional[str] = None
|
||||||
|
hotkey_enabled: Optional[bool] = None
|
||||||
chord_push_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
chord_push_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||||
chord_toggle_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
chord_toggle_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||||
|
|
||||||
@@ -294,6 +296,68 @@ class GenerationSettingsUpdate(BaseModel):
|
|||||||
autoplay_on_generate: Optional[bool] = None
|
autoplay_on_generate: Optional[bool] = None
|
||||||
|
|
||||||
|
|
||||||
|
class MCPClientBindingResponse(BaseModel):
|
||||||
|
"""Per-MCP-client voice binding — what voice / engine / intent the server
|
||||||
|
should use when a given client_id calls voicebox.speak without args."""
|
||||||
|
|
||||||
|
client_id: str
|
||||||
|
label: Optional[str] = None
|
||||||
|
profile_id: Optional[str] = None
|
||||||
|
default_engine: Optional[str] = Field(
|
||||||
|
None,
|
||||||
|
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||||
|
)
|
||||||
|
default_intent: Optional[str] = Field(
|
||||||
|
None, pattern="^(respond|rewrite|compose)$"
|
||||||
|
)
|
||||||
|
last_seen_at: Optional[datetime] = None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
|
||||||
|
class MCPClientBindingUpsert(BaseModel):
|
||||||
|
"""Create or update a binding. Matched by ``client_id``."""
|
||||||
|
|
||||||
|
client_id: str = Field(..., min_length=1, max_length=64)
|
||||||
|
label: Optional[str] = Field(None, max_length=128)
|
||||||
|
profile_id: Optional[str] = None
|
||||||
|
default_engine: Optional[str] = Field(
|
||||||
|
None,
|
||||||
|
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||||
|
)
|
||||||
|
default_intent: Optional[str] = Field(
|
||||||
|
None, pattern="^(respond|rewrite|compose)$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MCPClientBindingListResponse(BaseModel):
|
||||||
|
items: List[MCPClientBindingResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class SpeakRequest(BaseModel):
|
||||||
|
"""Body for POST /speak — non-MCP REST surface that mirrors voicebox.speak."""
|
||||||
|
|
||||||
|
text: str = Field(..., min_length=1, max_length=10000)
|
||||||
|
profile: Optional[str] = Field(
|
||||||
|
None,
|
||||||
|
description="Voice profile name or id. Falls back to per-client binding, then default.",
|
||||||
|
)
|
||||||
|
engine: Optional[str] = Field(
|
||||||
|
None,
|
||||||
|
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||||
|
)
|
||||||
|
intent: Optional[str] = Field(
|
||||||
|
None, pattern="^(respond|rewrite|compose)$"
|
||||||
|
)
|
||||||
|
language: Optional[str] = Field(
|
||||||
|
None,
|
||||||
|
pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class LLMGenerateRequest(BaseModel):
|
class LLMGenerateRequest(BaseModel):
|
||||||
"""Request model for LLM text generation."""
|
"""Request model for LLM text generation."""
|
||||||
|
|
||||||
|
|||||||
@@ -62,6 +62,11 @@ pedalboard>=0.9.0
|
|||||||
# HTTP client (for CUDA backend download)
|
# HTTP client (for CUDA backend download)
|
||||||
httpx>=0.27.0
|
httpx>=0.27.0
|
||||||
|
|
||||||
|
# MCP server (Model Context Protocol) — lets local AI agents call
|
||||||
|
# voicebox.speak / .transcribe / .list_captures / .list_profiles
|
||||||
|
fastmcp>=3.0,<4.0
|
||||||
|
sse-starlette>=2.0
|
||||||
|
|
||||||
# Utilities
|
# Utilities
|
||||||
python-multipart>=0.0.6
|
python-multipart>=0.0.6
|
||||||
Pillow>=10.0.0
|
Pillow>=10.0.0
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ def register_routers(app: FastAPI) -> None:
|
|||||||
from .settings import router as settings_router
|
from .settings import router as settings_router
|
||||||
from .tasks import router as tasks_router
|
from .tasks import router as tasks_router
|
||||||
from .cuda import router as cuda_router
|
from .cuda import router as cuda_router
|
||||||
|
from .speak import router as speak_router
|
||||||
|
from .mcp_bindings import router as mcp_bindings_router
|
||||||
|
from .events import router as events_router
|
||||||
|
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
app.include_router(profiles_router)
|
app.include_router(profiles_router)
|
||||||
@@ -36,3 +39,6 @@ def register_routers(app: FastAPI) -> None:
|
|||||||
app.include_router(settings_router)
|
app.include_router(settings_router)
|
||||||
app.include_router(tasks_router)
|
app.include_router(tasks_router)
|
||||||
app.include_router(cuda_router)
|
app.include_router(cuda_router)
|
||||||
|
app.include_router(speak_router)
|
||||||
|
app.include_router(mcp_bindings_router)
|
||||||
|
app.include_router(events_router)
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Server-Sent-Event streams the frontend subscribes to.
|
||||||
|
|
||||||
|
``GET /events/speak`` — broadcasts ``speak-start`` / ``speak-end`` events
|
||||||
|
whenever an agent-initiated speak (MCP tool or POST /speak) runs. The
|
||||||
|
DictateWindow uses them to show the floating pill in a `speaking` state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Request
|
||||||
|
from sse_starlette.sse import EventSourceResponse
|
||||||
|
|
||||||
|
from ..mcp_server import events as mcp_events
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/events/speak")
|
||||||
|
async def speak_events(request: Request):
|
||||||
|
"""SSE stream of speak-start / speak-end events."""
|
||||||
|
|
||||||
|
async def event_stream():
|
||||||
|
queue = mcp_events.subscribe()
|
||||||
|
try:
|
||||||
|
# Immediate hello so EventSource knows the connection is live.
|
||||||
|
yield {"event": "ready", "data": "{}"}
|
||||||
|
while True:
|
||||||
|
if await request.is_disconnected():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
event = await asyncio.wait_for(queue.get(), timeout=15.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
# Heartbeat so proxies don't reap idle streams.
|
||||||
|
yield {"event": "ping", "data": "{}"}
|
||||||
|
continue
|
||||||
|
kind = event.pop("kind", "message")
|
||||||
|
yield {"event": kind, "data": json.dumps(event)}
|
||||||
|
finally:
|
||||||
|
mcp_events.unsubscribe(queue)
|
||||||
|
|
||||||
|
return EventSourceResponse(event_stream())
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""REST endpoints for per-MCP-client voice binding settings.
|
||||||
|
|
||||||
|
The Settings UI uses these to let users configure distinct voices per
|
||||||
|
agent (Claude Code in Morgan, Cursor in Scarlett, ...). The ``client_id``
|
||||||
|
column is the same value the MCP client sends in ``X-Voicebox-Client-Id``
|
||||||
|
(or the stdio shim pulls from ``VOICEBOX_CLIENT_ID``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .. import models
|
||||||
|
from ..database import get_db
|
||||||
|
from ..database.models import MCPClientBinding
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/mcp/bindings",
|
||||||
|
response_model=models.MCPClientBindingListResponse,
|
||||||
|
)
|
||||||
|
async def list_mcp_bindings(db: Session = Depends(get_db)):
|
||||||
|
rows = (
|
||||||
|
db.query(MCPClientBinding)
|
||||||
|
.order_by(MCPClientBinding.client_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return models.MCPClientBindingListResponse(
|
||||||
|
items=[models.MCPClientBindingResponse.model_validate(r) for r in rows]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.put(
|
||||||
|
"/mcp/bindings",
|
||||||
|
response_model=models.MCPClientBindingResponse,
|
||||||
|
)
|
||||||
|
async def upsert_mcp_binding(
|
||||||
|
data: models.MCPClientBindingUpsert,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
"""Create-or-update a binding. Matches by client_id."""
|
||||||
|
row = (
|
||||||
|
db.query(MCPClientBinding)
|
||||||
|
.filter(MCPClientBinding.client_id == data.client_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
row = MCPClientBinding(client_id=data.client_id)
|
||||||
|
db.add(row)
|
||||||
|
|
||||||
|
row.label = data.label
|
||||||
|
row.profile_id = data.profile_id
|
||||||
|
row.default_engine = data.default_engine
|
||||||
|
row.default_intent = data.default_intent
|
||||||
|
row.updated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
db.refresh(row)
|
||||||
|
return models.MCPClientBindingResponse.model_validate(row)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/mcp/bindings/{client_id}")
|
||||||
|
async def delete_mcp_binding(
|
||||||
|
client_id: str,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
row = (
|
||||||
|
db.query(MCPClientBinding)
|
||||||
|
.filter(MCPClientBinding.client_id == client_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
raise HTTPException(status_code=404, detail="Binding not found")
|
||||||
|
db.delete(row)
|
||||||
|
db.commit()
|
||||||
|
return {"deleted": client_id}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
"""POST /speak — REST wrapper around voicebox.speak for non-MCP callers.
|
||||||
|
|
||||||
|
Shell scripts, ACP, A2A, or any agent that doesn't speak MCP can hit this
|
||||||
|
endpoint to play text through a cloned voice. Uses the same profile
|
||||||
|
resolution and generation pipeline as the MCP tool, so per-client
|
||||||
|
bindings (via X-Voicebox-Client-Id) work identically.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .. import models
|
||||||
|
from ..database import get_db
|
||||||
|
from ..mcp_server import events as mcp_events
|
||||||
|
from ..mcp_server.resolve import resolve_profile
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/speak", response_model=models.GenerationResponse)
|
||||||
|
async def speak(
|
||||||
|
data: models.SpeakRequest,
|
||||||
|
request: Request,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
"""Speak text in a voice profile. Mirrors voicebox.speak (MCP).
|
||||||
|
|
||||||
|
Response shape matches POST /generate — a ``GenerationResponse`` with
|
||||||
|
``status="generating"`` and an ``id`` the caller polls at
|
||||||
|
``GET /generate/{id}/status``.
|
||||||
|
"""
|
||||||
|
client_id = request.headers.get("X-Voicebox-Client-Id")
|
||||||
|
profile = resolve_profile(data.profile, client_id, db)
|
||||||
|
if profile is None:
|
||||||
|
if data.profile:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail=f"Voice profile '{data.profile}' not found.",
|
||||||
|
)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=(
|
||||||
|
"No voice profile resolved. Pass `profile` (name or id), "
|
||||||
|
"or configure a default in Voicebox → Settings → MCP."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Persona path if intent requested AND profile has a personality prompt.
|
||||||
|
if data.intent is not None and profile.personality:
|
||||||
|
from .profiles import speak_in_character
|
||||||
|
|
||||||
|
generation = await speak_in_character(
|
||||||
|
profile.id,
|
||||||
|
models.PersonalitySpeakRequest(
|
||||||
|
text=data.text,
|
||||||
|
persist=True,
|
||||||
|
language=data.language,
|
||||||
|
engine=data.engine,
|
||||||
|
intent=data.intent,
|
||||||
|
),
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Plain TTS path — matches POST /generate.
|
||||||
|
from .generations import generate_speech
|
||||||
|
|
||||||
|
generation = await generate_speech(
|
||||||
|
models.GenerationRequest(
|
||||||
|
profile_id=profile.id,
|
||||||
|
text=data.text,
|
||||||
|
language=data.language or "en",
|
||||||
|
engine=data.engine or "qwen",
|
||||||
|
),
|
||||||
|
db,
|
||||||
|
)
|
||||||
|
|
||||||
|
mcp_events.publish(
|
||||||
|
"speak-start",
|
||||||
|
{
|
||||||
|
"generation_id": getattr(generation, "id", None),
|
||||||
|
"profile_name": profile.name,
|
||||||
|
"source": "rest",
|
||||||
|
"client_id": client_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return generation
|
||||||
@@ -134,6 +134,7 @@ async def run_generation(
|
|||||||
db=bg_db,
|
db=bg_db,
|
||||||
error="Generation cancelled",
|
error="Generation cancelled",
|
||||||
)
|
)
|
||||||
|
_notify_speak_end(generation_id, status="cancelled")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
await history.update_generation_status(
|
await history.update_generation_status(
|
||||||
@@ -142,11 +143,28 @@ async def run_generation(
|
|||||||
db=bg_db,
|
db=bg_db,
|
||||||
error=str(e),
|
error=str(e),
|
||||||
)
|
)
|
||||||
|
_notify_speak_end(generation_id, status="failed")
|
||||||
|
else:
|
||||||
|
_notify_speak_end(generation_id, status="completed")
|
||||||
finally:
|
finally:
|
||||||
task_manager.complete_generation(generation_id)
|
task_manager.complete_generation(generation_id)
|
||||||
bg_db.close()
|
bg_db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _notify_speak_end(generation_id: str, *, status: str) -> None:
|
||||||
|
"""Publish a speak-end event; the frontend ignores unknown ids."""
|
||||||
|
try:
|
||||||
|
from ..mcp_server import events as mcp_events
|
||||||
|
|
||||||
|
mcp_events.publish(
|
||||||
|
"speak-end",
|
||||||
|
{"generation_id": generation_id, "status": status},
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Never let event pub/sub break generation completion.
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _save_generate(
|
def _save_generate(
|
||||||
*,
|
*,
|
||||||
generation_id: str,
|
generation_id: str,
|
||||||
|
|||||||
@@ -275,6 +275,27 @@ async def get_profile(
|
|||||||
return _profile_to_response(profile)
|
return _profile_to_response(profile)
|
||||||
|
|
||||||
|
|
||||||
|
def get_profile_orm_by_name_or_id(
|
||||||
|
name_or_id: str,
|
||||||
|
db: Session,
|
||||||
|
) -> DBVoiceProfile | None:
|
||||||
|
"""Resolve a profile from a user-supplied string that may be either id or name.
|
||||||
|
|
||||||
|
Id is tried first (fast path, matches UUIDs). Name fallback is
|
||||||
|
case-insensitive so agents can say "Morgan" regardless of casing.
|
||||||
|
"""
|
||||||
|
if not name_or_id:
|
||||||
|
return None
|
||||||
|
row = db.query(DBVoiceProfile).filter(DBVoiceProfile.id == name_or_id).first()
|
||||||
|
if row is not None:
|
||||||
|
return row
|
||||||
|
return (
|
||||||
|
db.query(DBVoiceProfile)
|
||||||
|
.filter(func.lower(DBVoiceProfile.name) == name_or_id.lower())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def get_profile_samples(
|
async def get_profile_samples(
|
||||||
profile_id: str,
|
profile_id: str,
|
||||||
db: Session,
|
db: Session,
|
||||||
|
|||||||
@@ -46,6 +46,8 @@ tmp_ret = collect_all('espeakng_loader')
|
|||||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||||
tmp_ret = collect_all('en_core_web_sm')
|
tmp_ret = collect_all('en_core_web_sm')
|
||||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||||
|
tmp_ret = collect_all('unidic_lite')
|
||||||
|
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||||
tmp_ret = collect_all('mlx')
|
tmp_ret = collect_all('mlx')
|
||||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||||
tmp_ret = collect_all('mlx_audio')
|
tmp_ret = collect_all('mlx_audio')
|
||||||
|
|||||||
+25
-16
@@ -1,23 +1,30 @@
|
|||||||
---
|
---
|
||||||
title: "Voicebox Documentation"
|
title: "Voicebox Documentation"
|
||||||
description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs."
|
description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine."
|
||||||
---
|
---
|
||||||
|
|
||||||
Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor.
|
Voicebox is the **open-source, local-first AI voice studio** — a free
|
||||||
|
alternative to ElevenLabs and WisprFlow in one app. Clone voices, generate
|
||||||
|
speech across 7 TTS engines, dictate into any app with a global hotkey,
|
||||||
|
compose multi-voice projects, and let any MCP-aware agent speak in a voice
|
||||||
|
you own. Everything runs on your hardware.
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
- **Complete privacy** -- models and voice data stay on your machine
|
- **Dictation** — hold a chord anywhere on your machine, speak, release; the transcript pastes into the focused field
|
||||||
- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro
|
- **Captures tab** — paired audio + transcript archive, retranscribe / refine / play-as-voice
|
||||||
- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice
|
- **Voice personalities** — per-profile compose / rewrite / respond, powered by a local LLM
|
||||||
- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more
|
- **Agents speak back** — any MCP-aware agent can call Voicebox to speak in one of your cloned voices
|
||||||
- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters
|
- **7 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro
|
||||||
- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
- **Cloning and preset voices** — zero-shot cloning or 50+ curated preset voices
|
||||||
- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters
|
- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili
|
||||||
- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives
|
- **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, filters
|
||||||
- **API-first** -- REST API for integrating voice synthesis into your own projects
|
- **Expressive speech** — paralinguistic tags (`[laugh]`, `[sigh]`) and natural-language delivery control
|
||||||
- **Native performance** -- built with Tauri (Rust), not Electron
|
- **Unlimited length** — auto-chunking with crossfade for long scripts
|
||||||
- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker
|
- **Stories editor** — multi-track timeline for conversations, podcasts, narratives
|
||||||
|
- **API-first** — REST + WebSocket API, MCP server for agent integrations
|
||||||
|
- **Complete privacy** — models, audio, transcripts, LLM output never leave your machine
|
||||||
|
- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux (ROCm / CPU), Intel Arc, Docker
|
||||||
|
|
||||||
## Download
|
## Download
|
||||||
|
|
||||||
@@ -32,6 +39,8 @@ Voicebox is a **local-first voice cloning studio** -- a free and open-source alt
|
|||||||
|
|
||||||
## Get Started
|
## Get Started
|
||||||
|
|
||||||
- [Installation](/overview/installation) -- download and install Voicebox
|
- [Installation](/overview/installation) — download and install Voicebox
|
||||||
- [Quick Start](/overview/quick-start) -- get up and running in 5 minutes
|
- [Quick Start](/overview/quick-start) — get up and running in 5 minutes
|
||||||
- [API Reference](/api-reference) -- integrate voice synthesis into your apps
|
- [Dictation](/overview/dictation) — start talking to your computer
|
||||||
|
- [Voice Personalities](/overview/voice-personalities) — compose, rewrite, respond in any profile
|
||||||
|
- [API Reference](/api-reference) — integrate voice synthesis into your apps
|
||||||
|
|||||||
@@ -0,0 +1,194 @@
|
|||||||
|
---
|
||||||
|
title: "Captures"
|
||||||
|
description: "The paired audio + transcript archive — every dictation, recording, and uploaded audio file shows up here, replayable and retranscribable."
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
A **capture** is an audio clip paired with its transcript. The Captures tab
|
||||||
|
is where every dictation, manual recording, and uploaded audio file lands,
|
||||||
|
with the original audio kept alongside the text so you can replay, re-run
|
||||||
|
transcription with a different model, refine the transcript, or send the
|
||||||
|
content somewhere else — including generating it back as speech in any of
|
||||||
|
your voice profiles.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
The Captures tab shipped in **0.5.0**, alongside global dictation and the
|
||||||
|
per-profile personality modes. If you've used earlier versions, note that
|
||||||
|
the Audio tab moved into **Settings → Audio Channels** to make room for
|
||||||
|
this one.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Where captures come from
|
||||||
|
|
||||||
|
| Source | How it shows up | Badge |
|
||||||
|
|---|---|---|
|
||||||
|
| **Dictation** | Triggered by the global hotkey (see [Dictation](/overview/dictation)). Auto-refined by default. | `dictation` |
|
||||||
|
| **In-app recording** | Recorded directly in the Captures tab using the built-in mic. | `recording` |
|
||||||
|
| **File upload** | Any audio file dropped into the Captures tab — `.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`. | `file` |
|
||||||
|
|
||||||
|
All three paths share the same backend pipeline, the same model picker, and
|
||||||
|
the same refinement flags. The source badge is there so you can visually
|
||||||
|
scan a long list.
|
||||||
|
|
||||||
|
## List view
|
||||||
|
|
||||||
|
The main Captures view is a chronological list. Each row shows:
|
||||||
|
|
||||||
|
- The transcript (raw or refined — the refined version wins if present)
|
||||||
|
- Duration + timestamp
|
||||||
|
- Source badge
|
||||||
|
- A play button for the original audio
|
||||||
|
- A meatballs menu with per-row actions
|
||||||
|
|
||||||
|
Filtering and search are a Tier-2 ask — ping if you need them.
|
||||||
|
|
||||||
|
## Detail view
|
||||||
|
|
||||||
|
Clicking into a capture opens the detail view:
|
||||||
|
|
||||||
|
- **Waveform player** for the original audio
|
||||||
|
- **Transcript editor** — click in and edit. Changes save on blur.
|
||||||
|
- **Refined vs. raw toggle** if refinement ran on this capture
|
||||||
|
- **Per-capture action bar** — retranscribe, refine, play as voice, delete
|
||||||
|
- **Settings snapshot** — STT model used, refinement flags at the time
|
||||||
|
this capture was processed, and the voice model if any was played
|
||||||
|
|
||||||
|
## Retranscribe
|
||||||
|
|
||||||
|
Runs the capture's original audio through a different Whisper model without
|
||||||
|
re-uploading or re-refining anything. Useful when:
|
||||||
|
|
||||||
|
- The default model mis-heard something and you want to try a larger model
|
||||||
|
- You used Base for a noisy clip and want to rerun with Turbo
|
||||||
|
- A non-English clip needs an explicit language hint
|
||||||
|
|
||||||
|
**Settings → Captures → Transcription** controls the default model and
|
||||||
|
language lock for new captures. Retranscribe uses those defaults unless you
|
||||||
|
override them per capture.
|
||||||
|
|
||||||
|
## Refine
|
||||||
|
|
||||||
|
Runs the raw transcript through the local LLM to produce a cleaned-up
|
||||||
|
version. The flags on the capture are snapshotted when refinement first
|
||||||
|
runs, so you can re-refine later with different flags without losing the raw
|
||||||
|
transcript:
|
||||||
|
|
||||||
|
| Flag | Effect |
|
||||||
|
|---|---|
|
||||||
|
| **Smart cleanup** | Remove fillers (`um`, `uh`, `like`), tidy punctuation and capitalization. |
|
||||||
|
| **Remove self-corrections** | Keep the final version when the speaker backtracks ("actually, no, on Tuesday"). |
|
||||||
|
| **Preserve technical terms** | Leave identifiers (`handleSubmit`, `npm install`) untouched. |
|
||||||
|
|
||||||
|
See the Refinement section of [Dictation](/overview/dictation#refinement) for
|
||||||
|
how Voicebox strips Whisper loop hallucinations *before* the LLM sees the
|
||||||
|
transcript — a capture can be re-refined any number of times without
|
||||||
|
re-introducing "thanks for watching thanks for watching" echoes.
|
||||||
|
|
||||||
|
The refinement model picker (three bundled Qwen3 sizes) lives in
|
||||||
|
**Settings → Captures → Refinement**.
|
||||||
|
|
||||||
|
## Play as voice
|
||||||
|
|
||||||
|
This is the capability no one else in the dictation category ships: take any
|
||||||
|
capture and play it back as speech in any of your voice profiles. One
|
||||||
|
dropdown over every profile, one click, and the capture's text runs through
|
||||||
|
`/generate` with the selected voice.
|
||||||
|
|
||||||
|
Use cases:
|
||||||
|
|
||||||
|
- Hear your own dictation back in a cloned voice of someone you like
|
||||||
|
- Send a message you dictated as an audio reply in a specific character
|
||||||
|
- Quickly prototype a line for a story without retyping
|
||||||
|
|
||||||
|
Playback uses whatever engine the selected profile is bound to — the same
|
||||||
|
rules as the Generate tab. There's no LLM in this path; the transcript goes
|
||||||
|
through unchanged. If you want the agent-style "transform the content before
|
||||||
|
speaking" flow, that's what the
|
||||||
|
[personality modes](/overview/voice-personalities) do — and the same
|
||||||
|
primitive is exposed to MCP-aware agents via the
|
||||||
|
[MCP Server](/overview/mcp-server) so Claude Code, Cursor, or Cline can speak
|
||||||
|
in one of your voices on their own.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
The default voice for the Captures tab's Play-as action is set in
|
||||||
|
**Settings → Captures → Playback → Default voice**. You can still override
|
||||||
|
it per capture.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Send-to menu
|
||||||
|
|
||||||
|
Each capture has a Send-to menu for moving its content into other parts of
|
||||||
|
Voicebox:
|
||||||
|
|
||||||
|
- **Copy transcript** — to clipboard
|
||||||
|
- **Use as voice sample…** — promote this capture to a sample on a voice
|
||||||
|
profile of your choice. Opens a profile picker (with "+ New voice" for
|
||||||
|
cold starts) and a reference-text confirm dialog, because cloning needs
|
||||||
|
the `reference_text` to match the audio verbatim. Edit as needed and
|
||||||
|
save — the capture stays in the Captures tab untouched; the sample is a
|
||||||
|
copy, not a move.
|
||||||
|
|
||||||
|
## Storage and retention
|
||||||
|
|
||||||
|
**Settings → Captures → Storage** controls how long captures live on disk:
|
||||||
|
|
||||||
|
| Setting | Effect |
|
||||||
|
|---|---|
|
||||||
|
| **Retention: forever** | Never auto-delete. Default. |
|
||||||
|
| **Retention: 90 days / 30 days / 7 days** | Captures older than the window are pruned on app start. |
|
||||||
|
| **Clear all captures** | One-click nuke of every capture and its audio on disk. No undo. |
|
||||||
|
|
||||||
|
The original audio is always kept alongside the transcript — archival is on
|
||||||
|
by default. Every capture's audio file and metadata row can be re-processed
|
||||||
|
(retranscribe, refine, Play-as) as long as the audio file still exists.
|
||||||
|
|
||||||
|
## Short-recording guard
|
||||||
|
|
||||||
|
Audio clips under **300 ms** are short-circuited client-side and never
|
||||||
|
uploaded. This prevents a fumbled chord tap from landing an empty capture.
|
||||||
|
The threshold is tuned to filter accidents without cutting off intentional
|
||||||
|
short dictations.
|
||||||
|
|
||||||
|
## Keyboard shortcuts
|
||||||
|
|
||||||
|
Inside the Captures tab:
|
||||||
|
|
||||||
|
| Keys | Action |
|
||||||
|
|---|---|
|
||||||
|
| `Space` | Play / pause the selected capture |
|
||||||
|
| `↑` / `↓` | Previous / next capture in the list |
|
||||||
|
| `Enter` | Open the selected capture in detail view |
|
||||||
|
| `⌘ / Ctrl` + `C` (in detail view) | Copy the transcript |
|
||||||
|
|
||||||
|
## API surface
|
||||||
|
|
||||||
|
The Captures tab is backed by a small set of REST endpoints:
|
||||||
|
|
||||||
|
| Method | Endpoint | Use |
|
||||||
|
|---|---|---|
|
||||||
|
| `POST` | `/captures` | Upload audio + start the pipeline (STT, optional refinement, archival). |
|
||||||
|
| `GET` | `/captures` | List captures. |
|
||||||
|
| `GET` | `/captures/{id}` | Fetch one capture. |
|
||||||
|
| `POST` | `/captures/{id}/retranscribe` | Rerun STT with a chosen model. |
|
||||||
|
| `POST` | `/captures/{id}/refine` | Rerun refinement with chosen flags. |
|
||||||
|
| `POST` | `/profiles/{id}/samples/from-capture/{capture_id}` | Promote a capture to a voice profile sample. |
|
||||||
|
|
||||||
|
These endpoints are stable and usable from your own scripts — see
|
||||||
|
[Remote Mode](/overview/remote-mode) for running Voicebox as a server the rest
|
||||||
|
of your machine can talk to.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
<Cards>
|
||||||
|
<Card title="Dictation" href="/overview/dictation">
|
||||||
|
The global hotkey flow that feeds most captures.
|
||||||
|
</Card>
|
||||||
|
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||||
|
Per-profile compose / rewrite / respond modes for captures you want to
|
||||||
|
transform, not just transcribe.
|
||||||
|
</Card>
|
||||||
|
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||||
|
Promote a capture into a voice sample on a profile.
|
||||||
|
</Card>
|
||||||
|
</Cards>
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
---
|
||||||
|
title: "Dictation"
|
||||||
|
description: "Hold a key anywhere on your machine, speak, release — the transcript lands in whatever text field you had focused."
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Dictation lets you turn speech into clean text anywhere on your computer. Hold
|
||||||
|
a chord, talk, release — Voicebox transcribes what you said with Whisper,
|
||||||
|
optionally cleans it up with a local LLM, and pastes the result into the text
|
||||||
|
field you had focused when you started.
|
||||||
|
|
||||||
|
Everything happens on your hardware. No cloud, no accounts, no audio leaving
|
||||||
|
the machine.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
Dictation was introduced in **0.5.0** alongside the Captures tab and the
|
||||||
|
per-profile personality modes. It's the "input" half of Voicebox's voice I/O
|
||||||
|
loop — cloning and TTS are still the "output" half.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## The flow
|
||||||
|
|
||||||
|
<Steps>
|
||||||
|
<Step title="Hold the chord">
|
||||||
|
Hold the push-to-talk chord anywhere on your machine. A small pill fades
|
||||||
|
in over your current app.
|
||||||
|
</Step>
|
||||||
|
<Step title="Speak">
|
||||||
|
The pill shows `Recording` with a live waveform and an elapsed-time
|
||||||
|
counter. Speak naturally — you don't have to wait for anything.
|
||||||
|
</Step>
|
||||||
|
<Step title="Release">
|
||||||
|
On release, the pill flips to `Transcribing`, then `Refining` if
|
||||||
|
auto-refine is on, then disappears.
|
||||||
|
</Step>
|
||||||
|
<Step title="Text lands in your app">
|
||||||
|
If auto-paste is enabled and Voicebox has Accessibility permission, the
|
||||||
|
transcript pastes into the text field you had focused when you started
|
||||||
|
talking — not wherever focus drifted while you were speaking.
|
||||||
|
</Step>
|
||||||
|
</Steps>
|
||||||
|
|
||||||
|
Either way, every capture also appears in the **Captures tab** with the
|
||||||
|
original audio and the transcript paired together. See
|
||||||
|
[Captures](/overview/captures) for what you can do with them after the fact.
|
||||||
|
|
||||||
|
## Push-to-talk and toggle modes
|
||||||
|
|
||||||
|
Voicebox ships two chord behaviors out of the box:
|
||||||
|
|
||||||
|
| Mode | Default (macOS) | Default (Windows) | Behavior |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **Push-to-talk** | Right `⌘` + Right `⌥` | Right `Ctrl` + Right `Shift` | Recording stops when you release the chord. |
|
||||||
|
| **Toggle-to-talk** | Push-to-talk + `Space` | Push-to-talk + `Space` | Recording keeps going until you tap the chord again. |
|
||||||
|
|
||||||
|
**Holding PTT and tapping `Space` mid-hold upgrades a hold into a toggled
|
||||||
|
session** without a gap in the audio. This is the single most useful detail of
|
||||||
|
the chord system — short bursts feel fast, long-form narration feels
|
||||||
|
hands-free, and there's no decision up front about which mode you wanted.
|
||||||
|
|
||||||
|
## The on-screen pill
|
||||||
|
|
||||||
|
While you're dictating, a floating pill appears over the current app. It walks
|
||||||
|
through the states of the capture cycle and shows live signals for each:
|
||||||
|
|
||||||
|
| State | What it shows |
|
||||||
|
|---|---|
|
||||||
|
| `Recording` | Live waveform + elapsed time. |
|
||||||
|
| `Transcribing` | Thinking waveform while Whisper runs. |
|
||||||
|
| `Refining` | Same thinking waveform while the LLM cleans up the transcript (only if auto-refine is on). |
|
||||||
|
| Error | Red tint. Click the pill to copy the error to your clipboard. Auto-dismisses. |
|
||||||
|
|
||||||
|
The pill is transparent, always-on-top, and pre-created hidden at app start —
|
||||||
|
so it appears instantly when you hit the chord, with no window flash.
|
||||||
|
|
||||||
|
## Customizing the chord
|
||||||
|
|
||||||
|
Open **Settings → Captures → Dictation** to change either chord.
|
||||||
|
|
||||||
|
- **Left vs right modifier badges.** When you hold keys into the chord
|
||||||
|
picker, Voicebox records whether each modifier is the left or right variant.
|
||||||
|
That means you can bind to just the right `⌥` while leaving the left `⌥`
|
||||||
|
alone — useful if you want dictation on one hand and keep your
|
||||||
|
other-hand shortcuts intact.
|
||||||
|
- **Chord defaults are picked to stay out of your way.** On macOS, the
|
||||||
|
defaults deliberately avoid left-hand `Cmd+Option` chords so
|
||||||
|
`Cmd+Option+I` (devtools), `Cmd+Option+Esc` (force quit), and
|
||||||
|
`Cmd+Option+Space` (Spotlight) all remain yours. On Windows, the defaults
|
||||||
|
route around AltGr collisions on German / French / Spanish layouts where
|
||||||
|
`Ctrl+Alt` synthesizes AltGr.
|
||||||
|
- **Live reload.** Changing a chord in Settings takes effect immediately —
|
||||||
|
no restart, no tab reload.
|
||||||
|
|
||||||
|
## Auto-paste into the focused app
|
||||||
|
|
||||||
|
Once transcription finishes, Voicebox can synthesize a native paste into
|
||||||
|
whatever text field had focus when you started the chord. Your clipboard is
|
||||||
|
saved before and restored after, so nothing you had copied goes missing.
|
||||||
|
|
||||||
|
| Platform | Mechanism |
|
||||||
|
|---|---|
|
||||||
|
| macOS | `CGEventPost` at the HID tap with a full `⌘V` key sequence, preceded by reactivating the original app via `NSRunningApplication`. |
|
||||||
|
| Windows | `SendInput` with correct scan codes, plus a `SetForegroundWindow` + `AttachThreadInput` handshake to defeat foreground-lock when pasting into a window that wasn't frontmost at chord-start. |
|
||||||
|
|
||||||
|
**Focus is snapshotted at chord-start.** The paste targets the original field
|
||||||
|
even if focus drifts during transcribe / refine — that's the "pastes where you
|
||||||
|
were talking *from*, not where you're looking *now*" behavior.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
Auto-paste is optional. If Accessibility permission isn't granted (macOS),
|
||||||
|
or you prefer to keep synthetic input off, dictation still runs — transcripts
|
||||||
|
land in the Captures tab and you can copy them manually. The setting lives
|
||||||
|
inline next to the Accessibility prompt in Settings → Captures → Dictation,
|
||||||
|
not as a global banner.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Refinement
|
||||||
|
|
||||||
|
If auto-refine is on, a local LLM cleans up the raw Whisper transcript
|
||||||
|
before it's pasted. The goal is to remove verbal clutter without rewriting
|
||||||
|
what you actually said.
|
||||||
|
|
||||||
|
What refinement typically fixes:
|
||||||
|
|
||||||
|
- Filler words (`um`, `uh`, `like` used as pauses, `you know`)
|
||||||
|
- Self-corrections — the LLM keeps the final version and drops earlier
|
||||||
|
attempts (`could you uh run the migration real quick, and then, yeah,
|
||||||
|
check the logs` → `Could you run the migration, then check the logs?`)
|
||||||
|
- Basic punctuation and capitalization
|
||||||
|
- Whisper loop hallucinations — Voicebox strips repeated tokens (six or
|
||||||
|
more identical tokens in a row, case-insensitive) *before* the LLM
|
||||||
|
sees the transcript, so a small refinement model can't echo them back
|
||||||
|
|
||||||
|
What refinement deliberately preserves:
|
||||||
|
|
||||||
|
- Technical terms and code identifiers (`npm install`, `handleSubmit`)
|
||||||
|
- Legitimate repetition (`no, no, no, no, no` has fewer than six identical
|
||||||
|
tokens, so it survives)
|
||||||
|
- Your intent — refinement is cleanup, not rewriting
|
||||||
|
|
||||||
|
Flags are snapshotted per capture, so you can re-refine the same raw
|
||||||
|
transcript later with different flags without losing the original. The
|
||||||
|
refinement model picker (**Settings → Captures → Refinement**) offers three
|
||||||
|
bundled Qwen3 sizes:
|
||||||
|
|
||||||
|
| Model | Size | Best for |
|
||||||
|
|---|---|---|
|
||||||
|
| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual dictation. |
|
||||||
|
| Qwen3 1.7B | ~1.1 GB | Sweet spot when transcripts contain code identifiers. |
|
||||||
|
| Qwen3 4B | ~2.5 GB | Full quality, slowest. |
|
||||||
|
|
||||||
|
This is the same local LLM used by the per-profile personality modes — one
|
||||||
|
LLM in the app, not two. See [Voice Personalities](/overview/voice-personalities).
|
||||||
|
|
||||||
|
## Platform notes
|
||||||
|
|
||||||
|
### macOS
|
||||||
|
|
||||||
|
- **Accessibility permission** is required for auto-paste. The prompt lives
|
||||||
|
inline next to the toggle in **Settings → Captures → Dictation**, with a
|
||||||
|
deep link to **System Settings → Privacy & Security → Accessibility**.
|
||||||
|
- **TSM crash mitigation.** The global hotkey listener runs on a background
|
||||||
|
thread with `set_is_main_thread(false)` to sidestep a known
|
||||||
|
macOS 14+ crash in the `rdev` library. If you hit an unexpected dictation
|
||||||
|
failure on macOS, check the logs for TSM-related messages.
|
||||||
|
|
||||||
|
### Windows
|
||||||
|
|
||||||
|
- **UAC / UIPI caveat.** Synthetic paste into an *elevated* window from a
|
||||||
|
non-elevated Voicebox is blocked by Windows itself. Run Voicebox elevated
|
||||||
|
if you regularly dictate into elevated apps (e.g. an elevated terminal or
|
||||||
|
Task Manager).
|
||||||
|
- **Right-hand default chord** (`Ctrl+Shift`) avoids AltGr collisions on
|
||||||
|
keyboard layouts where `Ctrl+Alt` is the compose key (German, French,
|
||||||
|
Spanish, some others).
|
||||||
|
|
||||||
|
### Linux
|
||||||
|
|
||||||
|
- **Not yet in this release.** The Rust shim ships the macOS and Windows
|
||||||
|
paths in 0.5.0. Linux `uinput` / AT-SPI support and the Wayland paste
|
||||||
|
story are tracked in `docs/plans/VOICE_IO.md`.
|
||||||
|
|
||||||
|
## When auto-paste skips itself
|
||||||
|
|
||||||
|
A few cases where Voicebox deliberately does *not* synthesize a paste:
|
||||||
|
|
||||||
|
- **Focus was inside Voicebox** when the chord started. The transcript goes
|
||||||
|
to the Captures tab so a dictation-into-Voicebox round-trip doesn't
|
||||||
|
accidentally paste into the generate box.
|
||||||
|
- **No text focus detected.** The transcript still lands in the Captures
|
||||||
|
tab; copy it from there with one click.
|
||||||
|
- **Accessibility permission not granted** on macOS. Same — Captures tab
|
||||||
|
only.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
<Cards>
|
||||||
|
<Card title="Captures" href="/overview/captures">
|
||||||
|
The paired audio + transcript archive every dictation lands in.
|
||||||
|
</Card>
|
||||||
|
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||||
|
The same local LLM doubles as per-profile compose / rewrite / respond.
|
||||||
|
</Card>
|
||||||
|
<Card title="Transcription" href="/developer/transcription">
|
||||||
|
Developer-level details on Whisper, Whisper Turbo, and the STT backend.
|
||||||
|
</Card>
|
||||||
|
</Cards>
|
||||||
@@ -1,23 +1,48 @@
|
|||||||
---
|
---
|
||||||
title: "Introduction"
|
title: "Introduction"
|
||||||
description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs."
|
description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine."
|
||||||
---
|
---
|
||||||
|
|
||||||
## What is Voicebox?
|
## What is Voicebox?
|
||||||
|
|
||||||
Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio or pick from 50+ preset voices, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor.
|
Voicebox is the **open-source, local-first AI voice studio**. It closes the
|
||||||
|
voice I/O loop in both directions on one machine, with no cloud and no
|
||||||
|
accounts:
|
||||||
|
|
||||||
- **Complete privacy** -- models and voice data stay on your machine
|
- **Humans talk** — hold a chord anywhere on your machine and your
|
||||||
- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro
|
dictation lands as clean text in whatever text field you had focused
|
||||||
- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or curated preset voices via Kokoro (50 voices) and Qwen CustomVoice (9 voices)
|
- **Agents talk back** — any MCP-aware agent can call Voicebox to speak in
|
||||||
- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more
|
one of your cloned voices
|
||||||
- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters
|
- **Voices speak for themselves** — voice profiles can carry a personality
|
||||||
- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
that compose, rewrite, or respond to text before it's spoken
|
||||||
- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters
|
|
||||||
- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives
|
It's the free, local alternative to both ElevenLabs (voice cloning and TTS)
|
||||||
- **API-first** -- REST API for integrating voice synthesis into your own projects
|
and WisprFlow (voice dictation for agents and power users) — covering both
|
||||||
- **Native performance** -- built with Tauri (Rust), not Electron
|
sides of the same loop in one app, with a single model directory and LLM
|
||||||
- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker
|
shared between input and output.
|
||||||
|
|
||||||
|
## What's in the app
|
||||||
|
|
||||||
|
- **Dictation** — global hotkey, push-to-talk and toggle modes, auto-paste
|
||||||
|
into the focused field on macOS and Windows (see [Dictation](/overview/dictation))
|
||||||
|
- **Captures tab** — paired audio + transcript archive, retranscribe,
|
||||||
|
refine, play-as-voice, promote-to-sample (see [Captures](/overview/captures))
|
||||||
|
- **Voice cloning** — 5 cloning engines covering 23 languages. Zero-shot
|
||||||
|
cloning from a reference sample (see [Voice Cloning](/overview/voice-cloning))
|
||||||
|
- **Preset voices** — 50+ curated voices via Kokoro and Qwen CustomVoice
|
||||||
|
for when you don't want to clone (see [Preset Voices](/overview/preset-voices))
|
||||||
|
- **Voice personalities** — optional free-form personality on any profile
|
||||||
|
plus compose / rewrite / respond modes powered by a local LLM (see
|
||||||
|
[Voice Personalities](/overview/voice-personalities))
|
||||||
|
- **Post-processing effects** — pitch shift, reverb, delay, chorus,
|
||||||
|
compression, filters (Spotify's Pedalboard)
|
||||||
|
- **Expressive speech** — paralinguistic tags like `[laugh]` and `[sigh]`
|
||||||
|
via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
||||||
|
- **Unlimited length** — auto-chunking with crossfade for long scripts
|
||||||
|
- **Stories editor** — multi-track timeline for conversations and podcasts
|
||||||
|
- **API-first** — REST + WebSocket API; MCP server for agent integrations
|
||||||
|
- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux
|
||||||
|
(ROCm / CPU), Intel Arc, Docker
|
||||||
|
|
||||||
## TTS Engines
|
## TTS Engines
|
||||||
|
|
||||||
@@ -30,9 +55,21 @@ Seven engines with different strengths, switchable per-generation:
|
|||||||
| **LuxTTS** | Cloned | English | Lightweight (~1GB VRAM), 48kHz output, 150x realtime on CPU |
|
| **LuxTTS** | Cloned | English | Lightweight (~1GB VRAM), 48kHz output, 150x realtime on CPU |
|
||||||
| **Chatterbox Multilingual** | Cloned | 23 | Broadest language coverage |
|
| **Chatterbox Multilingual** | Cloned | 23 | Broadest language coverage |
|
||||||
| **Chatterbox Turbo** | Cloned | English | Fast 350M model with paralinguistic emotion/sound tags |
|
| **Chatterbox Turbo** | Cloned | English | Fast 350M model with paralinguistic emotion/sound tags |
|
||||||
| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model -- 700s+ coherent audio |
|
| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model — 700s+ coherent audio |
|
||||||
| **Kokoro** | Preset (50 voices) | 9 | 82M parameters, CPU realtime, lowest VRAM of any engine |
|
| **Kokoro** | Preset (50 voices) | 9 | 82M parameters, CPU realtime, lowest VRAM of any engine |
|
||||||
|
|
||||||
|
## STT and local LLM
|
||||||
|
|
||||||
|
Voicebox also runs a full speech recognition and local LLM stack, shared
|
||||||
|
between dictation, the Captures tab, and per-profile personality modes:
|
||||||
|
|
||||||
|
| Layer | Models |
|
||||||
|
|---|---|
|
||||||
|
| **STT** | Whisper Base / Small / Medium / Large / Turbo (PyTorch or MLX) |
|
||||||
|
| **LLM** | Qwen3 0.6B / 1.7B / 4B (refinement + per-profile compose / rewrite / respond) |
|
||||||
|
|
||||||
|
No cloud fallback, no bring-your-own-API-key. Local is the product.
|
||||||
|
|
||||||
## GPU Support
|
## GPU Support
|
||||||
|
|
||||||
| Platform | Backend | Notes |
|
| Platform | Backend | Notes |
|
||||||
@@ -46,11 +83,13 @@ Seven engines with different strengths, switchable per-generation:
|
|||||||
|
|
||||||
## Use Cases
|
## Use Cases
|
||||||
|
|
||||||
- **Game development** -- generate dynamic dialogue for characters
|
- **Dictation for humans and agents** — speak instead of type, in any app
|
||||||
- **Content creation** -- produce podcasts and video voiceovers
|
- **Agent voice output** — any MCP-aware agent can speak in a cloned voice
|
||||||
- **Accessibility** -- build text-to-speech tools for users who need them
|
- **Game development** — generate dynamic dialogue for characters
|
||||||
- **Voice assistants** -- create custom voice interfaces
|
- **Content creation** — podcasts, video voiceovers, audiobooks
|
||||||
- **Production pipelines** -- automate voiceover workflows via the REST API
|
- **Accessibility** — speech-to-text for any field, TTS with a voice you own
|
||||||
|
- **Voice assistants** — custom voice interfaces without a cloud bill
|
||||||
|
- **Production pipelines** — automate voice workflows via the REST API
|
||||||
|
|
||||||
## Tech Stack
|
## Tech Stack
|
||||||
|
|
||||||
@@ -61,8 +100,9 @@ Seven engines with different strengths, switchable per-generation:
|
|||||||
| State | Zustand, React Query |
|
| State | Zustand, React Query |
|
||||||
| Backend | FastAPI (Python) |
|
| Backend | FastAPI (Python) |
|
||||||
| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro |
|
| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro |
|
||||||
|
| STT | Whisper / Whisper Turbo (PyTorch or MLX) |
|
||||||
|
| Local LLM | Qwen3 0.6B / 1.7B / 4B (MLX or PyTorch) |
|
||||||
| Effects | Pedalboard (Spotify) |
|
| Effects | Pedalboard (Spotify) |
|
||||||
| Transcription | Whisper / Whisper Turbo (PyTorch or MLX) |
|
|
||||||
| Inference | MLX (Apple Silicon) / PyTorch (CUDA/ROCm/XPU/CPU) |
|
| Inference | MLX (Apple Silicon) / PyTorch (CUDA/ROCm/XPU/CPU) |
|
||||||
| Database | SQLite |
|
| Database | SQLite |
|
||||||
| Audio | WaveSurfer.js, librosa |
|
| Audio | WaveSurfer.js, librosa |
|
||||||
|
|||||||
@@ -0,0 +1,300 @@
|
|||||||
|
---
|
||||||
|
title: "MCP Server"
|
||||||
|
description: "Let Claude Code, Cursor, Cline, or any MCP-aware agent speak in one of your cloned voices — locally, with no cloud."
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Voicebox ships a built-in **Model Context Protocol** server so local AI
|
||||||
|
agents can call your Voicebox install directly: speak text in a voice
|
||||||
|
profile, transcribe audio, and list captures or profiles. The server runs
|
||||||
|
inside the same process as the rest of Voicebox and is mounted at `/mcp`
|
||||||
|
over Streamable HTTP.
|
||||||
|
|
||||||
|
Agent asks to speak → Voicebox plays audio on your speakers → an on-screen
|
||||||
|
pill surfaces the voice name for the whole duration so you always see what's
|
||||||
|
coming out of your machine.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
MCP shipped in **0.5.0** alongside [Dictation](/overview/dictation) and
|
||||||
|
[Voice Personalities](/overview/voice-personalities). The design goal is
|
||||||
|
"local voice layer for every agent on your machine" — the same app that
|
||||||
|
captures your voice can generate a response in any voice profile you've
|
||||||
|
cloned.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Quick install
|
||||||
|
|
||||||
|
### Claude Code
|
||||||
|
|
||||||
|
```
|
||||||
|
claude mcp add voicebox \
|
||||||
|
--transport http \
|
||||||
|
--url http://127.0.0.1:17493/mcp \
|
||||||
|
--header "X-Voicebox-Client-Id: claude-code"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cursor / Windsurf / VS Code MCP / any HTTP MCP client
|
||||||
|
|
||||||
|
Drop this into the client's MCP config (usually `.mcp.json` or a Settings UI):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"url": "http://127.0.0.1:17493/mcp",
|
||||||
|
"headers": { "X-Voicebox-Client-Id": "cursor" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Change `cursor` to whatever name you want the binding to show up as in
|
||||||
|
Voicebox → Settings → MCP. The value is just an identifier for the
|
||||||
|
per-client voice binding — not a secret, not a credential.
|
||||||
|
|
||||||
|
### Clients that only speak stdio
|
||||||
|
|
||||||
|
A stdio shim binary `voicebox-mcp` is bundled with the desktop app. Point
|
||||||
|
the client at that binary's absolute path:
|
||||||
|
|
||||||
|
<Tabs items={["macOS", "Windows", "Linux"]}>
|
||||||
|
<Tab value="macOS">
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||||
|
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</Tab>
|
||||||
|
<Tab value="Windows">
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"command": "C:\\Program Files\\Voicebox\\voicebox-mcp.exe",
|
||||||
|
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</Tab>
|
||||||
|
<Tab value="Linux">
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mcpServers": {
|
||||||
|
"voicebox": {
|
||||||
|
"command": "/opt/voicebox/voicebox-mcp",
|
||||||
|
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</Tab>
|
||||||
|
</Tabs>
|
||||||
|
|
||||||
|
The shim waits up to 30 seconds for the Voicebox backend to come up, then
|
||||||
|
proxies JSON-RPC from stdio over Streamable HTTP. Voicebox must be running
|
||||||
|
for the shim to connect.
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| Tool | Use |
|
||||||
|
|---|---|
|
||||||
|
| `voicebox.speak` | Speak text in a voice profile. Returns a `generation_id` to poll. |
|
||||||
|
| `voicebox.transcribe` | Whisper transcription of base64 audio or an absolute local path. |
|
||||||
|
| `voicebox.list_captures` | Recent captures with transcripts, paginated. |
|
||||||
|
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||||
|
|
||||||
|
### `voicebox.speak`
|
||||||
|
|
||||||
|
```ts
|
||||||
|
voicebox.speak({
|
||||||
|
text: "Deploy complete.",
|
||||||
|
profile?: "Morgan", // name or id; falls back to per-client binding, then default
|
||||||
|
engine?: "qwen", // qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro
|
||||||
|
intent?: "respond", // respond | rewrite | compose — only if the profile has a personality
|
||||||
|
language?: "en",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"generation_id": "…",
|
||||||
|
"status": "generating",
|
||||||
|
"profile": "Morgan",
|
||||||
|
"source": "mcp",
|
||||||
|
"poll_url": "/generate/<id>/status"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Plain TTS** — omit `intent`. Text is spoken as-is.
|
||||||
|
- **Persona mode** — pass `intent` and the profile must have a personality set.
|
||||||
|
The LLM transforms the text (respond to it, rewrite it in character, or
|
||||||
|
compose a fresh utterance) before TTS. See [Voice Personalities](/overview/voice-personalities).
|
||||||
|
|
||||||
|
### `voicebox.transcribe`
|
||||||
|
|
||||||
|
```ts
|
||||||
|
voicebox.transcribe({
|
||||||
|
audio_base64?: "<base64>", // exactly one of these two
|
||||||
|
audio_path?: "/absolute/path/to/file.wav",
|
||||||
|
language?: "en",
|
||||||
|
model?: "turbo", // base | small | medium | large | turbo
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Returns `{ text, duration, language, model }`. 200 MB ceiling on either path.
|
||||||
|
|
||||||
|
### `voicebox.list_captures`
|
||||||
|
|
||||||
|
`{ limit?: 20, offset?: 0 }` → `{ captures: [...], total }`. `limit` is
|
||||||
|
clamped to `1..=200`.
|
||||||
|
|
||||||
|
### `voicebox.list_profiles`
|
||||||
|
|
||||||
|
No args → `{ profiles: [{ id, name, voice_type, language, has_personality }] }`.
|
||||||
|
|
||||||
|
## Voice resolution
|
||||||
|
|
||||||
|
Every call to `voicebox.speak` (and `POST /speak`) resolves the voice profile
|
||||||
|
in this order:
|
||||||
|
|
||||||
|
<Steps>
|
||||||
|
<Step title="Explicit `profile` arg">
|
||||||
|
Passed as a name (case-insensitive) or id. If the name/id doesn't match,
|
||||||
|
the call errors — the server doesn't silently fall back.
|
||||||
|
</Step>
|
||||||
|
<Step title="Per-client binding">
|
||||||
|
Looked up by the `X-Voicebox-Client-Id` header. Managed in
|
||||||
|
**Voicebox → Settings → MCP**. Lets you pin Claude Code to Morgan,
|
||||||
|
Cursor to Scarlett, etc.
|
||||||
|
</Step>
|
||||||
|
<Step title="Global default">
|
||||||
|
`capture_settings.default_playback_voice_id` — same default voice the
|
||||||
|
Captures tab's "Play as voice" action uses.
|
||||||
|
</Step>
|
||||||
|
</Steps>
|
||||||
|
|
||||||
|
If none of the three produce a profile the tool returns a helpful error
|
||||||
|
pointing at Settings.
|
||||||
|
|
||||||
|
## Per-client bindings
|
||||||
|
|
||||||
|
Voicebox → Settings → MCP shows one row per `client_id` Voicebox has heard
|
||||||
|
from, plus the config snippets you can copy into each agent. Each row
|
||||||
|
carries:
|
||||||
|
|
||||||
|
| Field | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `label` | Display name in the Settings UI (e.g. "Claude Code"). |
|
||||||
|
| `profile_id` | The voice this client uses when `profile` isn't passed. |
|
||||||
|
| `default_engine` | Override the TTS engine for this client. |
|
||||||
|
| `default_intent` | Default persona mode (`respond` / `rewrite` / `compose`). |
|
||||||
|
| `last_seen_at` | Last time the server saw a request from this client. |
|
||||||
|
|
||||||
|
`last_seen_at` is stamped automatically by middleware on every `/mcp/*`
|
||||||
|
request — useful when you're not sure whether your config took.
|
||||||
|
|
||||||
|
## The speaking pill
|
||||||
|
|
||||||
|
Every agent-initiated speak surfaces the floating pill the same way
|
||||||
|
[Dictation](/overview/dictation) does, in a new `Speaking` state showing the
|
||||||
|
profile name and an elapsed timer. The pill is intentionally unmissable —
|
||||||
|
silent background TTS is a trust hazard, so Voicebox always shows what's
|
||||||
|
being spoken and in what voice.
|
||||||
|
|
||||||
|
Behind the scenes, the backend broadcasts `speak-start` and `speak-end`
|
||||||
|
events on `GET /events/speak`, which `DictateWindow` subscribes to via SSE.
|
||||||
|
The pill overrides the capture session when both would render — you can't
|
||||||
|
hear two pills at once.
|
||||||
|
|
||||||
|
## Non-MCP REST surface
|
||||||
|
|
||||||
|
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||||
|
don't speak MCP — shell scripts, ACP, A2A, GitHub Actions, whatever.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://127.0.0.1:17493/speak \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-H 'X-Voicebox-Client-Id: ci' \
|
||||||
|
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Body fields match the MCP tool: `text`, optional `profile`, `engine`,
|
||||||
|
`intent`, `language`. Returns a `GenerationResponse` — the same shape as
|
||||||
|
`POST /generate`.
|
||||||
|
|
||||||
|
## Debugging
|
||||||
|
|
||||||
|
Use the MCP Inspector to poke tools directly without plumbing through an
|
||||||
|
agent:
|
||||||
|
|
||||||
|
```
|
||||||
|
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
Start with `voicebox.list_profiles` to confirm wiring, then
|
||||||
|
`voicebox.speak` for end-to-end — you should hear audio and see the
|
||||||
|
generation land in the Captures tab.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
If an agent can't reach the server, the first thing to check is that
|
||||||
|
Voicebox is running — the backend only listens while the desktop app is
|
||||||
|
open. The stdio shim surfaces this as a JSON-RPC error on the client
|
||||||
|
side after its 30-second health-wait window elapses.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
- **Localhost only.** The server binds to `127.0.0.1`. If you ever point
|
||||||
|
Voicebox at a non-loopback interface (e.g. remote-mode over a trusted
|
||||||
|
network), add a bearer token — it's on the roadmap but not in 0.5.0.
|
||||||
|
- **No auth today.** Any process that can connect to your loopback can
|
||||||
|
call MCP. That's the same trust boundary as the rest of Voicebox's REST
|
||||||
|
API and is appropriate for a single-user local tool.
|
||||||
|
- **`audio_path` reads are unrestricted** against the same trust
|
||||||
|
boundary. If you're scripting against a shared host, prefer
|
||||||
|
`audio_base64` so you don't have to think about path sandboxing.
|
||||||
|
- **Voice cloning consent applies.** See [Voice Cloning](/overview/voice-cloning#limitations)
|
||||||
|
— an agent being able to call `voicebox.speak` in someone's voice
|
||||||
|
doesn't change the ethics of whose voices you clone.
|
||||||
|
|
||||||
|
## Implementation notes
|
||||||
|
|
||||||
|
- **Transport:** Streamable HTTP (Nov-2025 MCP spec, post-SSE). Claude
|
||||||
|
Code, Cursor, Windsurf, and VS Code MCP extensions all support it.
|
||||||
|
- **Package naming:** the backend package is `backend/mcp_server/`, not
|
||||||
|
`mcp`, to avoid shadowing the PyPI `mcp` package FastMCP imports
|
||||||
|
internally.
|
||||||
|
- **Dependencies:** `fastmcp>=3.0,<4.0`, `sse-starlette>=2.0`.
|
||||||
|
- **Lifespan:** mounting FastMCP requires the `lifespan=` kwarg on
|
||||||
|
`FastAPI()` — the startup/shutdown event decorators are incompatible
|
||||||
|
with FastMCP's Streamable HTTP session manager. The Voicebox app.py
|
||||||
|
composes both into one async context manager.
|
||||||
|
|
||||||
|
For the full developer-facing tour of the code layout, see
|
||||||
|
`backend/mcp_server/README.md` in the repo.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
<Cards>
|
||||||
|
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||||
|
Persona mode (`intent=respond/rewrite/compose`) for agents that should
|
||||||
|
transform text in-character before speaking.
|
||||||
|
</Card>
|
||||||
|
<Card title="Dictation" href="/overview/dictation">
|
||||||
|
The pill that surfaces agent speech is the same one that surfaces
|
||||||
|
your dictations — one mental model for both directions of the loop.
|
||||||
|
</Card>
|
||||||
|
<Card title="Captures" href="/overview/captures">
|
||||||
|
Every agent-initiated speak lands in the Captures tab with its
|
||||||
|
generated audio — replay, download, repurpose.
|
||||||
|
</Card>
|
||||||
|
</Cards>
|
||||||
@@ -7,8 +7,12 @@
|
|||||||
"docker",
|
"docker",
|
||||||
"quick-start",
|
"quick-start",
|
||||||
"gpu-acceleration",
|
"gpu-acceleration",
|
||||||
|
"dictation",
|
||||||
|
"captures",
|
||||||
"voice-cloning",
|
"voice-cloning",
|
||||||
"preset-voices",
|
"preset-voices",
|
||||||
|
"voice-personalities",
|
||||||
|
"mcp-server",
|
||||||
"stories-editor",
|
"stories-editor",
|
||||||
"recording-transcription",
|
"recording-transcription",
|
||||||
"generation-history",
|
"generation-history",
|
||||||
|
|||||||
@@ -1,64 +1,106 @@
|
|||||||
---
|
---
|
||||||
title: "Recording & Transcription"
|
title: "Recording & Transcription"
|
||||||
description: "Record audio and transcribe speech with Whisper"
|
description: "A map of the three places you can record and transcribe audio in Voicebox — dictation, captures, and voice-profile samples."
|
||||||
---
|
---
|
||||||
|
|
||||||
## Recording
|
## Overview
|
||||||
|
|
||||||
Voicebox includes built-in recording capabilities for creating voice samples and capturing audio.
|
Voicebox records and transcribes audio in three different contexts, each
|
||||||
|
feeding a different surface in the app. This page is a map; follow the links
|
||||||
|
for the detail.
|
||||||
|
|
||||||
### Features
|
| Goal | Where | Docs |
|
||||||
|
|---|---|---|
|
||||||
|
| Speak and have your words land in another app | Global hotkey → Captures tab + auto-paste | [Dictation](/overview/dictation) |
|
||||||
|
| Record a thought, a meeting, or a voice memo inside Voicebox | Captures tab | [Captures](/overview/captures) |
|
||||||
|
| Record a clip to clone a voice from | Voices tab → profile samples | [Creating Voice Profiles](/overview/creating-voice-profiles) |
|
||||||
|
|
||||||
- **Microphone input** - Record from any audio input device
|
All three paths share the same STT backend — it's the surrounding workflow
|
||||||
- **System audio capture** - Record desktop audio (macOS/Windows)
|
that differs.
|
||||||
- **Waveform visualization** - See audio levels in real-time
|
|
||||||
- **Multiple formats** - Export as WAV, MP3, or M4A
|
|
||||||
|
|
||||||
### How to Record
|
## Dictation
|
||||||
|
|
||||||
<Steps>
|
The 0.5.0 headline feature. Hold a chord anywhere on your machine, speak,
|
||||||
<Step title="Select Input">
|
release. The transcript lands in whatever text field you had focused,
|
||||||
Choose your microphone or system audio
|
cleaned up by a local LLM if auto-refine is on. Captures accumulate in the
|
||||||
</Step>
|
Captures tab for later replay or re-transcription.
|
||||||
<Step title="Start Recording">
|
|
||||||
Click the record button and speak clearly
|
|
||||||
</Step>
|
|
||||||
<Step title="Stop & Save">
|
|
||||||
Click stop when finished
|
|
||||||
</Step>
|
|
||||||
<Step title="Use or Export">
|
|
||||||
Use as voice sample or export to file
|
|
||||||
</Step>
|
|
||||||
</Steps>
|
|
||||||
|
|
||||||
## Transcription
|
Covered end-to-end in [Dictation](/overview/dictation).
|
||||||
|
|
||||||
Automatic speech-to-text powered by OpenAI's Whisper model.
|
## Captures tab
|
||||||
|
|
||||||
### Features
|
When you don't need to paste into another app — you just want a clean
|
||||||
|
transcript of some audio — the Captures tab is the home. Record in-app,
|
||||||
|
drop in a file (`.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`), or dig
|
||||||
|
through dictations that already landed there. Every capture keeps its
|
||||||
|
original audio, can be retranscribed with a different model, and can be
|
||||||
|
played back through any voice profile you have.
|
||||||
|
|
||||||
- **High accuracy** - Industry-leading speech recognition
|
Covered in [Captures](/overview/captures).
|
||||||
- **Multiple languages** - Supports 50+ languages
|
|
||||||
- **Automatic detection** - Language auto-detection
|
|
||||||
- **Timestamps** - Word-level timing information
|
|
||||||
|
|
||||||
### How to Transcribe
|
## Voice profile samples
|
||||||
|
|
||||||
<Steps>
|
A separate flow, in the Voices tab. When you're creating a profile from an
|
||||||
<Step title="Select Audio">
|
audio clip, the sample is what the cloning engine actually learns from —
|
||||||
Choose a recording or upload an audio file
|
the `reference_text` on a sample must match the audio *verbatim*, which is
|
||||||
</Step>
|
why samples are a different data model from captures.
|
||||||
<Step title="Choose Language">
|
|
||||||
Select language or use auto-detect
|
You can promote a capture to a sample from the Captures tab's Send-to menu
|
||||||
</Step>
|
("Use as voice sample…"), which opens a reference-text confirm dialog so
|
||||||
<Step title="Transcribe">
|
you can correct the last ~10% of transcript accuracy before saving.
|
||||||
Click transcribe and wait for processing
|
|
||||||
</Step>
|
Covered in [Creating Voice Profiles](/overview/creating-voice-profiles).
|
||||||
<Step title="Review & Export">
|
|
||||||
Review text and export as needed
|
## Transcription models
|
||||||
</Step>
|
|
||||||
</Steps>
|
All three paths share the same Whisper models. Pick a default in
|
||||||
|
**Settings → Captures → Transcription**; override per capture if you need
|
||||||
|
to.
|
||||||
|
|
||||||
|
| Model | Size | When to pick it |
|
||||||
|
|---|---|---|
|
||||||
|
| Whisper Base | ~300 MB | Fast. Default. Good for clean speech. |
|
||||||
|
| Whisper Small | ~500 MB | Better quality, still fast. |
|
||||||
|
| Whisper Medium | ~1.5 GB | High quality. |
|
||||||
|
| Whisper Large | ~3 GB | Best quality, slow on CPU. |
|
||||||
|
| Whisper Turbo | ~1.5 GB | Large-tier quality, ~5× faster than Large. |
|
||||||
|
|
||||||
|
On Apple Silicon the model runs through **MLX-Whisper** (~8× faster than
|
||||||
|
PyTorch). Everywhere else it runs through PyTorch `transformers`. The
|
||||||
|
backend picks the right one — you don't configure it.
|
||||||
|
|
||||||
<Callout type="info">
|
<Callout type="info">
|
||||||
Transcription is useful for creating voice samples from existing audio or generating subtitles.
|
For noisy clips, prefer **Turbo** or **Large**. Base can hallucinate on
|
||||||
|
hard inputs — most famously the "thanks for watching" loop. Voicebox
|
||||||
|
strips those loops deterministically before LLM refinement runs, so a
|
||||||
|
capture can be cleanly re-refined even if the raw transcript has them.
|
||||||
</Callout>
|
</Callout>
|
||||||
|
|
||||||
|
## Language
|
||||||
|
|
||||||
|
You can pass a language hint for short clips (under ~5 seconds) where
|
||||||
|
Whisper's auto-detect is unreliable. Set a default language lock in
|
||||||
|
**Settings → Captures → Transcription → Language**, or override per capture.
|
||||||
|
|
||||||
|
## Transcription API
|
||||||
|
|
||||||
|
Developer-level detail on the STT backend, model loading, preprocessing, and
|
||||||
|
the `/transcribe` endpoint lives in the
|
||||||
|
[Transcription developer guide](/developer/transcription). The Captures
|
||||||
|
pipeline also exposes `/captures` as a higher-level endpoint that wraps
|
||||||
|
STT + archival + optional refinement in one call — see
|
||||||
|
[Captures](/overview/captures#api-surface).
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
<Cards>
|
||||||
|
<Card title="Dictation" href="/overview/dictation">
|
||||||
|
Hold a chord anywhere on your machine, speak, release.
|
||||||
|
</Card>
|
||||||
|
<Card title="Captures" href="/overview/captures">
|
||||||
|
The paired audio + transcript archive.
|
||||||
|
</Card>
|
||||||
|
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||||
|
Record or upload samples for voice cloning.
|
||||||
|
</Card>
|
||||||
|
</Cards>
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
---
|
||||||
|
title: "Voice Personalities"
|
||||||
|
description: "Attach a personality to a voice profile and use Compose, Rewrite, or Respond to generate in-character speech — all powered by a local LLM."
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
A **personality** is an optional free-form description attached to a voice
|
||||||
|
profile — who this voice is, how they speak, what they care about. Set one
|
||||||
|
and three new actions appear on the profile, each powered by a bundled
|
||||||
|
Qwen3 LLM running entirely locally:
|
||||||
|
|
||||||
|
- **Compose** — generate a fresh utterance in this character's voice
|
||||||
|
- **Rewrite** — restate your text in their voice while preserving every idea
|
||||||
|
- **Respond** — treat your text as a prompt and produce the character's reply
|
||||||
|
|
||||||
|
The LLM produces the text. The voice profile speaks it. No cloud round-trip,
|
||||||
|
no external API — the whole loop runs on your hardware.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
Personalities shipped in **0.5.0**. The same local LLM doubles as the
|
||||||
|
refinement model for [Dictation](/overview/dictation) — one LLM in the app,
|
||||||
|
not two, sharing one model cache and one GPU-memory footprint.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## Setting a personality
|
||||||
|
|
||||||
|
Open a voice profile's edit view. The **Personality** field is free-form text
|
||||||
|
up to **2,000 characters**. Describe the voice however helps you — past
|
||||||
|
lines they'd say, speech patterns, tone, boundaries.
|
||||||
|
|
||||||
|
Good descriptions tend to include:
|
||||||
|
|
||||||
|
- A one-line identity (who they are)
|
||||||
|
- Speech patterns (rhythm, vocabulary, what they avoid)
|
||||||
|
- Representative phrases — example lines show the LLM the target tone
|
||||||
|
better than adjectives
|
||||||
|
- What the character *wouldn't* do (they don't explain, they don't
|
||||||
|
apologize, they refuse to break character, etc.)
|
||||||
|
|
||||||
|
You can set a personality on any voice profile type — cloned or preset. The
|
||||||
|
three modes work identically regardless of engine.
|
||||||
|
|
||||||
|
## The three modes
|
||||||
|
|
||||||
|
Each mode is tuned for a specific job and the LLM temperature is adjusted
|
||||||
|
to match.
|
||||||
|
|
||||||
|
### Compose
|
||||||
|
|
||||||
|
Generate a fresh utterance in the character's voice, with no seed text.
|
||||||
|
Click again to get a different take.
|
||||||
|
|
||||||
|
- **When to use:** prototyping, sampling a character's voice, brainstorming
|
||||||
|
a line without typing one first
|
||||||
|
- **Temperature:** hot — variety is the point
|
||||||
|
- **Typical output:** a short, punchy line that fits the character's
|
||||||
|
register
|
||||||
|
|
||||||
|
### Rewrite
|
||||||
|
|
||||||
|
Take your input text and restate it in the character's voice while
|
||||||
|
preserving every idea. High-fidelity mode — the content doesn't change, only
|
||||||
|
the voice does.
|
||||||
|
|
||||||
|
- **When to use:** turning a dictated memo into in-character speech; lifting
|
||||||
|
a plain-English script into a specific voice without editing by hand
|
||||||
|
- **Temperature:** cold — faithfulness wins
|
||||||
|
- **Typical output:** same ideas, same order, different phrasing and cadence
|
||||||
|
|
||||||
|
### Respond
|
||||||
|
|
||||||
|
Treat your input as a prompt and produce the character's reply — as if
|
||||||
|
you'd said it *to* them.
|
||||||
|
|
||||||
|
- **When to use:** spoken-input agents; Q&A with a specific voice;
|
||||||
|
interactive character experiences
|
||||||
|
- **Temperature:** balanced — creative but grounded
|
||||||
|
- **Typical output:** a reply to your prompt, written in-character
|
||||||
|
|
||||||
|
## Speech-only framing
|
||||||
|
|
||||||
|
All three modes enforce **speech-only** output. The LLM is prompted to
|
||||||
|
produce things a person would actually say out loud — no narration, no
|
||||||
|
action tags (`*sighs*`, `[laughs]`), no meta-commentary, no markdown
|
||||||
|
formatting, no stage directions.
|
||||||
|
|
||||||
|
This is deliberate: the output is going straight into TTS, and anything that
|
||||||
|
isn't speakable ends up either ignored or read literally. The speech-only
|
||||||
|
framing also makes the output land cleanly inside dialogue, so you can drop
|
||||||
|
a Respond result straight into a Story.
|
||||||
|
|
||||||
|
## The local LLM
|
||||||
|
|
||||||
|
The bundled LLM is **Qwen3**, available in three sizes:
|
||||||
|
|
||||||
|
| Model | Download size | Best for |
|
||||||
|
|---|---|---|
|
||||||
|
| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual use. |
|
||||||
|
| Qwen3 1.7B | ~1.1 GB | Sweet spot for character personalities with specific phrasing. |
|
||||||
|
| Qwen3 4B | ~2.5 GB | Full quality. Slowest. Useful for very particular tone. |
|
||||||
|
|
||||||
|
The model runs through the same backend split Voicebox already uses for TTS
|
||||||
|
— **MLX** (4-bit community quants) on Apple Silicon, **PyTorch** (transformers
|
||||||
|
`AutoModelForCausalLM`) everywhere else. Downloads go through the same cache
|
||||||
|
and model-management UI as TTS models.
|
||||||
|
|
||||||
|
Pick a size in **Settings → Captures → Refinement → Refinement model** — the
|
||||||
|
personality modes reuse it. If you switch models, both refinement and
|
||||||
|
personality output pick up the change on the next call.
|
||||||
|
|
||||||
|
## Using the modes
|
||||||
|
|
||||||
|
The three actions appear as buttons on the profile when a personality is
|
||||||
|
set. For each:
|
||||||
|
|
||||||
|
<Steps>
|
||||||
|
<Step title="Type or dictate your input">
|
||||||
|
Rewrite and Respond need input text. Compose doesn't.
|
||||||
|
</Step>
|
||||||
|
<Step title="Click the mode">
|
||||||
|
The LLM runs, then the result fills the generate box.
|
||||||
|
</Step>
|
||||||
|
<Step title="Hit generate">
|
||||||
|
The TTS engine speaks the LLM output in the profile's voice. The result
|
||||||
|
lands in generation history as a normal generation.
|
||||||
|
</Step>
|
||||||
|
</Steps>
|
||||||
|
|
||||||
|
Each button also has an inline regenerate affordance — click again to
|
||||||
|
resample. Compose will give you something totally different; Rewrite and
|
||||||
|
Respond will give you a variation on the same content.
|
||||||
|
|
||||||
|
## Use cases
|
||||||
|
|
||||||
|
- **Agents that speak in a voice you own.** Combine Respond with the
|
||||||
|
built-in [MCP Server](/overview/mcp-server) so Claude Code, Cursor,
|
||||||
|
Cline, or any MCP-aware agent can talk back through a profile with a
|
||||||
|
personality. The agent calls `voicebox.speak({ text, profile, intent:
|
||||||
|
"respond" })` and Voicebox produces in-character speech in your cloned
|
||||||
|
voice.
|
||||||
|
- **Interactive characters.** Games, narrative tools, accessibility
|
||||||
|
experiences. A character with a personality description plus a cloned
|
||||||
|
voice becomes a reusable prop.
|
||||||
|
- **Accessibility.** People who can't speak in their original voice can
|
||||||
|
keep a personality description of how they used to sound and use Rewrite
|
||||||
|
to turn typed input into in-character speech.
|
||||||
|
- **Creative drafting.** Write a plain outline, Rewrite line-by-line into
|
||||||
|
the character's voice, drop the audio into a Story.
|
||||||
|
|
||||||
|
## API surface
|
||||||
|
|
||||||
|
Personalities and the three modes are accessible via REST:
|
||||||
|
|
||||||
|
| Method | Endpoint | Body |
|
||||||
|
|---|---|---|
|
||||||
|
| `PUT` | `/profiles/{id}` | Include a `personality` field up to 2,000 chars to set it. |
|
||||||
|
| `POST` | `/profiles/{id}/speak` | Runs the LLM + TTS in one shot. Body includes `text`, `intent` (`compose`, `rewrite`, or `respond`), optional `engine`, `language`. |
|
||||||
|
|
||||||
|
The `/profiles/{id}/speak` endpoint is the same primitive MCP's
|
||||||
|
`voicebox.speak` tool calls when that ships. Scripts and agents can use it
|
||||||
|
directly today.
|
||||||
|
|
||||||
|
## Limits and gotchas
|
||||||
|
|
||||||
|
- **The personality is a prompt, not a fine-tune.** The LLM will sometimes
|
||||||
|
drift out of character, especially on Compose at high temperature. Click
|
||||||
|
again for another take.
|
||||||
|
- **Long personalities are not always better.** 2,000 chars is a ceiling,
|
||||||
|
not a goal. A sharp 300-char description with two example lines
|
||||||
|
typically outperforms a long one.
|
||||||
|
- **Speech-only framing is enforced, but not bulletproof.** Very large
|
||||||
|
prompts or unusual inputs can sneak an action tag through. If you see
|
||||||
|
`[laughs]` in TTS output, it's usually a personality-field hint the
|
||||||
|
model anchored onto — remove it from the description.
|
||||||
|
- **Rewrite is stricter than Respond.** If the output is changing your
|
||||||
|
meaning, you probably want Respond (or a wholesale Compose with context
|
||||||
|
in the input), not Rewrite.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
<Cards>
|
||||||
|
<Card title="Dictation" href="/overview/dictation">
|
||||||
|
Dictate the input for Rewrite or Respond from anywhere on your machine.
|
||||||
|
</Card>
|
||||||
|
<Card title="Captures" href="/overview/captures">
|
||||||
|
Captures feed personalities naturally — dictate a memo, rewrite it in
|
||||||
|
a character voice, generate speech.
|
||||||
|
</Card>
|
||||||
|
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||||
|
Add a personality to an existing profile.
|
||||||
|
</Card>
|
||||||
|
</Cards>
|
||||||
@@ -0,0 +1,344 @@
|
|||||||
|
# MCP Server — Voicebox Speed Run
|
||||||
|
|
||||||
|
**Status:** v1 shipped — HTTP transport, all 4 tools, per-client bindings, `POST /speak`, stdio shim (binary built, bundled into Tauri sidecar), Settings UI, speak-pill via SSE with Rust-side `dictate:show` handler so agent-initiated speech surfaces the pill on screen. `cargo check` clean, `tsc` clean, full Inspector round-trip verified.
|
||||||
|
**Last reviewed:** 2026-04-23
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
### Shipped (backend)
|
||||||
|
- **`fastmcp` + `sse-starlette`** pinned in `backend/requirements.txt`.
|
||||||
|
- **`backend/mcp_server/`** package with `server.py`, `tools.py`, `context.py`, `resolve.py`, `events.py`, `README.md`. Named `mcp_server` (not `mcp`) to sidestep a shadowing conflict with the installed `mcp` PyPI package that FastMCP imports internally.
|
||||||
|
- **Streamable HTTP mount at `/mcp`** via FastMCP's `http_app(transport='http')`. Sub-app lifespan composed with Voicebox's own startup/shutdown through an `@asynccontextmanager lifespan=` in `backend/app.py` (migrated away from the deprecated `@app.on_event` handlers).
|
||||||
|
- **Four MCP tools**, dot-named to match the landing and ecosystem convention:
|
||||||
|
- `voicebox.speak(text, profile?, engine?, intent?, language?)`
|
||||||
|
- `voicebox.transcribe(audio_base64?, audio_path?, language?, model?)`
|
||||||
|
- `voicebox.list_captures(limit, offset)`
|
||||||
|
- `voicebox.list_profiles()`
|
||||||
|
- **`ClientIdMiddleware`** pulls `X-Voicebox-Client-Id` into a `ContextVar` on every `/mcp*` request; auto-stamps `MCPClientBinding.last_seen_at`, auto-creating the row if the client is new.
|
||||||
|
- **Profile resolution precedence** `explicit → per-client binding → capture_settings.default_playback_voice_id → error`. `services/profiles.get_profile_orm_by_name_or_id()` lets agents pass a voice by name ("Morgan") instead of UUID.
|
||||||
|
- **`MCPClientBinding` table** (new) via `Base.metadata.create_all` — no migration needed.
|
||||||
|
- **Bindings REST:** `GET|PUT /mcp/bindings`, `DELETE /mcp/bindings/{client_id}`.
|
||||||
|
- **`POST /speak`** REST wrapper for non-MCP callers (shell / ACP / A2A). Same `resolve_profile` precedence, same code path as the MCP tool.
|
||||||
|
- **Stdio shim** at `backend/mcp_shim/__main__.py` — ~200 lines of `httpx` proxy; reads env (`VOICEBOX_PORT`, `VOICEBOX_HOST`, `VOICEBOX_CLIENT_ID`), waits for `/health`, then streams JSON-RPC ↔ SSE. Rolled our own after the `mcp` SDK's session-management helpers mis-shook-hands. Smoke-tested: `initialize`, `tools/list`, and `tools/call` all round-trip cleanly.
|
||||||
|
- **Pill SSE:** `GET /events/speak` (`sse-starlette`) emits `speak-start` from the MCP tool and `POST /speak`, `speak-end` from `services/generation.run_generation`'s finally block.
|
||||||
|
- **PyInstaller:**
|
||||||
|
- `backend/build_binary.py` `--shim` flag builds a minimal `voicebox-mcp` binary (torch/transformers/mlx/etc. explicitly excluded, target <20 MB).
|
||||||
|
- The main server spec picks up `fastmcp`, `mcp`, `sse_starlette`, and `backend.mcp_server.*` via `--collect-all` / `--hidden-import`.
|
||||||
|
- **`backend/mcp_server/README.md`** quickstart (Inspector, `.mcp.json` snippets, tool reference).
|
||||||
|
|
||||||
|
### Shipped (frontend)
|
||||||
|
- **`Settings → MCP`** page (`app/src/components/ServerTab/MCPPage.tsx`):
|
||||||
|
- Three copy-paste snippets auto-filled with the detected `serverUrl`: HTTP (recommended), Claude Code CLI one-liner, stdio fallback.
|
||||||
|
- Default voice picker (bound to `capture_settings.default_playback_voice_id`, shared with Captures-tab "Play as voice").
|
||||||
|
- Per-client bindings table with inline profile picker, remove button, and a connection-status indicator that refreshes every 10 s.
|
||||||
|
- Add-binding form with client_id / label / profile dropdown.
|
||||||
|
- **`useMCPBindings`** TanStack hook (optimistic delete, invalidate on upsert).
|
||||||
|
- **`useSpeakEvents`** hook — auto-reconnecting `EventSource('/events/speak')`, tracks the active generation_id, exposes an elapsed-ms timer that ticks so the pill's clock advances.
|
||||||
|
- **`CapturePill`** has a new `'speaking'` state + "Speaking" label + playing-bars mode.
|
||||||
|
- **`DictateWindow`** subscribes to speak events and overrides `pillState` when an agent is speaking. Emits `dictate:show` on speak-start so the Rust side can surface the pill window.
|
||||||
|
- Router + `ServerTab` tab bar wired to `/settings/mcp`.
|
||||||
|
|
||||||
|
### Shipped (native shell)
|
||||||
|
- **`tauri.conf.json`** — `voicebox-mcp` added to `externalBin` (alongside `voicebox-server`).
|
||||||
|
- **`dictate:show` listener** in `tauri/src-tauri/src/main.rs` — invokes a new `show_dictate_window(app_handle)` helper that mirrors the hotkey-monitor's position+show logic (undo click-through, reposition to top-center of the current monitor, show). Agent-initiated speech now pops the pill visible on screen.
|
||||||
|
|
||||||
|
### Validated end-to-end (this session, via curl)
|
||||||
|
- `/mcp/` init → `tools/list` → `tools/call voicebox.speak` → actual audio plays (Jarvis, 1.68 s).
|
||||||
|
- `POST /speak` with `X-Voicebox-Client-Id: claude-code` resolves to the bound Jarvis profile without passing `profile`.
|
||||||
|
- `/events/speak` emits `ready`, `speak-start`, `speak-end` in order, generation_id threads through both.
|
||||||
|
- Stdio shim: `echo {…} | python -m backend.mcp_shim` returns valid JSON-RPC for all 4 methods.
|
||||||
|
- `last_seen_at` auto-stamps on first call; binding row auto-creates.
|
||||||
|
- Frontend `tsc --noEmit`: clean.
|
||||||
|
- `cargo check` on the Tauri crate: clean.
|
||||||
|
|
||||||
|
### Outstanding (must-do before release)
|
||||||
|
- **CI build for shim on Windows/Linux** — `python backend/build_binary.py --shim` is wired up and built cleanly for `aarch64-apple-darwin` (18 MB, installed at `tauri/src-tauri/binaries/voicebox-mcp-aarch64-apple-darwin`, Tauri `cargo check` green). The Windows and Linux triples (`x86_64-pc-windows-msvc`, `x86_64-unknown-linux-gnu`) need the same build in their respective CI runners and artifacts dropped alongside the macOS binary.
|
||||||
|
- **Windows/Linux paths in the stdio snippet** — the Settings page hardcodes the macOS path (`/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp`). Needs a per-OS switch (`%LOCALAPPDATA%\Programs\Voicebox\voicebox-mcp.exe`, Linux bundled-path), ideally with the Tauri shell resolving its own app path at runtime and injecting it into the snippet.
|
||||||
|
|
||||||
|
### Nice-to-have (follow-up passes)
|
||||||
|
- **One-click install buttons** — write/merge into `~/.claude/settings.json`, `~/.cursor/mcp.json`, etc. via a Tauri command. Copy-paste works today; this is pure ergonomics.
|
||||||
|
- **`.mcpb` desktop extension** for Claude Desktop (single file, double-click to install). Claude Desktop-only, so lower priority than the agent-harness crowd.
|
||||||
|
- **Refactor the hotkey_monitor.rs show-logic** to call `show_dictate_window()` instead of duplicating the position+show block. Skipped at ship to avoid regressing the well-tested chord path.
|
||||||
|
- **Source attribution on `Generation.source`** — currently `"manual" | "personality_speak"`; adding `"mcp"` / `"rest"` would let the Captures tab filter by MCP-originated rows.
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Voicebox already ships the I/O surface (Captures, Generate, personality-driven `/profiles/{id}/speak`), but local AI agents can't reach any of it. This plan adds a Model Context Protocol server so Claude Code / Cursor / Cline can call `voicebox.speak`, `voicebox.transcribe`, `voicebox.list_captures`, and `voicebox.list_profiles` — turning Voicebox into the local voice layer for every agent on the user's machine (Phase 5 of `docs/plans/VOICE_IO.md`).
|
||||||
|
|
||||||
|
The shortest path to "Claude Code speaks in a cloned voice": mount **FastMCP** inside the existing FastAPI/uvicorn process at `/mcp` (Streamable HTTP), and users install it as a URL (`{"url": "http://127.0.0.1:17493/mcp"}`) — the ecosystem-idiomatic shape for a long-running local service. Per-client voice binding via a new `mcp_client_bindings` table + Settings UI, resolved from an `X-Voicebox-Client-Id` header. A **stdio shim binary** `voicebox-mcp` is bundled as a fallback sidecar for clients that can't speak HTTP MCP. A public `POST /speak` REST wrapper covers non-MCP callers (shell scripts, ACP, A2A). A `speaking` pill state gives agent-initiated audio visibility — trust-critical, non-negotiable.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Claude Code / Cursor / Windsurf / VS Code MCP
|
||||||
|
│
|
||||||
|
├─ HTTP (primary) ────────────────────┐
|
||||||
|
│ {"url": ".../mcp"} │
|
||||||
|
│ │
|
||||||
|
└─ stdio (fallback) ───────────────▶ [voicebox-mcp shim binary]
|
||||||
|
{"command": "/abs/path/voicebox-mcp"} (absolute path;
|
||||||
|
│ Settings page
|
||||||
|
│ copies it for you)
|
||||||
|
▼
|
||||||
|
uvicorn + FastAPI (port 17493)
|
||||||
|
├─ /mcp (FastMCP, Streamable HTTP)
|
||||||
|
└─ /speak (REST wrapper for non-MCP callers)
|
||||||
|
└─ tools call existing services
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Transport:** Streamable HTTP as primary (Nov-2025 spec, post-SSE). Claude Code, Cursor, Windsurf, and the VS Code MCP extensions all support HTTP — it's the idiomatic shape for a long-running local service, which Voicebox already is.
|
||||||
|
- **Stdio fallback:** `voicebox-mcp` binary bundled inside the app for clients that can't speak HTTP MCP. The Settings page renders the exact snippet with the detected absolute path — user copies, pastes, done. No PATH manipulation, no custom CLI wrapper.
|
||||||
|
- **Identity:** HTTP clients set `X-Voicebox-Client-Id` header in their MCP config's `headers` block. Stdio clients set `VOICEBOX_CLIENT_ID` env var, which the shim forwards as the same HTTP header. Server reads it into a `ContextVar`.
|
||||||
|
- **Profile resolution precedence:** explicit tool arg → per-client `MCPClientBinding.profile_id` → `capture_settings.default_playback_voice_id` → error.
|
||||||
|
- **Port:** `17493`, matching `tauri/src-tauri/src/main.rs:63` (`SERVER_PORT` constant). Shim default with `VOICEBOX_PORT` env override.
|
||||||
|
- **Non-MCP access:** `POST /speak` is a thin REST wrapper around the same tool path — one endpoint for shell scripts, ACP, A2A, and anything that isn't MCP-native.
|
||||||
|
|
||||||
|
## Library choice
|
||||||
|
|
||||||
|
- **`fastmcp`** (PyPI — verify on install whether the canonical import is `fastmcp` standalone or `mcp.server.fastmcp` from the consolidated `mcp` package; the API is identical).
|
||||||
|
- **`sse-starlette`** for the `/events/speak` pill-state broadcast.
|
||||||
|
- **`httpx` + `anyio`** already present — used by the shim.
|
||||||
|
|
||||||
|
## Data model
|
||||||
|
|
||||||
|
New table, **one row per client_id** (not a singleton — scales to unknown clients, maps 1:1 to the Settings UI list):
|
||||||
|
|
||||||
|
```python
|
||||||
|
# backend/database/models.py
|
||||||
|
class MCPClientBinding(Base):
|
||||||
|
__tablename__ = "mcp_client_bindings"
|
||||||
|
client_id = Column(String, primary_key=True) # "claude-code", "cursor", ...
|
||||||
|
label = Column(String, nullable=True)
|
||||||
|
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
|
||||||
|
default_engine = Column(String, nullable=True)
|
||||||
|
default_intent = Column(String, nullable=True) # "respond" | "rewrite" | "compose"
|
||||||
|
created_at = Column(DateTime, default=datetime.utcnow)
|
||||||
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||||
|
```
|
||||||
|
|
||||||
|
Global default stays in `capture_settings.default_playback_voice_id` — no duplication. Migration: new `_migrate_mcp_client_bindings()` in `backend/database/migrations.py` using `CREATE TABLE IF NOT EXISTS`, mirroring the existing idempotent-add-column pattern.
|
||||||
|
|
||||||
|
## File plan
|
||||||
|
|
||||||
|
### Backend — new
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `backend/mcp/__init__.py` | Package marker |
|
||||||
|
| `backend/mcp/server.py` | `build_mcp_server()` + `mount_into(app)`; registers tools, middleware, mount at `/mcp` |
|
||||||
|
| `backend/mcp/tools.py` | The 4 `@mcp.tool()` functions — thin wrappers over existing services |
|
||||||
|
| `backend/mcp/context.py` | `current_client_id: ContextVar[str \| None]` + Starlette middleware |
|
||||||
|
| `backend/mcp/resolve.py` | `resolve_profile(explicit, client_id, db) -> VoiceProfile \| None` |
|
||||||
|
| `backend/mcp/events.py` | In-memory `asyncio.Queue` pub/sub for speak-start / speak-end |
|
||||||
|
| `backend/mcp/README.md` | MCP Inspector quickstart + `.mcp.json` snippets |
|
||||||
|
| `backend/mcp_shim/__init__.py`, `__main__.py` | Stdio ↔ Streamable HTTP proxy (~150 lines) |
|
||||||
|
| `backend/voicebox-mcp.spec` | PyInstaller spec for the shim (strips torch/transformers from `hiddenimports`) |
|
||||||
|
| `backend/routes/speak.py` | `POST /speak {text, profile?, engine?, intent?, language?}` — REST wrapper around `resolve_profile()` + `speak_in_character()` for non-MCP agents |
|
||||||
|
|
||||||
|
### Backend — modified
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `backend/app.py` | Migrate `@app.on_event("startup"/"shutdown")` (lines 185, 268) to `lifespan=` kwarg on `FastAPI()` using `AsyncExitStack`; call `mount_into(application)` after `register_routers`. Register `ClientIdMiddleware`. |
|
||||||
|
| `backend/routes/profiles.py` | In `speak_in_character` (line 453): `events.publish("speak-start", {...})` on entry; completion hook publishes `speak-end`. Accept optional `source="mcp"` marker. |
|
||||||
|
| `backend/services/generation.py` | `run_generation` completion path publishes `speak-end`. |
|
||||||
|
| `backend/services/profiles.py` | New `async def get_profile_by_name_or_id(name_or_id, db)` — id lookup first, case-insensitive name fallback. |
|
||||||
|
| `backend/database/models.py` | Add `MCPClientBinding`. |
|
||||||
|
| `backend/database/migrations.py` | Add `_migrate_mcp_client_bindings`. |
|
||||||
|
| `backend/models.py` | Add `MCPClientBindingResponse`, `MCPClientBindingUpdate`. |
|
||||||
|
| `backend/routes/__init__.py` | Register `mcp_bindings_router`, `speak_router`, `events_router`. |
|
||||||
|
| `backend/routes/mcp_bindings.py` (new) | REST CRUD for bindings (list, upsert, delete). |
|
||||||
|
| `backend/routes/events.py` (new) | `GET /events/speak` — `EventSourceResponse` subscribed to the events queue. |
|
||||||
|
| `backend/requirements.txt` | `+ fastmcp` (or `mcp>=1.0`), `+ sse-starlette` |
|
||||||
|
| `backend/voicebox-server.spec` | `hiddenimports += ['mcp', 'mcp.server', 'fastmcp']` |
|
||||||
|
| `backend/build_binary.py` | Second PyInstaller invocation for `voicebox-mcp.spec`; copy to `tauri/src-tauri/binaries/` with target-triple suffix |
|
||||||
|
|
||||||
|
### Frontend — new
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `app/src/components/ServerSettings/MCPBindings.tsx` | Settings section — default voice + per-client binding rows + `.mcp.json` copy-paste cheatsheet |
|
||||||
|
| `app/src/lib/hooks/useMCPBindings.ts` | TanStack Query mirror of `useCaptureSettings` |
|
||||||
|
| `app/src/lib/api/mcp.ts` | `listMCPBindings` / `upsertMCPBinding` / `deleteMCPBinding` |
|
||||||
|
|
||||||
|
### Frontend — modified
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `app/src/components/DictateWindow/DictateWindow.tsx` | Open `EventSource('/events/speak')`; on `speak-start` set pill to `speaking` with profile name; dismiss on `speak-end`. |
|
||||||
|
| `app/src/components/CapturePill/CapturePill.tsx` | Add `speaking` branch — reuse the active waveform, swap status label to profile name. |
|
||||||
|
| `app/src/lib/hooks/useCaptureRecordingSession.ts` | Union a `speaking` injection into the derived pill state. |
|
||||||
|
| `app/src/lib/api/types.ts` | `MCPClientBinding`, `MCPClientBindingUpdate` types. |
|
||||||
|
| `app/src/components/ServerSettings/index.tsx` | Register the new MCP section in the tab aggregator. |
|
||||||
|
|
||||||
|
### Tauri
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `tauri/src-tauri/tauri.conf.json` | `"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"]` |
|
||||||
|
| `tauri/src-tauri/binaries/voicebox-mcp-<triple>` | Build artifact from PyInstaller |
|
||||||
|
|
||||||
|
## Tool signatures
|
||||||
|
|
||||||
|
All tools read `current_client_id.get()` (from middleware). Return JSON-serializable dicts.
|
||||||
|
|
||||||
|
Tools are registered with **dotted names** (`voicebox.speak`, etc.) to match the landing page and the industry convention (`filesystem.read_file`, `github.create_issue`). Python function names stay snake_case; the dot goes in the `name=` kwarg.
|
||||||
|
|
||||||
|
```python
|
||||||
|
# backend/mcp/tools.py
|
||||||
|
|
||||||
|
@mcp.tool(name="voicebox.speak")
|
||||||
|
async def speak(text: str,
|
||||||
|
profile: str | None = None, # name OR id
|
||||||
|
engine: str | None = None,
|
||||||
|
intent: str = "respond", # "respond" | "rewrite" | "compose"
|
||||||
|
language: str | None = None) -> dict:
|
||||||
|
"""Speak text in a voice profile. Returns {generation_id, status, profile, poll}."""
|
||||||
|
# resolve profile via precedence, call speak_in_character (profiles.py:453)
|
||||||
|
# with persist=True so it lands in history.
|
||||||
|
|
||||||
|
@mcp.tool(name="voicebox.transcribe")
|
||||||
|
async def transcribe(audio_base64: str | None = None,
|
||||||
|
audio_path: str | None = None, # absolute local path
|
||||||
|
language: str | None = None,
|
||||||
|
model: str | None = None) -> dict:
|
||||||
|
"""Transcribe audio. Exactly one of audio_base64/audio_path. Returns {text, duration, language}."""
|
||||||
|
# validate path readable, size < 200 MB, then call services.transcribe.transcribe_bytes
|
||||||
|
|
||||||
|
@mcp.tool(name="voicebox.list_captures")
|
||||||
|
async def list_captures(limit: int = 20, offset: int = 0) -> dict:
|
||||||
|
"""Recent captures with transcripts. Returns {captures: [...]}"""
|
||||||
|
|
||||||
|
@mcp.tool(name="voicebox.list_profiles")
|
||||||
|
async def list_profiles() -> dict:
|
||||||
|
"""Available voice profiles. Returns {profiles: [{id, name, voice_type, has_personality}]}"""
|
||||||
|
```
|
||||||
|
|
||||||
|
### `POST /speak` (non-MCP REST wrapper)
|
||||||
|
|
||||||
|
```python
|
||||||
|
# backend/routes/speak.py
|
||||||
|
@router.post("/speak", response_model=GenerationResponse)
|
||||||
|
async def speak(data: SpeakRequest, request: Request, db: Session = Depends(get_db)):
|
||||||
|
"""Same behavior as the MCP tool — for shell scripts, ACP, A2A, or anything non-MCP."""
|
||||||
|
client_id = request.headers.get("X-Voicebox-Client-Id")
|
||||||
|
profile = resolve_profile(data.profile, client_id, db)
|
||||||
|
if profile is None: raise HTTPException(400, "No voice profile resolved.")
|
||||||
|
persist_req = PersonalitySpeakRequest(text=data.text, persist=True, language=data.language,
|
||||||
|
engine=data.engine, intent=data.intent or "respond")
|
||||||
|
return await speak_in_character(profile.id, persist_req, db)
|
||||||
|
```
|
||||||
|
|
||||||
|
`SpeakRequest`: `{ text: str, profile: str | None, engine: str | None, intent: str | None, language: str | None }`. Accepts name OR id for `profile` (via `resolve_profile`), and resolves via the same precedence as the MCP tool so the two surfaces behave identically.
|
||||||
|
|
||||||
|
## Mount point (`backend/app.py`)
|
||||||
|
|
||||||
|
```python
|
||||||
|
# After register_routers(application):
|
||||||
|
from .mcp.server import mount_into
|
||||||
|
mount_into(application)
|
||||||
|
```
|
||||||
|
|
||||||
|
`mount_into` installs `ClientIdMiddleware` and calls `app.mount("/mcp", mcp.streamable_http_app())`.
|
||||||
|
|
||||||
|
**Lifespan migration is load-bearing** — FastMCP's session manager requires the `lifespan=` kwarg, not `@app.on_event`. Wrap the existing startup/shutdown bodies in an `@asynccontextmanager` using `contextlib.AsyncExitStack` so both Voicebox's init and FastMCP's session manager run. Verify dev + packaged build after the migration.
|
||||||
|
|
||||||
|
## Stdio shim (`backend/mcp_shim/__main__.py`)
|
||||||
|
|
||||||
|
1. Port: `int(os.environ.get("VOICEBOX_PORT", "17493"))`.
|
||||||
|
2. Client id: `os.environ.get("VOICEBOX_CLIENT_ID", "unknown")`.
|
||||||
|
3. Health probe `GET /health` with 30 s tolerance (torch imports slowly). On failure, emit JSON-RPC error on stdout, exit 1.
|
||||||
|
4. Connect Streamable HTTP MCP client to `http://127.0.0.1:{port}/mcp` with `X-Voicebox-Client-Id: {client_id}` header.
|
||||||
|
5. Proxy JSON-RPC bidirectionally — stdin → HTTP, SSE → stdout. Use `mcp` SDK's built-in stdio↔HTTP bridge if available; otherwise ~40 lines of asyncio.
|
||||||
|
6. Stdout = JSON-RPC only. All logs to stderr.
|
||||||
|
|
||||||
|
PyInstaller spec keeps only `mcp`, `httpx`, `anyio`, `click` — target binary <20 MB.
|
||||||
|
|
||||||
|
## Pill `speaking` state
|
||||||
|
|
||||||
|
- `backend/mcp/events.py`: module-level `_subscribers: list[asyncio.Queue]` + `publish(kind, payload)` + `subscribe() -> Queue`.
|
||||||
|
- `speak_in_character` publishes `speak-start` with `{generation_id, profile_id, profile_name, source}` immediately after `task_manager.start_generation`; `run_generation`'s completion path publishes `speak-end`.
|
||||||
|
- `/events/speak` → `EventSourceResponse`.
|
||||||
|
- `DictateWindow` opens `EventSource` next to existing `dictate:*` listeners, maps `speak-start/end` → pill `speaking` mode with profile name.
|
||||||
|
- Optional filter: only show pill when `source === "mcp"` (avoids pill churn during manual speak flows). Settings toggle later.
|
||||||
|
|
||||||
|
## Settings UI (`MCPBindings.tsx`)
|
||||||
|
|
||||||
|
- **Global default voice** picker bound to `capture_settings.default_playback_voice_id` (reuses `useCaptureSettings`).
|
||||||
|
- **Per-client table** — add/edit/remove rows of `{client_id, label, profile_id, default_engine, default_intent}`. Uses `useMCPBindings`.
|
||||||
|
- **Connection cheatsheet** — two tabs, HTTP (default) and Stdio (fallback), with copy-to-clipboard snippets per known client:
|
||||||
|
|
||||||
|
HTTP form (primary):
|
||||||
|
```json
|
||||||
|
{"mcpServers": {"voicebox": {
|
||||||
|
"url": "http://127.0.0.1:17493/mcp",
|
||||||
|
"headers": {"X-Voicebox-Client-Id": "claude-code"}
|
||||||
|
}}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Stdio form (fallback, absolute path auto-filled from detected app location):
|
||||||
|
```json
|
||||||
|
{"mcpServers": {"voicebox": {
|
||||||
|
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||||
|
"env": {"VOICEBOX_CLIENT_ID": "claude-code"}
|
||||||
|
}}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Plus the Claude-Code-specific one-liner:
|
||||||
|
```
|
||||||
|
claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code"
|
||||||
|
```
|
||||||
|
- **One-click install buttons** for known clients (v1: Claude Code via `claude mcp add` invocation, and a config-file writer for Cursor/Windsurf whose config locations are known). Each has a matching "Remove" button. Hide buttons for clients not detected on disk.
|
||||||
|
- **Connection status** — small indicator next to each binding showing the last time that `client_id` actually called the server (rolling timestamp recorded by middleware), so users can tell their install worked.
|
||||||
|
|
||||||
|
## Ordered task list (shortest path first)
|
||||||
|
|
||||||
|
1. `fastmcp` + `sse-starlette` → `backend/requirements.txt`; install.
|
||||||
|
2. Add `backend/mcp/{server,tools,context,resolve}.py` with the 4 tools registered as `voicebox.speak` etc. (no middleware yet — global default profile only).
|
||||||
|
3. Migrate `app.py` to `lifespan=`; mount FastMCP at `/mcp`.
|
||||||
|
4. **Milestone:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp` — call `voicebox.speak`, hear audio.
|
||||||
|
5. Add `get_profile_by_name_or_id`; wire the tool's `profile` arg.
|
||||||
|
6. `MCPClientBinding` model + migration; middleware; full `resolve_profile` precedence.
|
||||||
|
7. `backend/routes/speak.py` — `POST /speak` REST wrapper, reusing `resolve_profile` + `speak_in_character`.
|
||||||
|
8. `/mcp/bindings` REST + `MCPBindings.tsx` UI with HTTP and stdio copy-snippets, one-click install for detected clients, and connection-status indicators. **Users can install Voicebox as an MCP server after this step.**
|
||||||
|
9. `backend/mcp_shim/__main__.py` + PyInstaller spec + `build_binary.py` second pass; register `voicebox-mcp` as a Tauri sidecar. (Fallback path goes live.)
|
||||||
|
10. Events queue + `/events/speak` SSE + `DictateWindow` `speaking` pill state.
|
||||||
|
11. `backend/mcp/README.md` quickstart.
|
||||||
|
|
||||||
|
Claude Code can call `voicebox.speak` after step 4 (direct HTTP, manual config). Step 8 makes that a one-click experience. Step 9 adds the stdio fallback for clients that don't speak HTTP MCP.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
- **Step 4 smoke:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp`. Call `voicebox.list_profiles`, then `voicebox.speak(text="hello from mcp")`. Audio plays; generation appears in History with `source="personality_speak"` (or new `source="mcp"` if we add one).
|
||||||
|
- **REST wrapper:** `curl -X POST http://127.0.0.1:17493/speak -d '{"text":"hi","profile":"Morgan"}'` — same behavior, same pill surface.
|
||||||
|
- **Per-client:** open two Inspector sessions with different `X-Voicebox-Client-Id` headers, bind each to a different profile in Settings, verify distinct voices without `profile` arg.
|
||||||
|
- **Claude Code end-to-end (HTTP):** `claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code"`, then ask Claude Code to speak. Pill shows `speaking: <profile>`, audio plays, capture appears in history.
|
||||||
|
- **Stdio fallback:** manually paste the stdio snippet from Settings into a client's config, verify same behavior. `VOICEBOX_CLIENT_ID=claude-code python -m backend.mcp_shim` while backend is up; pipe a tools/list JSON-RPC in, verify response over stdout.
|
||||||
|
- **Transcribe:** point at `/tmp/test.wav`; diff against `POST /transcribe` response.
|
||||||
|
- **Failure modes:** kill backend mid-speak — shim must surface a JSON-RPC error, not deadlock. When backend isn't running, HTTP clients should get a clear connection-refused surfaced by the client.
|
||||||
|
|
||||||
|
## Risks / open decisions
|
||||||
|
|
||||||
|
- **`fastmcp` vs `mcp` package name** — confirm on `pip install`; APIs are near-identical, adjust imports.
|
||||||
|
- **Lifespan migration** touches critical path (DB init, task queue, watchdog). Dev + packaged build both need a smoke after.
|
||||||
|
- **Shim binary size** — if `mcp` pulls in enough dep weight that PyInstaller output is awkward, fall back to a Rust shim (Tauri shell is already Rust; JSON-RPC framing is trivial).
|
||||||
|
- **Source attribution** — consider `source="mcp"` on the `Generation` model, or a dedicated `originator_client` column, if the Captures tab should filter MCP-originated generations.
|
||||||
|
- **`audio_path` in `voicebox_transcribe`** — local-only today, but if the server ever binds beyond 127.0.0.1 we need to restrict reads to `data_dir` + user-whitelist.
|
||||||
|
- **Auth** — none for now (127.0.0.1 only). If we bind outside, bearer token via `~/.voicebox/secret` + plumb through shim.
|
||||||
|
- **HTTP MCP client support** — the plan leads with direct HTTP. Claude Code, Cursor, Windsurf, and VS Code MCP extensions all support it as of 2026, but if we discover an important client is stdio-only we still have the shim fallback ready.
|
||||||
|
- **`.mcpb` desktop extension for Claude Desktop** (v2 polish) — Claude Desktop supports a double-clickable extension bundle format. Worth revisiting after v1 ships for an even cleaner install; skipped for now since Claude Desktop isn't the primary user (Claude Code + IDE users are).
|
||||||
|
|
||||||
|
## Critical files
|
||||||
|
|
||||||
|
- `backend/app.py`
|
||||||
|
- `backend/routes/profiles.py`
|
||||||
|
- `backend/routes/speak.py` (new)
|
||||||
|
- `backend/database/models.py`
|
||||||
|
- `backend/database/migrations.py`
|
||||||
|
- `backend/services/generation.py`
|
||||||
|
- `backend/build_binary.py`
|
||||||
|
- `tauri/src-tauri/tauri.conf.json`
|
||||||
|
- `tauri/src-tauri/src/main.rs` (port constant — no change, just reference)
|
||||||
|
- `app/src/components/DictateWindow/DictateWindow.tsx`
|
||||||
|
- `app/src/components/CapturePill/CapturePill.tsx`
|
||||||
|
- `app/src/components/ServerSettings/`
|
||||||
@@ -60,18 +60,7 @@ const TONE_CLASSES: Record<Scenario['log'][number]['tone'], string> = {
|
|||||||
|
|
||||||
// ─── Console mockup ─────────────────────────────────────────────────────────
|
// ─── Console mockup ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
function AgentConsole() {
|
function AgentConsole({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) {
|
||||||
const [idx, setIdx] = useState(0);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
const iv = window.setInterval(() => {
|
|
||||||
setIdx((i) => (i + 1) % SCENARIOS.length);
|
|
||||||
}, 4200);
|
|
||||||
return () => window.clearInterval(iv);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const scenario = SCENARIOS[idx];
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="rounded-xl border border-app-line bg-app-darkerBox overflow-hidden shadow-[0_20px_60px_rgba(0,0,0,0.35)]">
|
<div className="rounded-xl border border-app-line bg-app-darkerBox overflow-hidden shadow-[0_20px_60px_rgba(0,0,0,0.35)]">
|
||||||
{/* Titlebar */}
|
{/* Titlebar */}
|
||||||
@@ -88,12 +77,11 @@ function AgentConsole() {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Body */}
|
{/* Body */}
|
||||||
<div className="p-5 font-mono text-[12px] leading-relaxed min-h-[280px] flex flex-col">
|
<div className="p-5 font-mono text-[12px] leading-relaxed min-h-[220px] flex flex-col">
|
||||||
{/* Log lines */}
|
<div className="space-y-1.5">
|
||||||
<div className="space-y-1.5 mb-5">
|
|
||||||
{scenario.log.map((line, i) => (
|
{scenario.log.map((line, i) => (
|
||||||
<motion.div
|
<motion.div
|
||||||
key={`${idx}-line-${i}`}
|
key={`${cycleKey}-line-${i}`}
|
||||||
className="flex items-start gap-2"
|
className="flex items-start gap-2"
|
||||||
initial={{ opacity: 0, y: 2 }}
|
initial={{ opacity: 0, y: 2 }}
|
||||||
animate={{ opacity: 1, y: 0 }}
|
animate={{ opacity: 1, y: 0 }}
|
||||||
@@ -111,13 +99,57 @@ function AgentConsole() {
|
|||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* The pill in speaking state — the payoff */}
|
{/* Idle cursor so the terminal doesn't feel empty */}
|
||||||
|
<div className="mt-auto flex items-center gap-2 pt-4">
|
||||||
|
<span className="text-ink-faint/50">$</span>
|
||||||
|
<span className="inline-block h-3.5 w-[7px] bg-ink-faint/40 animate-pulse" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Desktop-floating pill stage ────────────────────────────────────────────
|
||||||
|
|
||||||
|
function AgentSpeakStage({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) {
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="relative rounded-xl border border-app-line bg-app-darkerBox/60 overflow-hidden min-h-[180px] flex-1"
|
||||||
|
style={{
|
||||||
|
backgroundImage: `
|
||||||
|
linear-gradient(to right, hsl(30 10% 94% / 0.04) 1px, transparent 1px),
|
||||||
|
linear-gradient(to bottom, hsl(30 10% 94% / 0.04) 1px, transparent 1px)
|
||||||
|
`,
|
||||||
|
backgroundSize: '28px 28px',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{/* Caption in the corner — "this is on the desktop, not in a terminal" */}
|
||||||
|
<div className="absolute top-3 left-4 text-[9px] font-mono uppercase tracking-[0.22em] text-ink-faint/50">
|
||||||
|
On your desktop
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Voice-tinted glow behind the pill */}
|
||||||
|
<div className="pointer-events-none absolute inset-0 flex items-center justify-center">
|
||||||
<motion.div
|
<motion.div
|
||||||
key={`pill-${idx}`}
|
key={`glow-${cycleKey}`}
|
||||||
className="mt-auto self-start inline-flex items-center gap-2.5 px-3 h-9 rounded-full bg-black/55 backdrop-blur-sm shadow-[0_6px_20px_rgba(0,0,0,0.4)]"
|
className="w-[320px] h-[140px] rounded-full blur-[70px]"
|
||||||
initial={{ opacity: 0, y: 6 }}
|
style={{
|
||||||
|
background: `linear-gradient(135deg, ${scenario.voiceGradient[0]}, ${scenario.voiceGradient[1]})`,
|
||||||
|
}}
|
||||||
|
initial={{ opacity: 0 }}
|
||||||
|
animate={{ opacity: 0.3 }}
|
||||||
|
transition={{ duration: 0.5 }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Pill + utterance caption */}
|
||||||
|
<div className="absolute inset-0 flex flex-col items-center justify-center gap-3 px-6">
|
||||||
|
<motion.div
|
||||||
|
key={`pill-${cycleKey}`}
|
||||||
|
className="inline-flex items-center gap-3 px-4 h-11 rounded-full bg-black/55 backdrop-blur-md shadow-[0_12px_40px_rgba(0,0,0,0.45)]"
|
||||||
|
initial={{ opacity: 0, y: 8 }}
|
||||||
animate={{ opacity: 1, y: 0 }}
|
animate={{ opacity: 1, y: 0 }}
|
||||||
transition={{ duration: 0.35, delay: 0.6 }}
|
transition={{ duration: 0.4, delay: 0.6 }}
|
||||||
>
|
>
|
||||||
<div
|
<div
|
||||||
className="h-4 w-4 rounded-full shrink-0 ring-1 ring-white/10"
|
className="h-4 w-4 rounded-full shrink-0 ring-1 ring-white/10"
|
||||||
@@ -125,19 +157,19 @@ function AgentConsole() {
|
|||||||
background: `linear-gradient(135deg, ${scenario.voiceGradient[0]}, ${scenario.voiceGradient[1]})`,
|
background: `linear-gradient(135deg, ${scenario.voiceGradient[0]}, ${scenario.voiceGradient[1]})`,
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<span className="text-[11px] font-medium text-foreground/90">
|
<span className="text-[12px] font-medium text-foreground/90 shrink-0">
|
||||||
Speaking · <span className="text-accent">{scenario.voice}</span>
|
Speaking · <span className="text-accent">{scenario.voice}</span>
|
||||||
</span>
|
</span>
|
||||||
<div className="flex items-center gap-[2px] h-4">
|
<div className="flex items-center gap-[2.5px] h-5 shrink-0">
|
||||||
{[0, 1, 2, 3, 4, 5].map((i) => (
|
{[0, 1, 2, 3, 4, 5].map((i) => (
|
||||||
<motion.div
|
<motion.div
|
||||||
key={`bar-${scenario.voice}-${i}`}
|
key={`bar-${scenario.voice}-${i}`}
|
||||||
className="w-[2px] rounded-full bg-accent"
|
className="w-[2.5px] rounded-full bg-accent"
|
||||||
animate={{ height: ['4px', '12px', '6px', '10px', '4px'] }}
|
animate={{ height: ['5px', '14px', '7px', '12px', '5px'] }}
|
||||||
transition={{
|
transition={{
|
||||||
duration: 0.9,
|
duration: 1.0,
|
||||||
repeat: Infinity,
|
repeat: Infinity,
|
||||||
delay: i * 0.08,
|
delay: i * 0.09,
|
||||||
ease: 'easeInOut',
|
ease: 'easeInOut',
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
@@ -145,13 +177,12 @@ function AgentConsole() {
|
|||||||
</div>
|
</div>
|
||||||
</motion.div>
|
</motion.div>
|
||||||
|
|
||||||
{/* The utterance — what the agent said */}
|
|
||||||
<motion.div
|
<motion.div
|
||||||
key={`utter-${idx}`}
|
key={`utter-${cycleKey}`}
|
||||||
className="mt-3 text-[11px] text-ink-dull"
|
className="text-[12px] text-ink-dull/80 italic text-center max-w-sm"
|
||||||
initial={{ opacity: 0 }}
|
initial={{ opacity: 0 }}
|
||||||
animate={{ opacity: 1 }}
|
animate={{ opacity: 1 }}
|
||||||
transition={{ duration: 0.4, delay: 0.8 }}
|
transition={{ duration: 0.4, delay: 0.9 }}
|
||||||
>
|
>
|
||||||
“{scenario.utterance}”
|
“{scenario.utterance}”
|
||||||
</motion.div>
|
</motion.div>
|
||||||
@@ -165,8 +196,7 @@ function AgentConsole() {
|
|||||||
const MCP_CONFIG = `{
|
const MCP_CONFIG = `{
|
||||||
"mcpServers": {
|
"mcpServers": {
|
||||||
"voicebox": {
|
"voicebox": {
|
||||||
"command": "voicebox",
|
"url": "http://127.0.0.1:17493/mcp"
|
||||||
"args": ["mcp"]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}`;
|
}`;
|
||||||
@@ -246,13 +276,24 @@ const BULLETS = [
|
|||||||
// ─── Section ────────────────────────────────────────────────────────────────
|
// ─── Section ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
export function AgentIntegration() {
|
export function AgentIntegration() {
|
||||||
|
const [idx, setIdx] = useState(0);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const iv = window.setInterval(() => {
|
||||||
|
setIdx((i) => (i + 1) % SCENARIOS.length);
|
||||||
|
}, 4200);
|
||||||
|
return () => window.clearInterval(iv);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const scenario = SCENARIOS[idx];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section id="agents" className="border-t border-border py-24">
|
<section id="mcp" className="border-t border-border py-24">
|
||||||
<div className="mx-auto max-w-6xl px-6">
|
<div className="mx-auto max-w-6xl px-6">
|
||||||
{/* Header */}
|
{/* Header */}
|
||||||
<div className="max-w-3xl mx-auto text-center mb-14">
|
<div className="max-w-3xl mx-auto text-center mb-14">
|
||||||
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-accent mb-4">
|
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-accent mb-4">
|
||||||
Agents
|
MCP
|
||||||
</div>
|
</div>
|
||||||
<h2 className="text-4xl md:text-5xl font-semibold tracking-tight text-foreground mb-5">
|
<h2 className="text-4xl md:text-5xl font-semibold tracking-tight text-foreground mb-5">
|
||||||
Every agent gets a voice.
|
Every agent gets a voice.
|
||||||
@@ -265,10 +306,13 @@ export function AgentIntegration() {
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Code + console split */}
|
{/* Code (left) + console with pill stage stacked underneath (right) */}
|
||||||
<div className="grid md:grid-cols-2 gap-6 mb-12">
|
<div className="grid md:grid-cols-2 gap-6 mb-12 items-stretch">
|
||||||
<CodePanel />
|
<CodePanel />
|
||||||
<AgentConsole />
|
<div className="flex flex-col gap-4">
|
||||||
|
<AgentConsole scenario={scenario} cycleKey={idx} />
|
||||||
|
<AgentSpeakStage scenario={scenario} cycleKey={idx} />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Bullets */}
|
{/* Bullets */}
|
||||||
|
|||||||
@@ -40,8 +40,28 @@ export function Footer() {
|
|||||||
<h4 className="text-sm font-semibold mb-3">Product</h4>
|
<h4 className="text-sm font-semibold mb-3">Product</h4>
|
||||||
<ul className="space-y-2 text-sm text-muted-foreground">
|
<ul className="space-y-2 text-sm text-muted-foreground">
|
||||||
<li>
|
<li>
|
||||||
<a href="#features" className="hover:text-foreground transition-colors">
|
<a href="/#features" className="hover:text-foreground transition-colors">
|
||||||
Features
|
Clone
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="/capture" className="hover:text-foreground transition-colors">
|
||||||
|
Capture
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="/#mcp" className="hover:text-foreground transition-colors">
|
||||||
|
MCP
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="/#about" className="hover:text-foreground transition-colors">
|
||||||
|
Models
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="/#api" className="hover:text-foreground transition-colors">
|
||||||
|
API
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li>
|
<li>
|
||||||
@@ -49,11 +69,6 @@ export function Footer() {
|
|||||||
Download
|
Download
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li>
|
|
||||||
<a href="#about" className="hover:text-foreground transition-colors">
|
|
||||||
About
|
|
||||||
</a>
|
|
||||||
</li>
|
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ export function Navbar() {
|
|||||||
href="/#features"
|
href="/#features"
|
||||||
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
||||||
>
|
>
|
||||||
Features
|
Clone
|
||||||
</a>
|
</a>
|
||||||
<a
|
<a
|
||||||
href="/capture"
|
href="/capture"
|
||||||
@@ -63,10 +63,10 @@ export function Navbar() {
|
|||||||
</span>
|
</span>
|
||||||
</a>
|
</a>
|
||||||
<a
|
<a
|
||||||
href="/#agents"
|
href="/#mcp"
|
||||||
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
||||||
>
|
>
|
||||||
Agents
|
MCP
|
||||||
</a>
|
</a>
|
||||||
<a
|
<a
|
||||||
href="/#about"
|
href="/#about"
|
||||||
|
|||||||
@@ -166,48 +166,13 @@ impl Chord {
|
|||||||
// Monitor
|
// Monitor
|
||||||
// ========================================================================
|
// ========================================================================
|
||||||
|
|
||||||
/// Hardcoded Pass 1 defaults. Two right-hand modifiers so the usual left-hand
|
|
||||||
/// shortcuts pass through unaffected. Replaced in Pass 2 by reading from the
|
|
||||||
/// server-side `capture_settings` table via a Tauri command the frontend
|
|
||||||
/// invokes whenever `useCaptureSettings` resolves.
|
|
||||||
///
|
|
||||||
/// - **macOS:** `MetaRight + AltGr` — right Command + right Option. (rdev
|
|
||||||
/// labels right-Option as `AltGr` for Linux-convention symmetry; on macOS
|
|
||||||
/// it's the physical right-option key.)
|
|
||||||
/// - **Windows / Linux:** `ControlRight + ShiftRight` — right Ctrl + right
|
|
||||||
/// Shift. Deliberately avoids `AltGr`: on international Windows layouts
|
|
||||||
/// the OS synthesises `AltGr` as `Ctrl+Alt`, so any `AltGr`-involving
|
|
||||||
/// default would fire on every `@`, `€`, `\` keypress on German / French
|
|
||||||
/// / Spanish keyboards.
|
|
||||||
pub fn default_bindings() -> Bindings {
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
let (m1, m2) = (Key::MetaRight, Key::AltGr);
|
|
||||||
#[cfg(not(target_os = "macos"))]
|
|
||||||
let (m1, m2) = (Key::ControlRight, Key::ShiftRight);
|
|
||||||
|
|
||||||
let mut b = Bindings::new();
|
|
||||||
b.insert(ChordAction::PushToTalk, {
|
|
||||||
let mut s = HashSet::new();
|
|
||||||
s.insert(m1);
|
|
||||||
s.insert(m2);
|
|
||||||
s
|
|
||||||
});
|
|
||||||
b.insert(ChordAction::ToggleToTalk, {
|
|
||||||
let mut s = HashSet::new();
|
|
||||||
s.insert(m1);
|
|
||||||
s.insert(m2);
|
|
||||||
s.insert(Key::Space);
|
|
||||||
s
|
|
||||||
});
|
|
||||||
b
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct HotkeyMonitor {
|
pub struct HotkeyMonitor {
|
||||||
chord: Arc<Mutex<Chord>>,
|
chord: Arc<Mutex<Chord>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HotkeyMonitor {
|
impl HotkeyMonitor {
|
||||||
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
||||||
|
eprintln!("[HotkeyMonitor] spawn() called with {} bindings", bindings.len());
|
||||||
let chord = Arc::new(Mutex::new(Chord::new(bindings)));
|
let chord = Arc::new(Mutex::new(Chord::new(bindings)));
|
||||||
let chord_for_thread = chord.clone();
|
let chord_for_thread = chord.clone();
|
||||||
let app_for_thread = app.clone();
|
let app_for_thread = app.clone();
|
||||||
@@ -219,7 +184,9 @@ impl HotkeyMonitor {
|
|||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
rdev::set_is_main_thread(false);
|
rdev::set_is_main_thread(false);
|
||||||
|
|
||||||
|
eprintln!("[HotkeyMonitor] background thread entering rdev::listen");
|
||||||
let result = listen(move |event| {
|
let result = listen(move |event| {
|
||||||
|
eprintln!("[HotkeyMonitor] rdev event: {:?}", event.event_type);
|
||||||
let input = match event.event_type {
|
let input = match event.event_type {
|
||||||
EventType::KeyPress(k) => KeyEvent::Down(k),
|
EventType::KeyPress(k) => KeyEvent::Down(k),
|
||||||
EventType::KeyRelease(k) => KeyEvent::Up(k),
|
EventType::KeyRelease(k) => KeyEvent::Up(k),
|
||||||
@@ -231,11 +198,17 @@ impl HotkeyMonitor {
|
|||||||
Err(_) => return,
|
Err(_) => return,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if !effects.is_empty() {
|
||||||
|
eprintln!("[HotkeyMonitor] chord matched, effects: {:?}", effects);
|
||||||
|
}
|
||||||
|
|
||||||
for effect in effects {
|
for effect in effects {
|
||||||
apply_effect(&app_for_thread, effect);
|
apply_effect(&app_for_thread, effect);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// listen() blocks forever on success; reaching here means it errored.
|
||||||
|
eprintln!("[HotkeyMonitor] rdev::listen returned (this only happens on error): {:?}", result);
|
||||||
if let Err(err) = result {
|
if let Err(err) = result {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.",
|
"HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.",
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
//! Platform permission gate for the global keyboard tap.
|
||||||
|
//!
|
||||||
|
//! On macOS 10.15+, creating a CGEventTap that observes keyboard events
|
||||||
|
//! requires the host process to be listed under System Settings → Privacy &
|
||||||
|
//! Security → Input Monitoring. Without that trust, `rdev::listen` returns
|
||||||
|
//! immediately and no key events ever flow through the chord engine.
|
||||||
|
//!
|
||||||
|
//! The relevant TCC pair lives in IOKit, mirroring `AXIsProcessTrusted` /
|
||||||
|
//! `AXIsProcessTrustedWithOptions` on the Accessibility side:
|
||||||
|
//!
|
||||||
|
//! - `IOHIDCheckAccess(kIOHIDRequestTypeListenEvent)` — read the current
|
||||||
|
//! grant without prompting. We call this from the Captures settings UI
|
||||||
|
//! so the row can show "granted" / "missing" without surprising the user.
|
||||||
|
//! - `IOHIDRequestAccess(kIOHIDRequestTypeListenEvent)` — fire the
|
||||||
|
//! "Voicebox would like to receive keystrokes from any application"
|
||||||
|
//! dialog and add Voicebox to the Input Monitoring pane (toggle off).
|
||||||
|
//! Returns true when access is already granted; otherwise returns false
|
||||||
|
//! and queues the prompt. The user still has to flip the toggle on; this
|
||||||
|
//! just gets us into the list.
|
||||||
|
//!
|
||||||
|
//! `enable_hotkey` calls `request` on first invocation so the prompt fires
|
||||||
|
//! from a deterministic, user-initiated point (the Captures toggle) instead
|
||||||
|
//! of as a side-effect of `rdev::listen` creating its CGEventTap.
|
||||||
|
//!
|
||||||
|
//! Windows / Linux don't gate keyboard taps behind a TCC-style permission,
|
||||||
|
//! so those branches return `true`.
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
mod ffi {
|
||||||
|
use std::os::raw::c_uint;
|
||||||
|
|
||||||
|
/// `kIOHIDRequestTypeListenEvent` from `<IOKit/hidsystem/IOHIDLib.h>` —
|
||||||
|
/// the request-type discriminator for "I want to read keyboard / mouse
|
||||||
|
/// events created by other processes."
|
||||||
|
pub const REQUEST_TYPE_LISTEN_EVENT: c_uint = 1;
|
||||||
|
|
||||||
|
/// `kIOHIDAccessTypeGranted` from `IOHIDLib.h`. The other values are
|
||||||
|
/// `Denied = 1` and `Unknown = 2`; we only ever care about the granted
|
||||||
|
/// case so they don't get their own constants.
|
||||||
|
pub const ACCESS_TYPE_GRANTED: c_uint = 0;
|
||||||
|
|
||||||
|
#[link(name = "IOKit", kind = "framework")]
|
||||||
|
extern "C" {
|
||||||
|
/// Returns the current access state as an `IOHIDAccessType` enum
|
||||||
|
/// (Granted=0, Denied=1, Unknown=2). No prompt side-effect.
|
||||||
|
///
|
||||||
|
/// Declared as `c_uint` rather than `bool`: the C signature returns
|
||||||
|
/// the full enum, and reading a 3-valued enum into Rust's 1-bit
|
||||||
|
/// `bool` is undefined behaviour that silently inverts our gate.
|
||||||
|
pub fn IOHIDCheckAccess(request_type: c_uint) -> c_uint;
|
||||||
|
|
||||||
|
/// Returns true when access is already granted; otherwise queues
|
||||||
|
/// the system prompt and returns false synchronously. Safe to call
|
||||||
|
/// repeatedly — once the entry exists in the Input Monitoring pane
|
||||||
|
/// macOS won't re-prompt. Real `Boolean` (UInt8) return on the C
|
||||||
|
/// side, so `bool` here is correct.
|
||||||
|
pub fn IOHIDRequestAccess(request_type: c_uint) -> bool;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn is_trusted() -> bool {
|
||||||
|
unsafe { ffi::IOHIDCheckAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) == ffi::ACCESS_TYPE_GRANTED }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fire the Input Monitoring prompt if not already granted. Returns the
|
||||||
|
/// current grant state; a `false` here means the prompt was queued and the
|
||||||
|
/// user needs to flip the toggle in System Settings before key events flow.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
pub fn request() -> bool {
|
||||||
|
unsafe { ffi::IOHIDRequestAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
pub fn is_trusted() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
pub fn request() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
+192
-29
@@ -8,6 +8,7 @@ mod clipboard;
|
|||||||
mod focus_capture;
|
mod focus_capture;
|
||||||
#[cfg(desktop)]
|
#[cfg(desktop)]
|
||||||
mod hotkey_monitor;
|
mod hotkey_monitor;
|
||||||
|
mod input_monitoring;
|
||||||
#[cfg(desktop)]
|
#[cfg(desktop)]
|
||||||
mod key_codes;
|
mod key_codes;
|
||||||
mod synthetic_keys;
|
mod synthetic_keys;
|
||||||
@@ -57,6 +58,39 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewW
|
|||||||
Ok(window)
|
Ok(window)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Position, undo click-through, and show the dictate pill window.
|
||||||
|
///
|
||||||
|
/// The hide path parks the window at (-10_000, -10_000) and toggles
|
||||||
|
/// `ignore_cursor_events(true)` so invisible click targets don't leak; we
|
||||||
|
/// undo both here. Mirrors the logic the hotkey_monitor's
|
||||||
|
/// `Effect::StartRecording` path runs, minus the focus snapshot — this is
|
||||||
|
/// for agent-initiated speech, not dictation, so there's no focused text
|
||||||
|
/// field to paste into.
|
||||||
|
#[cfg(desktop)]
|
||||||
|
pub fn show_dictate_window(app: &tauri::AppHandle) {
|
||||||
|
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||||
|
// current_monitor() returns None when the window has been parked
|
||||||
|
// off any display by the hide path; fall back to the primary.
|
||||||
|
let monitor = window
|
||||||
|
.current_monitor()
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.or_else(|| window.primary_monitor().ok().flatten());
|
||||||
|
if let Some(monitor) = monitor {
|
||||||
|
let monitor_pos = monitor.position();
|
||||||
|
let monitor_size = monitor.size();
|
||||||
|
if let Ok(win_size) = window.outer_size() {
|
||||||
|
let x = monitor_pos.x
|
||||||
|
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||||
|
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||||
|
let _ = window.set_position(PhysicalPosition::new(x, y));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = window.set_ignore_cursor_events(false);
|
||||||
|
let _ = window.show();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const LEGACY_PORT: u16 = 8000;
|
const LEGACY_PORT: u16 = 8000;
|
||||||
const SERVER_PORT: u16 = 17493;
|
const SERVER_PORT: u16 = 17493;
|
||||||
|
|
||||||
@@ -791,22 +825,34 @@ fn check_accessibility_permission() -> bool {
|
|||||||
accessibility::is_trusted()
|
accessibility::is_trusted()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push a new chord configuration into the running `HotkeyMonitor`. The
|
/// Reports whether the process can observe global keyboard events. Read by
|
||||||
/// frontend calls this both at startup (replaying the saved chord from
|
/// the Captures settings UI to surface a "missing — open Settings" hint
|
||||||
/// capture_settings) and any time the user edits the chord in the picker —
|
/// beside the hotkey toggle. No prompt side-effect.
|
||||||
/// no app restart needed because the engine swap is atomic under the
|
|
||||||
/// monitor's mutex.
|
|
||||||
///
|
|
||||||
/// Returns an error when a key name doesn't map to an `rdev::Key`, so the
|
|
||||||
/// picker UI can surface "this key isn't supported" instead of silently
|
|
||||||
/// dropping it from the chord.
|
|
||||||
#[cfg(desktop)]
|
|
||||||
#[command]
|
#[command]
|
||||||
fn update_chord_bindings(
|
fn check_input_monitoring_permission() -> bool {
|
||||||
monitor: State<'_, hotkey_monitor::HotkeyMonitor>,
|
input_monitoring::is_trusted()
|
||||||
push_to_talk: Vec<String>,
|
}
|
||||||
toggle_to_talk: Vec<String>,
|
|
||||||
) -> Result<(), String> {
|
/// Holds the lazily-spawned global hotkey monitor. The monitor is `None`
|
||||||
|
/// until the user opts in via the Captures settings toggle — that opt-in is
|
||||||
|
/// what triggers the macOS Input Monitoring TCC prompt, so a fresh-install
|
||||||
|
/// user who never enables the hotkey never sees the prompt.
|
||||||
|
///
|
||||||
|
/// Once spawned, the monitor stays alive for the rest of the process: rdev's
|
||||||
|
/// `listen` blocks forever and offers no stop signal. "Disable" therefore
|
||||||
|
/// swaps the chord engine to empty bindings (matches nothing, fires nothing)
|
||||||
|
/// rather than tearing down the CGEventTap.
|
||||||
|
#[cfg(desktop)]
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct HotkeyState {
|
||||||
|
monitor: Mutex<Option<hotkey_monitor::HotkeyMonitor>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(desktop)]
|
||||||
|
fn build_chord_bindings(
|
||||||
|
push_to_talk: &[String],
|
||||||
|
toggle_to_talk: &[String],
|
||||||
|
) -> Result<hotkey_monitor::Bindings, String> {
|
||||||
use hotkey_monitor::{Bindings, ChordAction};
|
use hotkey_monitor::{Bindings, ChordAction};
|
||||||
use rdev::Key;
|
use rdev::Key;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
@@ -824,14 +870,103 @@ fn update_chord_bindings(
|
|||||||
Ok(chord)
|
Ok(chord)
|
||||||
}
|
}
|
||||||
|
|
||||||
let push_chord = build_chord("push-to-talk", &push_to_talk)?;
|
let push_chord = build_chord("push-to-talk", push_to_talk)?;
|
||||||
let toggle_chord = build_chord("toggle-to-talk", &toggle_to_talk)?;
|
let toggle_chord = build_chord("toggle-to-talk", toggle_to_talk)?;
|
||||||
|
|
||||||
let mut bindings = Bindings::new();
|
let mut bindings = Bindings::new();
|
||||||
bindings.insert(ChordAction::PushToTalk, push_chord);
|
bindings.insert(ChordAction::PushToTalk, push_chord);
|
||||||
bindings.insert(ChordAction::ToggleToTalk, toggle_chord);
|
bindings.insert(ChordAction::ToggleToTalk, toggle_chord);
|
||||||
|
Ok(bindings)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn the global hotkey monitor on first call; subsequent calls just push
|
||||||
|
/// the new bindings into the existing monitor. Idempotent on purpose — the
|
||||||
|
/// frontend invokes this both at startup (when `capture_settings.hotkey_enabled`
|
||||||
|
/// is true) and from the settings toggle.
|
||||||
|
///
|
||||||
|
/// On macOS this is the call that triggers the "Voicebox would like to receive
|
||||||
|
/// keystrokes from any application" TCC prompt, since `rdev::listen` creates
|
||||||
|
/// the CGEventTap inside `HotkeyMonitor::spawn`.
|
||||||
|
#[cfg(desktop)]
|
||||||
|
#[command]
|
||||||
|
fn enable_hotkey(
|
||||||
|
app: tauri::AppHandle,
|
||||||
|
state: State<'_, HotkeyState>,
|
||||||
|
push_to_talk: Vec<String>,
|
||||||
|
toggle_to_talk: Vec<String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
eprintln!("[enable_hotkey] called: push={:?}, toggle={:?}", push_to_talk, toggle_to_talk);
|
||||||
|
let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?;
|
||||||
|
|
||||||
|
// Fire the Input Monitoring TCC prompt explicitly from the user's
|
||||||
|
// toggle click, before rdev::listen would do it implicitly via
|
||||||
|
// CGEventTap creation. Two reasons: (1) the prompt timing becomes
|
||||||
|
// deterministic — it appears in response to a click instead of as a
|
||||||
|
// mysterious side-effect of "the app started"; (2) on subsequent
|
||||||
|
// launches we can short-circuit the spawn entirely if the user
|
||||||
|
// revoked the grant, instead of leaning on rdev silently failing.
|
||||||
|
// The call returns the current grant state; we ignore it because
|
||||||
|
// rdev::listen will surface its own error via stderr, and the
|
||||||
|
// settings UI polls `check_input_monitoring_permission` separately.
|
||||||
|
let granted = input_monitoring::request();
|
||||||
|
eprintln!("[enable_hotkey] IOHIDRequestAccess returned granted={}", granted);
|
||||||
|
eprintln!("[enable_hotkey] IOHIDCheckAccess says trusted={}", input_monitoring::is_trusted());
|
||||||
|
|
||||||
|
// The dictate pill webview must exist before the first chord fires so it
|
||||||
|
// can subscribe to `dictate:start`. Build it here (idempotent — Tauri
|
||||||
|
// returns the existing window when one with this label already exists).
|
||||||
|
if app.get_webview_window(DICTATE_WINDOW_LABEL).is_none() {
|
||||||
|
if let Err(e) = build_dictate_window(&app) {
|
||||||
|
eprintln!("Failed to build dictate window: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||||
|
match slot.as_ref() {
|
||||||
|
Some(monitor) => monitor.update_bindings(bindings),
|
||||||
|
None => {
|
||||||
|
*slot = Some(hotkey_monitor::HotkeyMonitor::spawn(app, bindings));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Quiet the global hotkey by swapping the chord engine to empty bindings.
|
||||||
|
/// The CGEventTap stays alive (rdev::listen has no stop) but the chord state
|
||||||
|
/// machine matches nothing, so no `dictate:*` events fire and the dictate
|
||||||
|
/// pill never shows. A subsequent `enable_hotkey` call re-arms it without
|
||||||
|
/// re-prompting for permission.
|
||||||
|
#[cfg(desktop)]
|
||||||
|
#[command]
|
||||||
|
fn disable_hotkey(state: State<'_, HotkeyState>) -> Result<(), String> {
|
||||||
|
let slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||||
|
if let Some(monitor) = slot.as_ref() {
|
||||||
|
monitor.update_bindings(hotkey_monitor::Bindings::new());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push a new chord configuration into the running `HotkeyMonitor`. Called
|
||||||
|
/// by the chord-picker UI when the user edits the chord. No-ops when the
|
||||||
|
/// monitor isn't spawned — the picker is gated behind the enable toggle, so
|
||||||
|
/// this can only happen if the frontend races; the next `enable_hotkey` will
|
||||||
|
/// pick up the saved chords.
|
||||||
|
///
|
||||||
|
/// Returns an error when a key name doesn't map to an `rdev::Key`, so the
|
||||||
|
/// picker UI can surface "this key isn't supported" instead of silently
|
||||||
|
/// dropping it from the chord.
|
||||||
|
#[cfg(desktop)]
|
||||||
|
#[command]
|
||||||
|
fn update_chord_bindings(
|
||||||
|
state: State<'_, HotkeyState>,
|
||||||
|
push_to_talk: Vec<String>,
|
||||||
|
toggle_to_talk: Vec<String>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?;
|
||||||
|
let slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||||
|
if let Some(monitor) = slot.as_ref() {
|
||||||
monitor.update_bindings(bindings);
|
monitor.update_bindings(bindings);
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -855,6 +990,26 @@ fn open_accessibility_settings(app: tauri::AppHandle) -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open the Privacy & Security → Input Monitoring pane in System Settings.
|
||||||
|
/// Used by the Captures settings UI when the toggle is on but the grant
|
||||||
|
/// is missing, so the user can flip the system toggle without hunting.
|
||||||
|
#[command]
|
||||||
|
fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> {
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
let url = "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent";
|
||||||
|
app.shell()
|
||||||
|
.open(url, None)
|
||||||
|
.map_err(|e| format!("Failed to open Input Monitoring settings: {e}"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
{
|
||||||
|
let _ = app;
|
||||||
|
Err("Input Monitoring settings pane is only implemented on macOS".into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Deliver `text` into the UI that had focus when the chord fired.
|
/// Deliver `text` into the UI that had focus when the chord fired.
|
||||||
///
|
///
|
||||||
/// Pipeline: activate the captured PID → settle → save the user's
|
/// Pipeline: activate the captured PID → settle → save the user's
|
||||||
@@ -1044,18 +1199,12 @@ pub fn run() {
|
|||||||
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
|
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
|
||||||
app.handle().plugin(tauri_plugin_process::init())?;
|
app.handle().plugin(tauri_plugin_process::init())?;
|
||||||
|
|
||||||
if let Err(e) = build_dictate_window(app.handle()) {
|
// HotkeyMonitor is spawned lazily via the `enable_hotkey`
|
||||||
eprintln!("Failed to pre-create dictate window: {}", e);
|
// command — see HotkeyState. The dictate pill webview is
|
||||||
}
|
// built in the same lazy path so we don't pay setup cost
|
||||||
|
// (and don't trigger the macOS Input Monitoring TCC prompt)
|
||||||
let monitor = hotkey_monitor::HotkeyMonitor::spawn(
|
// for users who never enable the global hotkey.
|
||||||
app.handle().clone(),
|
app.manage(HotkeyState::default());
|
||||||
hotkey_monitor::default_bindings(),
|
|
||||||
);
|
|
||||||
// Stored as state so the chord-picker UI can call
|
|
||||||
// `update_chord_bindings` to live-swap the engine's chords
|
|
||||||
// without restarting the listener thread.
|
|
||||||
app.manage(monitor);
|
|
||||||
|
|
||||||
// The frontend emits `dictate:hide` whenever the pill cycle
|
// The frontend emits `dictate:hide` whenever the pill cycle
|
||||||
// finishes (rest-fade → hidden). `hide()` alone has been
|
// finishes (rest-fade → hidden). `hide()` alone has been
|
||||||
@@ -1073,6 +1222,16 @@ pub fn run() {
|
|||||||
let _ = window.hide();
|
let _ = window.hide();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Agent-initiated speech (voicebox.speak over MCP or POST /speak)
|
||||||
|
// pops the pill up so the user can see what's coming out of their
|
||||||
|
// machine. The DictateWindow subscribes to /events/speak via SSE
|
||||||
|
// and emits `dictate:show` on speak-start; we repeat the same
|
||||||
|
// position+show dance the hotkey path uses.
|
||||||
|
let handle_for_show = app.handle().clone();
|
||||||
|
app.handle().listen("dictate:show", move |_event| {
|
||||||
|
show_dictate_window(&handle_for_show);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hide title bar icon on Windows
|
// Hide title bar icon on Windows
|
||||||
@@ -1148,8 +1307,12 @@ pub fn run() {
|
|||||||
debug_capture_focus,
|
debug_capture_focus,
|
||||||
debug_focus_roundtrip,
|
debug_focus_roundtrip,
|
||||||
check_accessibility_permission,
|
check_accessibility_permission,
|
||||||
|
check_input_monitoring_permission,
|
||||||
open_accessibility_settings,
|
open_accessibility_settings,
|
||||||
|
open_input_monitoring_settings,
|
||||||
paste_final_text,
|
paste_final_text,
|
||||||
|
enable_hotkey,
|
||||||
|
disable_hotkey,
|
||||||
update_chord_bindings
|
update_chord_bindings
|
||||||
])
|
])
|
||||||
.on_window_event({
|
.on_window_event({
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
"active": true,
|
"active": true,
|
||||||
"targets": "all",
|
"targets": "all",
|
||||||
"createUpdaterArtifacts": "v1Compatible",
|
"createUpdaterArtifacts": "v1Compatible",
|
||||||
"externalBin": ["binaries/voicebox-server"],
|
"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"],
|
||||||
"icon": [
|
"icon": [
|
||||||
"icons/32x32.png",
|
"icons/32x32.png",
|
||||||
"icons/128x128.png",
|
"icons/128x128.png",
|
||||||
|
|||||||
Reference in New Issue
Block a user