mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-16 13:20:39 -07:00
feat(mcp): local MCP server exposes voicebox.* tools to AI agents
Mounts FastMCP at /mcp (Streamable HTTP) so Claude Code, Cursor, Windsurf, and the VS Code MCP extensions can call voicebox.speak, voicebox.transcribe, voicebox.list_captures, and voicebox.list_profiles against the running Voicebox server. Backend - new backend/mcp_server package (tools, middleware, profile resolve, pub/sub events); named mcp_server to avoid shadowing the installed mcp PyPI package FastMCP imports internally - app.py migrated from @app.on_event to lifespan= so FastMCP's session manager cohabits with Voicebox's startup/shutdown - new MCPClientBinding table + /mcp/bindings CRUD; ClientIdMiddleware reads X-Voicebox-Client-Id into a ContextVar and stamps last_seen_at - profile resolution precedence: explicit -> per-client binding -> capture_settings.default_playback_voice_id - POST /speak REST wrapper for non-MCP callers (shell, ACP, A2A) - GET /events/speak SSE broadcasts speak-start / speak-end so the pill surfaces agent-initiated speech - backend/mcp_shim proxy (plain httpx) for stdio-only MCP clients - PyInstaller spec updates + new --shim build target (~18 MB) Frontend - Settings -> MCP page with HTTP / stdio / claude-mcp-add copy snippets, default voice picker, per-client bindings table, connection status - useMCPBindings, useSpeakEvents hooks - CapturePill gains 'speaking' state; DictateWindow subscribes to SSE and emits dictate:show so the Rust side surfaces the pill window Native - tauri.conf.json externalBin now includes voicebox-mcp - show_dictate_window helper + dictate:show listener in main.rs - (also in this commit: InputMonitoringGate UX, hotkey_monitor tweaks, landing footer/navbar updates, new overview docs for captures / dictation / mcp-server / voice-personalities) Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
87c582ad54
commit
0cef2c9fe1
@@ -10,6 +10,7 @@ export type PillState =
|
||||
| 'recording'
|
||||
| 'transcribing'
|
||||
| 'refining'
|
||||
| 'speaking'
|
||||
| 'completed'
|
||||
| 'rest'
|
||||
| 'error';
|
||||
@@ -18,13 +19,14 @@ const PILL_LABELS: Record<Exclude<PillState, 'rest' | 'error'>, string> = {
|
||||
recording: 'Recording',
|
||||
transcribing: 'Transcribing',
|
||||
refining: 'Refining',
|
||||
speaking: 'Speaking',
|
||||
completed: 'Done',
|
||||
};
|
||||
|
||||
function barModeFor(
|
||||
state: Exclude<PillState, 'error'>,
|
||||
): 'generating' | 'playing' | 'idle' {
|
||||
if (state === 'recording') return 'playing';
|
||||
if (state === 'recording' || state === 'speaking') return 'playing';
|
||||
if (state === 'completed' || state === 'rest') return 'idle';
|
||||
return 'generating';
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ import { BOTTOM_SAFE_AREA_PADDING } from '@/lib/constants/ui';
|
||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||
import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { displayLabelForKey, modifierSideHint } from '@/lib/utils/keyCodes';
|
||||
import { usePlayerStore } from '@/stores/playerStore';
|
||||
|
||||
const CAPTURE_AUDIO_MIME = 'audio/*,.wav,.mp3,.m4a,.flac,.ogg,.webm';
|
||||
@@ -86,6 +87,30 @@ function snippetOf(capture: CaptureResponse): string {
|
||||
return source.trim() || '(no transcript)';
|
||||
}
|
||||
|
||||
function ChordKeys({ keys }: { keys: string[] }) {
|
||||
if (keys.length === 0) return null;
|
||||
return (
|
||||
<div className="flex items-center gap-1">
|
||||
{keys.map((k) => {
|
||||
const side = modifierSideHint(k);
|
||||
return (
|
||||
<span
|
||||
key={k}
|
||||
className="relative inline-flex items-center justify-center h-6 min-w-[1.5rem] px-1.5 rounded-md border border-border bg-muted/60 font-mono text-[11px] font-medium shadow-sm text-foreground"
|
||||
>
|
||||
{displayLabelForKey(k)}
|
||||
{side ? (
|
||||
<span className="absolute -top-1 -right-1 h-3 min-w-[0.75rem] px-0.5 rounded-sm bg-accent text-[7px] font-bold leading-none flex items-center justify-center text-accent-foreground">
|
||||
{side}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SourceBadge({ source }: { source: CaptureSource }) {
|
||||
const Icon = source === 'dictation' ? Mic : source === 'recording' ? CircleDot : FileAudio;
|
||||
const label = source === 'dictation' ? 'Dictation' : source === 'recording' ? 'Recording' : 'File';
|
||||
@@ -160,6 +185,9 @@ export function CapturesTab() {
|
||||
const { settings: captureSettings } = useCaptureSettings();
|
||||
const sttModel = captureSettings?.stt_model ?? 'turbo';
|
||||
const llmModel = captureSettings?.llm_model ?? '0.6B';
|
||||
const hotkeyEnabled = captureSettings?.hotkey_enabled ?? false;
|
||||
const pushToTalkKeys = captureSettings?.chord_push_to_talk_keys ?? [];
|
||||
const toggleToTalkKeys = captureSettings?.chord_toggle_to_talk_keys ?? [];
|
||||
|
||||
const session = useCaptureRecordingSession({
|
||||
onCaptureCreated: (capture) => setSelectedId(capture.id),
|
||||
@@ -394,17 +422,11 @@ export function CapturesTab() {
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
</div>
|
||||
) : filtered.length === 0 ? (
|
||||
<div className="px-4 py-12 text-center text-sm text-muted-foreground space-y-3">
|
||||
<div className="px-4 py-12 text-center text-sm text-muted-foreground">
|
||||
{search ? (
|
||||
<p>No captures match "{search}"</p>
|
||||
) : (
|
||||
<>
|
||||
<p>No captures yet.</p>
|
||||
<Button variant="outline" size="sm" onClick={handleUploadClick}>
|
||||
<Upload className="h-3.5 w-3.5 mr-1.5" />
|
||||
Import audio
|
||||
</Button>
|
||||
</>
|
||||
<p>No captures yet.</p>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
@@ -744,22 +766,53 @@ export function CapturesTab() {
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex-1 flex items-center justify-center text-muted-foreground pt-20">
|
||||
<div className="text-center space-y-3">
|
||||
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||
{capturesLoading ? (
|
||||
{capturesLoading ? (
|
||||
<div className="text-center space-y-3">
|
||||
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||
<p className="text-sm">Loading captures…</p>
|
||||
) : captures.length ? (
|
||||
</div>
|
||||
) : captures.length ? (
|
||||
<div className="text-center space-y-3">
|
||||
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||
<p className="text-sm">Pick a capture to see the transcript.</p>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-sm">No captures yet.</p>
|
||||
<Button variant="outline" size="sm" onClick={handleUploadClick}>
|
||||
<Upload className="h-3.5 w-3.5 mr-1.5" />
|
||||
Import audio
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
) : hotkeyEnabled && (pushToTalkKeys.length || toggleToTalkKeys.length) ? (
|
||||
<div className="max-w-sm mx-auto text-center space-y-5">
|
||||
<div className="space-y-2">
|
||||
{pushToTalkKeys.length ? (
|
||||
<div className="flex items-center justify-center gap-3">
|
||||
<ChordKeys keys={pushToTalkKeys} />
|
||||
<span className="text-[11px] uppercase tracking-wider text-muted-foreground">
|
||||
Hold to record
|
||||
</span>
|
||||
</div>
|
||||
) : null}
|
||||
{toggleToTalkKeys.length ? (
|
||||
<div className="flex items-center justify-center gap-3">
|
||||
<ChordKeys keys={toggleToTalkKeys} />
|
||||
<span className="text-[11px] uppercase tracking-wider text-muted-foreground">
|
||||
Toggle hands-free
|
||||
</span>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
<p className="text-sm">
|
||||
Press the shortcut anywhere on your machine to start your first capture.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="max-w-sm mx-auto text-center space-y-3">
|
||||
<Captions className="h-10 w-10 mx-auto opacity-40" />
|
||||
<p className="text-sm">No captures yet.</p>
|
||||
<p className="text-xs text-muted-foreground leading-relaxed">
|
||||
Turn on the global shortcut to dictate from anywhere — or click
|
||||
Dictate above for an in-app capture.
|
||||
</p>
|
||||
<Button asChild variant="outline" size="sm">
|
||||
<Link to="/settings/captures">Open Captures settings</Link>
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -4,6 +4,7 @@ import { useEffect, useRef } from 'react';
|
||||
import { CapturePill } from '@/components/CapturePill/CapturePill';
|
||||
import type { FocusSnapshot } from '@/lib/api/types';
|
||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||
import { useSpeakEvents } from '@/lib/hooks/useSpeakEvents';
|
||||
|
||||
/**
|
||||
* Floating dictate surface shown in a separate transparent Tauri window.
|
||||
@@ -82,27 +83,47 @@ export function DictateWindow() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
// When the pill cycle ends, tell Rust to tuck the window away. The Rust
|
||||
// side is responsible for the hide + park-off-screen + click-through
|
||||
// combo because calling hide() directly from JS has been unreliable for
|
||||
// transparent always-on-top windows on macOS. Showing is the reverse —
|
||||
// the HotkeyMonitor restores position, clicks, and visibility when a
|
||||
// chord next fires.
|
||||
// Subscribe to agent-initiated speak events so the pill surfaces while
|
||||
// voicebox.speak (MCP) or POST /speak is producing audio. We ask Rust
|
||||
// to show the pill window by emitting `dictate:show` — the existing
|
||||
// `dictate:hide` happens on cycle end below.
|
||||
const speaking = useSpeakEvents();
|
||||
const prevSpeakingIdRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (session.pillState === 'hidden') {
|
||||
const id = speaking?.generationId ?? null;
|
||||
if (id && id !== prevSpeakingIdRef.current) {
|
||||
emit('dictate:show').catch(() => {});
|
||||
}
|
||||
prevSpeakingIdRef.current = id;
|
||||
}, [speaking?.generationId]);
|
||||
|
||||
// Compose the effective pill state: speak events override the capture
|
||||
// session when both would render, because agent speech is always
|
||||
// category-mattering (the user can't hear two pills). Elapsed time
|
||||
// restarts for speaking so the timer reflects playback length.
|
||||
const isSpeaking = Boolean(speaking);
|
||||
const effectiveState = isSpeaking ? 'speaking' : session.pillState;
|
||||
const effectiveElapsed = isSpeaking ? speaking!.elapsedMs : session.pillElapsedMs;
|
||||
|
||||
// When the pill cycle ends (no capture AND no speak), tell Rust to tuck
|
||||
// the window away. Rust owns the hide + park-off-screen + click-through
|
||||
// combo because calling hide() directly from JS has been unreliable for
|
||||
// transparent always-on-top windows on macOS.
|
||||
useEffect(() => {
|
||||
if (effectiveState === 'hidden') {
|
||||
emit('dictate:hide').catch(() => {});
|
||||
}
|
||||
}, [session.pillState]);
|
||||
}, [effectiveState]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="h-screen w-screen flex items-center justify-center px-3"
|
||||
style={{ background: 'transparent' }}
|
||||
>
|
||||
{session.pillState !== 'hidden' ? (
|
||||
{effectiveState !== 'hidden' ? (
|
||||
<CapturePill
|
||||
state={session.pillState}
|
||||
elapsedMs={session.pillElapsedMs}
|
||||
state={effectiveState}
|
||||
elapsedMs={effectiveElapsed}
|
||||
errorMessage={session.errorMessage}
|
||||
onDismiss={session.dismissError}
|
||||
onStop={session.isRecording ? session.stopRecording : undefined}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import { AlertTriangle, ExternalLink } from 'lucide-react';
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
|
||||
/**
|
||||
* Tracks macOS Input Monitoring permission state. Without it, `rdev::listen`
|
||||
* sees no key events and the chord engine never fires — but neither does
|
||||
* anything error-out visibly, so we surface an inline prompt next to the
|
||||
* hotkey toggle instead of leaving the user wondering why the shortcut is
|
||||
* dead.
|
||||
*
|
||||
* Re-checked on mount and on window focus (cheap way to pick up the user
|
||||
* flipping the toggle in System Settings and alt-tabbing back).
|
||||
*/
|
||||
export function useInputMonitoringPermission() {
|
||||
const platform = usePlatform();
|
||||
const [needsPermission, setNeedsPermission] = useState(false);
|
||||
const [checking, setChecking] = useState(false);
|
||||
|
||||
const recheck = useCallback(async (): Promise<boolean> => {
|
||||
if (!platform.metadata.isTauri) return true;
|
||||
setChecking(true);
|
||||
try {
|
||||
const trusted = await invoke<boolean>('check_input_monitoring_permission');
|
||||
setNeedsPermission(!trusted);
|
||||
return trusted;
|
||||
} catch (err) {
|
||||
console.warn('[input-monitoring] check failed:', err);
|
||||
return false;
|
||||
} finally {
|
||||
setChecking(false);
|
||||
}
|
||||
}, [platform.metadata.isTauri]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!platform.metadata.isTauri) return;
|
||||
recheck();
|
||||
const onFocus = () => {
|
||||
recheck();
|
||||
};
|
||||
window.addEventListener('focus', onFocus);
|
||||
return () => window.removeEventListener('focus', onFocus);
|
||||
}, [platform.metadata.isTauri, recheck]);
|
||||
|
||||
const openSettings = useCallback(async () => {
|
||||
try {
|
||||
await invoke('open_input_monitoring_settings');
|
||||
} catch (err) {
|
||||
console.warn('[input-monitoring] open settings failed:', err);
|
||||
}
|
||||
}, []);
|
||||
|
||||
return { needsPermission, checking, recheck, openSettings };
|
||||
}
|
||||
|
||||
/**
|
||||
* Inline notice rendered under the global-shortcut toggle when the user has
|
||||
* opted in but macOS Input Monitoring is not granted. Returns null when the
|
||||
* permission is present (or when the toggle is off and the notice would just
|
||||
* be noise).
|
||||
*/
|
||||
export function InputMonitoringNotice({ enabled }: { enabled: boolean }) {
|
||||
const { needsPermission, checking, recheck, openSettings } =
|
||||
useInputMonitoringPermission();
|
||||
const [stillMissing, setStillMissing] = useState(false);
|
||||
|
||||
const handleRecheck = useCallback(async () => {
|
||||
setStillMissing(false);
|
||||
const trusted = await recheck();
|
||||
if (!trusted) setStillMissing(true);
|
||||
}, [recheck]);
|
||||
|
||||
if (!enabled || !needsPermission) return null;
|
||||
|
||||
return (
|
||||
<div className="mt-3 rounded-lg border border-amber-500/30 bg-amber-500/10 px-3.5 py-3">
|
||||
<div className="flex items-start gap-3">
|
||||
<AlertTriangle className="h-4 w-4 shrink-0 mt-0.5 text-amber-500" />
|
||||
<div className="flex-1 min-w-0 space-y-1">
|
||||
<p className="text-sm font-medium text-foreground">
|
||||
Grant Input Monitoring to enable the global shortcut
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground leading-relaxed">
|
||||
Voicebox needs System Settings → Privacy & Security → Input
|
||||
Monitoring to detect your dictation chord. The toggle is on, but
|
||||
macOS is blocking key events until you allow it.
|
||||
</p>
|
||||
<div className="flex items-center gap-2 pt-1.5">
|
||||
<Button size="sm" onClick={openSettings} className="gap-1.5">
|
||||
<ExternalLink className="h-3.5 w-3.5" />
|
||||
Open Settings
|
||||
</Button>
|
||||
<Button variant="outline" size="sm" onClick={handleRecheck} disabled={checking}>
|
||||
{checking ? 'Checking…' : "I've enabled it"}
|
||||
</Button>
|
||||
</div>
|
||||
{stillMissing && !checking && (
|
||||
<p className="text-xs text-amber-600 dark:text-amber-400 pt-1">
|
||||
Still not detected. macOS usually requires quitting and reopening
|
||||
Voicebox after toggling the permission.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Check, ChevronDown, Keyboard, Laptop, Lock, Trash2, Volume2 } from 'lucide-react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { AccessibilityNotice } from '@/components/AccessibilityGate/AccessibilityGate';
|
||||
import { InputMonitoringNotice } from '@/components/InputMonitoringGate/InputMonitoringGate';
|
||||
import { CapturePill, type PillState } from '@/components/CapturePill/CapturePill';
|
||||
import { ChordPicker } from '@/components/ChordPicker/ChordPicker';
|
||||
import { Button } from '@/components/ui/button';
|
||||
@@ -140,13 +141,13 @@ export function CapturesPage() {
|
||||
const preserveTechnical = settings?.preserve_technical ?? true;
|
||||
const allowAutoPaste = settings?.allow_auto_paste ?? true;
|
||||
const defaultVoiceId = settings?.default_playback_voice_id ?? null;
|
||||
const hotkeyEnabled = settings?.hotkey_enabled ?? false;
|
||||
const pushToTalkKeys = settings?.chord_push_to_talk_keys ?? ['MetaRight', 'AltGr'];
|
||||
const toggleToTalkKeys = settings?.chord_toggle_to_talk_keys ?? ['MetaRight', 'AltGr', 'Space'];
|
||||
|
||||
// Mock-only settings — not yet wired to a backend. Keep local so the UI
|
||||
// still responds while Phase 7 (hotkey / clipboard / paste) catches up.
|
||||
const [archiveAudio, setArchiveAudio] = useState(true);
|
||||
const [hotkeyEnabled, setHotkeyEnabled] = useState(true);
|
||||
const [copyToClipboard, setCopyToClipboard] = useState(true);
|
||||
const [retention, setRetention] = useState('forever');
|
||||
const [chordEditor, setChordEditor] = useState<'push' | 'toggle' | null>(null);
|
||||
@@ -162,14 +163,21 @@ export function CapturesPage() {
|
||||
title="Dictation"
|
||||
description="Capture from anywhere on your machine with a global shortcut."
|
||||
>
|
||||
<SettingRow
|
||||
title="Global shortcut"
|
||||
description="Hold the shortcut to record. Release to transcribe. Requires an Accessibility permission the first time you enable it."
|
||||
htmlFor="hotkeyEnabled"
|
||||
action={
|
||||
<Toggle id="hotkeyEnabled" checked={hotkeyEnabled} onCheckedChange={setHotkeyEnabled} />
|
||||
}
|
||||
/>
|
||||
<div>
|
||||
<SettingRow
|
||||
title="Global shortcut"
|
||||
description="Hold the shortcut to record from anywhere on your machine. Release to transcribe. macOS will ask for Input Monitoring permission the first time you turn this on."
|
||||
htmlFor="hotkeyEnabled"
|
||||
action={
|
||||
<Toggle
|
||||
id="hotkeyEnabled"
|
||||
checked={hotkeyEnabled}
|
||||
onCheckedChange={(v) => update({ hotkey_enabled: v })}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
<InputMonitoringNotice enabled={hotkeyEnabled} />
|
||||
</div>
|
||||
|
||||
<SettingRow
|
||||
title="Push-to-talk shortcut"
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
import { Check, Copy, Plug, Trash2, Waypoints } from 'lucide-react';
|
||||
import { useState } from 'react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { useMCPBindings } from '@/lib/hooks/useMCPBindings';
|
||||
import { useProfiles } from '@/lib/hooks/useProfiles';
|
||||
import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
import { SettingRow, SettingSection } from './SettingRow';
|
||||
|
||||
/**
|
||||
* Settings → MCP — configure per-agent voice binding and show copy-paste
|
||||
* install snippets for major MCP clients. Backend runs at /mcp on the
|
||||
* existing Voicebox server; this page is the agent-onboarding surface.
|
||||
*/
|
||||
export function MCPPage() {
|
||||
const serverUrl = useServerStore((s) => s.serverUrl);
|
||||
const { bindings, upsertAsync, remove } = useMCPBindings();
|
||||
const { data: profiles } = useProfiles();
|
||||
const { settings: captureSettings, update: updateCapture } = useCaptureSettings();
|
||||
|
||||
const defaultProfileId = captureSettings?.default_playback_voice_id ?? '';
|
||||
const mcpUrl = `${serverUrl}/mcp`;
|
||||
|
||||
const [newClientId, setNewClientId] = useState('');
|
||||
const [newLabel, setNewLabel] = useState('');
|
||||
const [newProfileId, setNewProfileId] = useState('');
|
||||
const [adding, setAdding] = useState(false);
|
||||
|
||||
const handleAdd = async () => {
|
||||
if (!newClientId.trim()) return;
|
||||
setAdding(true);
|
||||
try {
|
||||
await upsertAsync({
|
||||
client_id: newClientId.trim(),
|
||||
label: newLabel.trim() || null,
|
||||
profile_id: newProfileId || null,
|
||||
});
|
||||
setNewClientId('');
|
||||
setNewLabel('');
|
||||
setNewProfileId('');
|
||||
} finally {
|
||||
setAdding(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex gap-8 items-start max-w-5xl">
|
||||
<div className="flex-1 min-w-0 max-w-2xl space-y-8">
|
||||
<SettingSection
|
||||
title="Install into your agent"
|
||||
description="Voicebox exposes a local MCP server whenever the app is open. Paste one of these snippets into your agent's MCP config."
|
||||
>
|
||||
<SnippetRow
|
||||
title="HTTP (recommended)"
|
||||
description="For clients that speak HTTP MCP — Claude Code, Cursor, Windsurf, VS Code."
|
||||
snippet={JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
voicebox: {
|
||||
url: mcpUrl,
|
||||
headers: { 'X-Voicebox-Client-Id': 'claude-code' },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}
|
||||
/>
|
||||
<SnippetRow
|
||||
title="Claude Code one-liner"
|
||||
description="Registers via the Claude Code CLI."
|
||||
snippet={`claude mcp add voicebox --transport http --url ${mcpUrl} --header "X-Voicebox-Client-Id: claude-code"`}
|
||||
/>
|
||||
<SnippetRow
|
||||
title="Stdio (fallback)"
|
||||
description="For clients that only spawn stdio processes. The shim binary ships with the app."
|
||||
snippet={JSON.stringify(
|
||||
{
|
||||
mcpServers: {
|
||||
voicebox: {
|
||||
command:
|
||||
'/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp',
|
||||
env: { VOICEBOX_CLIENT_ID: 'claude-code' },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}
|
||||
/>
|
||||
</SettingSection>
|
||||
|
||||
<SettingSection
|
||||
title="Default voice"
|
||||
description="Used when an agent calls voicebox.speak without a specific profile and has no per-client binding."
|
||||
>
|
||||
<SettingRow
|
||||
title="Default playback voice"
|
||||
description="Shared with the Captures-tab 'Play as voice' dropdown — one default voice for passive playback."
|
||||
action={
|
||||
<select
|
||||
value={defaultProfileId}
|
||||
onChange={(e) =>
|
||||
updateCapture({
|
||||
default_playback_voice_id: e.target.value || null,
|
||||
})
|
||||
}
|
||||
className="h-8 px-2 rounded-md border bg-background text-sm min-w-[180px]"
|
||||
>
|
||||
<option value="">(none)</option>
|
||||
{(profiles ?? []).map((p) => (
|
||||
<option key={p.id} value={p.id}>
|
||||
{p.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
}
|
||||
/>
|
||||
</SettingSection>
|
||||
|
||||
<SettingSection
|
||||
title="Per-agent voice"
|
||||
description="Bind specific agents to specific voices so you can tell who's speaking without looking. The agent identifies itself by the X-Voicebox-Client-Id header (or VOICEBOX_CLIENT_ID env for stdio)."
|
||||
>
|
||||
{bindings.length === 0 ? (
|
||||
<p className="text-sm text-muted-foreground py-4 italic">
|
||||
No bindings yet. Add one below, then configure your MCP client to
|
||||
send the matching <code>X-Voicebox-Client-Id</code>.
|
||||
</p>
|
||||
) : (
|
||||
<div className="divide-y divide-border/60">
|
||||
{bindings.map((b) => (
|
||||
<div
|
||||
key={b.client_id}
|
||||
className="py-3 grid grid-cols-[1fr_auto_auto] gap-4 items-center"
|
||||
>
|
||||
<div className="min-w-0">
|
||||
<div className="font-medium text-sm truncate">
|
||||
{b.label || b.client_id}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground truncate">
|
||||
<code className="text-[11px]">{b.client_id}</code>
|
||||
{' · '}
|
||||
{b.last_seen_at ? (
|
||||
<span title={`Last seen ${b.last_seen_at}`}>
|
||||
<Plug className="inline h-3 w-3 text-emerald-500" />{' '}
|
||||
last seen {formatRelative(b.last_seen_at)}
|
||||
</span>
|
||||
) : (
|
||||
<span>never connected</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<select
|
||||
value={b.profile_id ?? ''}
|
||||
onChange={(e) =>
|
||||
upsertAsync({
|
||||
client_id: b.client_id,
|
||||
label: b.label,
|
||||
profile_id: e.target.value || null,
|
||||
})
|
||||
}
|
||||
className="h-8 px-2 rounded-md border bg-background text-sm min-w-[160px]"
|
||||
>
|
||||
<option value="">(default)</option>
|
||||
{(profiles ?? []).map((p) => (
|
||||
<option key={p.id} value={p.id}>
|
||||
{p.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<Button
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
onClick={() => remove(b.client_id)}
|
||||
aria-label={`Remove binding for ${b.client_id}`}
|
||||
>
|
||||
<Trash2 className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="pt-4 space-y-2">
|
||||
<div className="text-sm font-medium">Add a binding</div>
|
||||
<div className="grid grid-cols-[1fr_1fr_auto] gap-2">
|
||||
<input
|
||||
type="text"
|
||||
placeholder="client id (e.g. claude-code)"
|
||||
value={newClientId}
|
||||
onChange={(e) => setNewClientId(e.target.value)}
|
||||
className="h-9 px-3 rounded-md border bg-background text-sm"
|
||||
/>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="label (optional)"
|
||||
value={newLabel}
|
||||
onChange={(e) => setNewLabel(e.target.value)}
|
||||
className="h-9 px-3 rounded-md border bg-background text-sm"
|
||||
/>
|
||||
<select
|
||||
value={newProfileId}
|
||||
onChange={(e) => setNewProfileId(e.target.value)}
|
||||
className="h-9 px-2 rounded-md border bg-background text-sm min-w-[140px]"
|
||||
>
|
||||
<option value="">(default)</option>
|
||||
{(profiles ?? []).map((p) => (
|
||||
<option key={p.id} value={p.id}>
|
||||
{p.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={handleAdd}
|
||||
disabled={!newClientId.trim() || adding}
|
||||
>
|
||||
Add binding
|
||||
</Button>
|
||||
</div>
|
||||
</SettingSection>
|
||||
</div>
|
||||
|
||||
<aside className="hidden lg:block w-[280px] shrink-0 space-y-6 sticky top-0">
|
||||
<div className="space-y-2">
|
||||
<h3 className="text-sm font-semibold">About MCP</h3>
|
||||
<p className="text-sm text-muted-foreground leading-relaxed">
|
||||
Model Context Protocol lets your AI coding agent — Claude Code,
|
||||
Cursor, Windsurf — call Voicebox tools. Speak in a cloned voice,
|
||||
transcribe audio, browse captures.
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<h3 className="text-sm font-semibold">Available tools</h3>
|
||||
<ul className="text-sm text-muted-foreground space-y-1.5 leading-relaxed">
|
||||
<li>
|
||||
<code className="text-accent">voicebox.speak</code>
|
||||
<div>Speak text in a voice profile.</div>
|
||||
</li>
|
||||
<li>
|
||||
<code className="text-accent">voicebox.transcribe</code>
|
||||
<div>Whisper STT on a clip.</div>
|
||||
</li>
|
||||
<li>
|
||||
<code className="text-accent">voicebox.list_captures</code>
|
||||
<div>Recent dictations / recordings.</div>
|
||||
</li>
|
||||
<li>
|
||||
<code className="text-accent">voicebox.list_profiles</code>
|
||||
<div>Available voice profiles.</div>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Waypoints className="h-3.5 w-3.5 text-accent" />
|
||||
<span>
|
||||
Also exposed as <code>POST /speak</code> for shell scripts, ACP,
|
||||
A2A.
|
||||
</span>
|
||||
</div>
|
||||
</aside>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SnippetRow({
|
||||
title,
|
||||
description,
|
||||
snippet,
|
||||
}: {
|
||||
title: string;
|
||||
description: string;
|
||||
snippet: string;
|
||||
}) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const copy = async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(snippet);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1500);
|
||||
} catch {
|
||||
// ignore; user can still select-and-copy the pre content
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="py-3 space-y-2">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<div>
|
||||
<div className="text-sm font-medium">{title}</div>
|
||||
<div className="text-xs text-muted-foreground">{description}</div>
|
||||
</div>
|
||||
<Button size="sm" variant="outline" onClick={copy}>
|
||||
{copied ? (
|
||||
<>
|
||||
<Check className="h-3.5 w-3.5 mr-1.5" />
|
||||
Copied
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Copy className="h-3.5 w-3.5 mr-1.5" />
|
||||
Copy
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
<pre className="text-[11px] font-mono p-3 rounded-md bg-muted/50 overflow-x-auto whitespace-pre-wrap break-all">
|
||||
{snippet}
|
||||
</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function formatRelative(iso: string): string {
|
||||
const then = new Date(iso).getTime();
|
||||
const now = Date.now();
|
||||
const diff = Math.max(0, now - then);
|
||||
if (diff < 60_000) return 'just now';
|
||||
if (diff < 3600_000) return `${Math.floor(diff / 60_000)} min ago`;
|
||||
if (diff < 86400_000) return `${Math.floor(diff / 3600_000)} h ago`;
|
||||
return `${Math.floor(diff / 86400_000)} d ago`;
|
||||
}
|
||||
@@ -12,6 +12,7 @@ interface SettingsTab {
|
||||
| '/settings'
|
||||
| '/settings/generation'
|
||||
| '/settings/captures'
|
||||
| '/settings/mcp'
|
||||
| '/settings/gpu'
|
||||
| '/settings/logs'
|
||||
| '/settings/changelog'
|
||||
@@ -23,6 +24,7 @@ const tabs: SettingsTab[] = [
|
||||
{ labelKey: 'settings.tabs.general', path: '/settings' },
|
||||
{ labelKey: 'settings.tabs.generation', path: '/settings/generation' },
|
||||
{ label: 'Captures', path: '/settings/captures' },
|
||||
{ label: 'MCP', path: '/settings/mcp' },
|
||||
{ labelKey: 'settings.tabs.gpu', path: '/settings/gpu', tauriOnly: true },
|
||||
{ labelKey: 'settings.tabs.logs', path: '/settings/logs', tauriOnly: true },
|
||||
{ labelKey: 'settings.tabs.changelog', path: '/settings/changelog' },
|
||||
|
||||
@@ -45,6 +45,9 @@ import type {
|
||||
CaptureSource,
|
||||
GenerationSettings,
|
||||
GenerationSettingsUpdate,
|
||||
MCPClientBinding,
|
||||
MCPClientBindingListResponse,
|
||||
MCPClientBindingUpsert,
|
||||
} from './types';
|
||||
|
||||
function formatErrorDetail(detail: unknown, fallback: string): string {
|
||||
@@ -503,6 +506,27 @@ class ApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
// MCP bindings — per-MCP-client voice/engine/intent mapping.
|
||||
async listMCPBindings(): Promise<MCPClientBindingListResponse> {
|
||||
return this.request<MCPClientBindingListResponse>('/mcp/bindings');
|
||||
}
|
||||
|
||||
async upsertMCPBinding(
|
||||
data: MCPClientBindingUpsert,
|
||||
): Promise<MCPClientBinding> {
|
||||
return this.request<MCPClientBinding>('/mcp/bindings', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(data),
|
||||
});
|
||||
}
|
||||
|
||||
async deleteMCPBinding(clientId: string): Promise<{ deleted: string }> {
|
||||
return this.request<{ deleted: string }>(
|
||||
`/mcp/bindings/${encodeURIComponent(clientId)}`,
|
||||
{ method: 'DELETE' },
|
||||
);
|
||||
}
|
||||
|
||||
// Model Management
|
||||
async getModelStatus(): Promise<ModelStatusListResponse> {
|
||||
return this.request<ModelStatusListResponse>('/models/status');
|
||||
|
||||
@@ -208,6 +208,9 @@ export interface CaptureSettings {
|
||||
preserve_technical: boolean;
|
||||
allow_auto_paste: boolean;
|
||||
default_playback_voice_id: string | null;
|
||||
/** Whether the global keyboard hotkey is armed. Off by default — turning
|
||||
* this on triggers the macOS Input Monitoring TCC prompt. */
|
||||
hotkey_enabled: boolean;
|
||||
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr"]. */
|
||||
chord_push_to_talk_keys: string[];
|
||||
/** rdev::Key variant names. Defaults: ["MetaRight","AltGr","Space"]. */
|
||||
@@ -465,3 +468,28 @@ export interface ApplyEffectsRequest {
|
||||
label?: string;
|
||||
set_as_default?: boolean;
|
||||
}
|
||||
|
||||
/* ─── MCP ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
export interface MCPClientBinding {
|
||||
client_id: string;
|
||||
label: string | null;
|
||||
profile_id: string | null;
|
||||
default_engine: string | null;
|
||||
default_intent: 'respond' | 'rewrite' | 'compose' | null;
|
||||
last_seen_at: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface MCPClientBindingUpsert {
|
||||
client_id: string;
|
||||
label?: string | null;
|
||||
profile_id?: string | null;
|
||||
default_engine?: string | null;
|
||||
default_intent?: 'respond' | 'rewrite' | 'compose' | null;
|
||||
}
|
||||
|
||||
export interface MCPClientBindingListResponse {
|
||||
items: MCPClientBinding[];
|
||||
}
|
||||
|
||||
@@ -4,32 +4,41 @@ import { useCaptureSettings } from '@/lib/hooks/useSettings';
|
||||
import { usePlatform } from '@/platform/PlatformContext';
|
||||
|
||||
/**
|
||||
* Push the user's saved chord into the running Rust `HotkeyMonitor`.
|
||||
* The monitor boots with hard-coded right-hand defaults; this hook
|
||||
* replaces them as soon as capture_settings resolves and re-applies on
|
||||
* every subsequent change so chord edits land without a restart.
|
||||
* Spawn (or quiet) the global hotkey monitor based on the saved
|
||||
* `capture_settings.hotkey_enabled` flag, and keep its bindings in sync with
|
||||
* the user's chord choices.
|
||||
*
|
||||
* Call once from the main app shell — multiple call sites would just
|
||||
* fire redundant invokes, since the chord engine swap is the same value
|
||||
* either way.
|
||||
* Boot sequence:
|
||||
* - hotkey_enabled = false → call `disable_hotkey` (no-op if monitor was
|
||||
* never spawned). Crucially, we do *not* call `enable_hotkey`, so the
|
||||
* macOS Input Monitoring TCC prompt is never triggered for users who
|
||||
* haven't opted in.
|
||||
* - hotkey_enabled = true → call `enable_hotkey` with the saved chords.
|
||||
* This is the call that creates the CGEventTap and triggers the TCC
|
||||
* prompt on first opt-in.
|
||||
*
|
||||
* Call once from the main app shell.
|
||||
*/
|
||||
export function useChordSync() {
|
||||
const platform = usePlatform();
|
||||
const { settings } = useCaptureSettings();
|
||||
const enabled = settings?.hotkey_enabled;
|
||||
const pushKeys = settings?.chord_push_to_talk_keys;
|
||||
const toggleKeys = settings?.chord_toggle_to_talk_keys;
|
||||
|
||||
useEffect(() => {
|
||||
if (!platform.metadata.isTauri) return;
|
||||
if (!pushKeys || !toggleKeys) return;
|
||||
invoke('update_chord_bindings', {
|
||||
pushToTalk: pushKeys,
|
||||
toggleToTalk: toggleKeys,
|
||||
}).catch((err) => {
|
||||
console.warn('[chord-sync] failed to update bindings:', err);
|
||||
if (enabled === undefined || !pushKeys || !toggleKeys) return;
|
||||
const command = enabled ? 'enable_hotkey' : 'disable_hotkey';
|
||||
const args = enabled
|
||||
? { pushToTalk: pushKeys, toggleToTalk: toggleKeys }
|
||||
: {};
|
||||
invoke(command, args).catch((err) => {
|
||||
console.warn(`[chord-sync] ${command} failed:`, err);
|
||||
});
|
||||
}, [
|
||||
platform.metadata.isTauri,
|
||||
enabled,
|
||||
// Stringify so a referentially-new array with the same content
|
||||
// doesn't fire a redundant invoke on every settings refetch.
|
||||
pushKeys?.join(','),
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type {
|
||||
MCPClientBindingListResponse,
|
||||
MCPClientBindingUpsert,
|
||||
} from '@/lib/api/types';
|
||||
|
||||
const MCP_BINDINGS_KEY = ['settings', 'mcp', 'bindings'] as const;
|
||||
|
||||
/** Manage per-MCP-client voice bindings (Claude Code → Morgan, etc.). */
|
||||
export function useMCPBindings() {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const query = useQuery({
|
||||
queryKey: MCP_BINDINGS_KEY,
|
||||
queryFn: () => apiClient.listMCPBindings(),
|
||||
// Keep fresh while the Settings page is open — the ``last_seen_at``
|
||||
// timestamp is useful for confirming an install works, and we want it
|
||||
// to tick forward when a client connects.
|
||||
refetchInterval: 10_000,
|
||||
});
|
||||
|
||||
const upsertMutation = useMutation({
|
||||
mutationFn: (data: MCPClientBindingUpsert) =>
|
||||
apiClient.upsertMCPBinding(data),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||
},
|
||||
});
|
||||
|
||||
const deleteMutation = useMutation({
|
||||
mutationFn: (clientId: string) => apiClient.deleteMCPBinding(clientId),
|
||||
onMutate: async (clientId) => {
|
||||
await queryClient.cancelQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||
const prev =
|
||||
queryClient.getQueryData<MCPClientBindingListResponse>(MCP_BINDINGS_KEY);
|
||||
if (prev) {
|
||||
queryClient.setQueryData<MCPClientBindingListResponse>(
|
||||
MCP_BINDINGS_KEY,
|
||||
{ items: prev.items.filter((b) => b.client_id !== clientId) },
|
||||
);
|
||||
}
|
||||
return { prev };
|
||||
},
|
||||
onError: (_err, _id, ctx) => {
|
||||
if (ctx?.prev) queryClient.setQueryData(MCP_BINDINGS_KEY, ctx.prev);
|
||||
},
|
||||
onSettled: () => {
|
||||
queryClient.invalidateQueries({ queryKey: MCP_BINDINGS_KEY });
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
bindings: query.data?.items ?? [],
|
||||
isLoading: query.isLoading,
|
||||
upsert: upsertMutation.mutate,
|
||||
upsertAsync: upsertMutation.mutateAsync,
|
||||
remove: deleteMutation.mutate,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
/** Payload for a speak-start SSE event broadcast by the backend. */
|
||||
export interface ActiveSpeak {
|
||||
generationId: string;
|
||||
profileName: string;
|
||||
source: 'mcp' | 'rest' | string;
|
||||
clientId: string | null;
|
||||
startedAt: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribes to `/events/speak` and reports whichever agent-initiated
|
||||
* speak is currently producing audio. Returns ``null`` when nothing is
|
||||
* speaking.
|
||||
*
|
||||
* Multiple concurrent speaks are rare (the model can only really do one
|
||||
* at a time) and we don't bother stacking them — newest wins, the old
|
||||
* one's speak-end will clear when it fires.
|
||||
*/
|
||||
export function useSpeakEvents(): ActiveSpeak | null {
|
||||
const [active, setActive] = useState<ActiveSpeak | null>(null);
|
||||
const activeRef = useRef<ActiveSpeak | null>(null);
|
||||
activeRef.current = active;
|
||||
|
||||
// Keep a live timer so the pill's elapsed label advances smoothly
|
||||
// without re-opening the SSE stream.
|
||||
const [, setTick] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!active) return;
|
||||
const iv = window.setInterval(() => setTick((t) => t + 1), 250);
|
||||
return () => window.clearInterval(iv);
|
||||
}, [active]);
|
||||
|
||||
useEffect(() => {
|
||||
const baseUrl = useServerStore.getState().serverUrl;
|
||||
if (!baseUrl) return;
|
||||
|
||||
let cancelled = false;
|
||||
let source: EventSource | null = null;
|
||||
|
||||
const connect = () => {
|
||||
if (cancelled) return;
|
||||
source = new EventSource(`${baseUrl}/events/speak`);
|
||||
|
||||
source.addEventListener('speak-start', (event: MessageEvent) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
const now = Date.now();
|
||||
setActive({
|
||||
generationId: String(data.generation_id ?? ''),
|
||||
profileName: String(data.profile_name ?? ''),
|
||||
source: String(data.source ?? 'mcp'),
|
||||
clientId: data.client_id ?? null,
|
||||
startedAt: now,
|
||||
elapsedMs: 0,
|
||||
});
|
||||
} catch {
|
||||
// malformed payload — ignore, don't crash the stream
|
||||
}
|
||||
});
|
||||
|
||||
source.addEventListener('speak-end', (event: MessageEvent) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
const endedId = String(data.generation_id ?? '');
|
||||
// Only clear if this end matches the currently-active id; late
|
||||
// ends from previous sessions are ignored.
|
||||
if (activeRef.current?.generationId === endedId) setActive(null);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
|
||||
source.onerror = () => {
|
||||
// EventSource auto-reconnects, but if the browser gives up we
|
||||
// manually retry with backoff.
|
||||
source?.close();
|
||||
if (!cancelled) window.setTimeout(connect, 2000);
|
||||
};
|
||||
};
|
||||
|
||||
connect();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
source?.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!active) return null;
|
||||
return { ...active, elapsedMs: Date.now() - active.startedAt };
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { GeneralPage } from '@/components/ServerTab/GeneralPage';
|
||||
import { GenerationPage } from '@/components/ServerTab/GenerationPage';
|
||||
import { GpuPage } from '@/components/ServerTab/GpuPage';
|
||||
import { LogsPage } from '@/components/ServerTab/LogsPage';
|
||||
import { MCPPage } from '@/components/ServerTab/MCPPage';
|
||||
import { SettingsLayout } from '@/components/ServerTab/ServerTab';
|
||||
import { Sidebar } from '@/components/Sidebar';
|
||||
import { StoriesTab } from '@/components/StoriesTab/StoriesTab';
|
||||
@@ -159,6 +160,12 @@ const settingsCapturesRoute = createRoute({
|
||||
component: CapturesPage,
|
||||
});
|
||||
|
||||
const settingsMCPRoute = createRoute({
|
||||
getParentRoute: () => settingsRoute,
|
||||
path: '/mcp',
|
||||
component: MCPPage,
|
||||
});
|
||||
|
||||
const settingsGpuRoute = createRoute({
|
||||
getParentRoute: () => settingsRoute,
|
||||
path: '/gpu',
|
||||
@@ -204,6 +211,7 @@ const routeTree = rootRoute.addChildren([
|
||||
settingsGeneralRoute,
|
||||
settingsGenerationRoute,
|
||||
settingsCapturesRoute,
|
||||
settingsMCPRoute,
|
||||
settingsGpuRoute,
|
||||
settingsLogsRoute,
|
||||
settingsChangelogRoute,
|
||||
|
||||
+100
-81
@@ -4,6 +4,7 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -68,15 +69,36 @@ def safe_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application."""
|
||||
from .mcp_server.server import build_mcp_server
|
||||
from .mcp_server.context import ClientIdMiddleware
|
||||
|
||||
# Build the MCP app up-front so we can wire its lifespan into FastAPI's —
|
||||
# FastMCP's Streamable HTTP transport only works if its session manager
|
||||
# runs inside the parent ASGI lifespan.
|
||||
mcp = build_mcp_server()
|
||||
mcp_app = mcp.http_app(path="/", transport="http")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
await _run_startup(app)
|
||||
async with mcp_app.router.lifespan_context(app):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await _run_shutdown()
|
||||
|
||||
application = FastAPI(
|
||||
title="voicebox API",
|
||||
description="Production-quality Qwen3-TTS voice cloning API",
|
||||
version=__version__,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
_configure_cors(application)
|
||||
application.add_middleware(ClientIdMiddleware)
|
||||
register_routers(application)
|
||||
_register_lifecycle(application)
|
||||
application.mount("/mcp", mcp_app)
|
||||
logger.info("MCP: mounted at /mcp")
|
||||
_mount_frontend(application)
|
||||
|
||||
return application
|
||||
@@ -179,107 +201,104 @@ def _get_gpu_status() -> str:
|
||||
return "None (CPU only)"
|
||||
|
||||
|
||||
def _register_lifecycle(application: FastAPI) -> None:
|
||||
"""Attach startup and shutdown event handlers."""
|
||||
async def _run_startup(application: FastAPI) -> None:
|
||||
"""Database init, warnings, model-cache prep. Runs on lifespan entry."""
|
||||
import platform
|
||||
import sys
|
||||
|
||||
@application.on_event("startup")
|
||||
async def startup_event():
|
||||
import platform
|
||||
import sys
|
||||
logger.info("Voicebox v%s starting up", __version__)
|
||||
logger.info(
|
||||
"Python %s on %s %s (%s)",
|
||||
sys.version.split()[0],
|
||||
platform.system(),
|
||||
platform.release(),
|
||||
platform.machine(),
|
||||
)
|
||||
|
||||
logger.info("Voicebox v%s starting up", __version__)
|
||||
logger.info(
|
||||
"Python %s on %s %s (%s)",
|
||||
sys.version.split()[0],
|
||||
platform.system(),
|
||||
platform.release(),
|
||||
platform.machine(),
|
||||
)
|
||||
database.init_db()
|
||||
|
||||
database.init_db()
|
||||
from .database.session import _db_path
|
||||
|
||||
from .database.session import _db_path
|
||||
logger.info("Database: %s", _db_path)
|
||||
logger.info("Data directory: %s", config.get_data_dir())
|
||||
|
||||
logger.info("Database: %s", _db_path)
|
||||
logger.info("Data directory: %s", config.get_data_dir())
|
||||
init_queue()
|
||||
|
||||
init_queue()
|
||||
# Mark stale "generating" records as failed -- leftovers from a killed process
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
# Mark stale "generating" records as failed -- leftovers from a killed process
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
db = next(get_db())
|
||||
try:
|
||||
result = db.execute(
|
||||
sa_text(
|
||||
"UPDATE generations SET status = 'failed', "
|
||||
"error = 'Server was shut down during generation' "
|
||||
"WHERE status IN ('generating', 'loading_model')"
|
||||
)
|
||||
db = next(get_db())
|
||||
try:
|
||||
result = db.execute(
|
||||
sa_text(
|
||||
"UPDATE generations SET status = 'failed', "
|
||||
"error = 'Server was shut down during generation' "
|
||||
"WHERE status IN ('generating', 'loading_model')"
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
logger.info("Marked %d stale generation(s) as failed", result.rowcount)
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
logger.info("Marked %d stale generation(s) as failed", result.rowcount)
|
||||
|
||||
from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration
|
||||
from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration
|
||||
|
||||
profile_count = db.query(DBVoiceProfile).count()
|
||||
generation_count = db.query(DBGeneration).count()
|
||||
logger.info("Profiles: %d, Generations: %d", profile_count, generation_count)
|
||||
profile_count = db.query(DBVoiceProfile).count()
|
||||
generation_count = db.query(DBGeneration).count()
|
||||
logger.info("Profiles: %d, Generations: %d", profile_count, generation_count)
|
||||
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.warning("Could not clean up stale generations: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.warning("Could not clean up stale generations: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
backend_type = get_backend_type()
|
||||
logger.info("Backend: %s", backend_type.upper())
|
||||
logger.info("GPU: %s", _get_gpu_status())
|
||||
backend_type = get_backend_type()
|
||||
logger.info("Backend: %s", backend_type.upper())
|
||||
logger.info("GPU: %s", _get_gpu_status())
|
||||
|
||||
# Warn if GPU architecture is not supported by this PyTorch build
|
||||
from .backends.base import check_cuda_compatibility
|
||||
from .backends.base import check_cuda_compatibility
|
||||
|
||||
_compatible, _cuda_warning = check_cuda_compatibility()
|
||||
if not _compatible:
|
||||
logger.warning("GPU COMPATIBILITY: %s", _cuda_warning)
|
||||
_compatible, _cuda_warning = check_cuda_compatibility()
|
||||
if not _compatible:
|
||||
logger.warning("GPU COMPATIBILITY: %s", _cuda_warning)
|
||||
|
||||
from .services.cuda import check_and_update_cuda_binary
|
||||
from .services.cuda import check_and_update_cuda_binary
|
||||
|
||||
create_background_task(check_and_update_cuda_binary())
|
||||
create_background_task(check_and_update_cuda_binary())
|
||||
|
||||
try:
|
||||
progress_manager = get_progress_manager()
|
||||
progress_manager._set_main_loop(asyncio.get_running_loop())
|
||||
except Exception as e:
|
||||
logger.warning("Could not initialize progress manager event loop: %s", e)
|
||||
try:
|
||||
progress_manager = get_progress_manager()
|
||||
progress_manager._set_main_loop(asyncio.get_running_loop())
|
||||
except Exception as e:
|
||||
logger.warning("Could not initialize progress manager event loop: %s", e)
|
||||
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Model cache: %s", cache_dir)
|
||||
except Exception as e:
|
||||
logger.warning("Could not create HuggingFace cache directory: %s", e)
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Model cache: %s", cache_dir)
|
||||
except Exception as e:
|
||||
logger.warning("Could not create HuggingFace cache directory: %s", e)
|
||||
|
||||
logger.info("Ready")
|
||||
logger.info("Ready")
|
||||
|
||||
@application.on_event("shutdown")
|
||||
async def shutdown_event():
|
||||
logger.info("Voicebox server shutting down...")
|
||||
try:
|
||||
tts.unload_tts_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload TTS model")
|
||||
try:
|
||||
transcribe.unload_whisper_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload Whisper model")
|
||||
try:
|
||||
llm.unload_llm_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload LLM model")
|
||||
|
||||
async def _run_shutdown() -> None:
|
||||
"""Unload models on lifespan exit."""
|
||||
logger.info("Voicebox server shutting down...")
|
||||
try:
|
||||
tts.unload_tts_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload TTS model")
|
||||
try:
|
||||
transcribe.unload_whisper_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload Whisper model")
|
||||
try:
|
||||
llm.unload_llm_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload LLM model")
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
+120
-2
@@ -295,6 +295,28 @@ def build_server(cuda=False):
|
||||
"unidic_lite",
|
||||
"--hidden-import",
|
||||
"loguru",
|
||||
# MCP server — Streamable-HTTP endpoint and the 4 voicebox.* tools.
|
||||
# FastMCP pulls in a chain of deps (mcp, cyclopts, openapi-pydantic,
|
||||
# etc.) that don't auto-discover cleanly under PyInstaller, so we
|
||||
# collect them whole. Small compared to torch.
|
||||
"--hidden-import",
|
||||
"backend.mcp_server",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.server",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.tools",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.context",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.resolve",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.events",
|
||||
"--collect-all",
|
||||
"fastmcp",
|
||||
"--collect-all",
|
||||
"mcp",
|
||||
"--hidden-import",
|
||||
"sse_starlette",
|
||||
]
|
||||
)
|
||||
|
||||
@@ -447,12 +469,108 @@ def build_server(cuda=False):
|
||||
logger.info("Binary built in %s", backend_dir / "dist" / binary_name)
|
||||
|
||||
|
||||
def build_shim():
|
||||
"""Build the voicebox-mcp stdio shim as a tiny standalone binary.
|
||||
|
||||
This is the bridge for MCP clients that only speak stdio — it proxies
|
||||
JSON-RPC to the main voicebox-server's /mcp endpoint. Keep it small: no
|
||||
torch, no ML deps, just httpx + asyncio.
|
||||
"""
|
||||
backend_dir = Path(__file__).parent
|
||||
|
||||
args = [
|
||||
"mcp_shim/__main__.py",
|
||||
"--onefile",
|
||||
"--name",
|
||||
"voicebox-mcp",
|
||||
# Stdio-only — no console hiding needed on Windows since the parent
|
||||
# MCP client is spawning this as a child process and wants stdio.
|
||||
"--hidden-import",
|
||||
"backend.mcp_shim",
|
||||
"--hidden-import",
|
||||
"backend.mcp_shim.__main__",
|
||||
"--hidden-import",
|
||||
"httpx",
|
||||
"--hidden-import",
|
||||
"httpx._transports.default",
|
||||
"--hidden-import",
|
||||
"anyio",
|
||||
# Exclude everything heavy that httpx/asyncio don't actually need so
|
||||
# the binary stays tiny (~15 MB instead of ~400 MB).
|
||||
"--exclude-module",
|
||||
"torch",
|
||||
"--exclude-module",
|
||||
"transformers",
|
||||
"--exclude-module",
|
||||
"mlx",
|
||||
"--exclude-module",
|
||||
"mlx_audio",
|
||||
"--exclude-module",
|
||||
"qwen_tts",
|
||||
"--exclude-module",
|
||||
"chatterbox",
|
||||
"--exclude-module",
|
||||
"zipvoice",
|
||||
"--exclude-module",
|
||||
"tada",
|
||||
"--exclude-module",
|
||||
"kokoro",
|
||||
"--exclude-module",
|
||||
"misaki",
|
||||
"--exclude-module",
|
||||
"spacy",
|
||||
"--exclude-module",
|
||||
"librosa",
|
||||
"--exclude-module",
|
||||
"numba",
|
||||
"--exclude-module",
|
||||
"numpy",
|
||||
"--exclude-module",
|
||||
"pedalboard",
|
||||
"--exclude-module",
|
||||
"fastapi",
|
||||
"--exclude-module",
|
||||
"uvicorn",
|
||||
"--exclude-module",
|
||||
"sqlalchemy",
|
||||
"--exclude-module",
|
||||
"fastmcp",
|
||||
"--exclude-module",
|
||||
"mcp",
|
||||
]
|
||||
|
||||
dist_dir = str(backend_dir / "dist")
|
||||
build_dir = str(backend_dir / "build")
|
||||
args.extend(
|
||||
[
|
||||
"--distpath",
|
||||
dist_dir,
|
||||
"--workpath",
|
||||
build_dir,
|
||||
"--noconfirm",
|
||||
"--clean",
|
||||
]
|
||||
)
|
||||
|
||||
os.chdir(backend_dir)
|
||||
PyInstaller.__main__.run(args)
|
||||
logger.info("Shim built: %s", backend_dir / "dist" / "voicebox-mcp")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Build voicebox-server binary")
|
||||
parser = argparse.ArgumentParser(description="Build voicebox binaries")
|
||||
parser.add_argument(
|
||||
"--cuda",
|
||||
action="store_true",
|
||||
help="Build CUDA-enabled binary (voicebox-server-cuda)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shim",
|
||||
action="store_true",
|
||||
help="Build the voicebox-mcp stdio shim binary instead of the server",
|
||||
)
|
||||
cli_args = parser.parse_args()
|
||||
build_server(cuda=cli_args.cuda)
|
||||
if cli_args.shim:
|
||||
build_shim()
|
||||
else:
|
||||
build_server(cuda=cli_args.cuda)
|
||||
|
||||
@@ -15,6 +15,7 @@ from .models import (
|
||||
Generation,
|
||||
GenerationSettings,
|
||||
GenerationVersion,
|
||||
MCPClientBinding,
|
||||
ProfileChannelMapping,
|
||||
ProfileSample,
|
||||
Project,
|
||||
@@ -35,6 +36,7 @@ __all__ = [
|
||||
"Generation",
|
||||
"GenerationSettings",
|
||||
"GenerationVersion",
|
||||
"MCPClientBinding",
|
||||
"ProfileChannelMapping",
|
||||
"ProfileSample",
|
||||
"Project",
|
||||
|
||||
@@ -224,6 +224,13 @@ def _migrate_capture_settings(engine, inspector, tables: set[str]) -> None:
|
||||
"chord_toggle_to_talk_keys TEXT NOT NULL DEFAULT '[\"MetaRight\",\"AltGr\",\"Space\"]'",
|
||||
"chord_toggle_to_talk_keys",
|
||||
)
|
||||
if "hotkey_enabled" not in columns:
|
||||
_add_column(
|
||||
engine,
|
||||
"capture_settings",
|
||||
"hotkey_enabled BOOLEAN NOT NULL DEFAULT 0",
|
||||
"hotkey_enabled",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_storage_paths(engine, tables: set[str]) -> None:
|
||||
|
||||
@@ -196,6 +196,12 @@ class CaptureSettings(Base):
|
||||
preserve_technical = Column(Boolean, nullable=False, default=True)
|
||||
allow_auto_paste = Column(Boolean, nullable=False, default=True)
|
||||
default_playback_voice_id = Column(String, nullable=True)
|
||||
# Default OFF — opting in is what triggers the macOS Input Monitoring TCC
|
||||
# prompt. We deliberately don't spawn the global keyboard tap until the
|
||||
# user flips this on so a fresh-install user doesn't see a scary
|
||||
# "Voicebox would like to receive keystrokes from any application" dialog
|
||||
# before they've even opened the Captures tab.
|
||||
hotkey_enabled = Column(Boolean, nullable=False, default=False)
|
||||
# Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand
|
||||
# modifiers by default so they don't collide with left-hand system
|
||||
# shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit).
|
||||
@@ -221,6 +227,29 @@ class GenerationSettings(Base):
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class MCPClientBinding(Base):
|
||||
"""Per-MCP-client settings (voice profile, engine, intent).
|
||||
|
||||
Lets users bind distinct voices to distinct agents — e.g. Claude Code
|
||||
speaks in "Morgan," Cursor in "Scarlett." The MCP client identifies
|
||||
itself via the ``X-Voicebox-Client-Id`` HTTP header; direct-HTTP
|
||||
clients set it in their MCP config's ``headers`` block, the stdio
|
||||
shim forwards it from the ``VOICEBOX_CLIENT_ID`` env var.
|
||||
"""
|
||||
|
||||
__tablename__ = "mcp_client_bindings"
|
||||
|
||||
client_id = Column(String, primary_key=True)
|
||||
label = Column(String, nullable=True) # display name
|
||||
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
|
||||
default_engine = Column(String, nullable=True)
|
||||
# "respond" | "rewrite" | "compose" — null means plain TTS (no LLM transform).
|
||||
default_intent = Column(String, nullable=True)
|
||||
last_seen_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class Capture(Base):
|
||||
"""A single voice input capture (dictation, recording, or uploaded file).
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Voicebox MCP server
|
||||
|
||||
Local **Model Context Protocol** server — lets any MCP-aware agent
|
||||
(Claude Code, Cursor, Windsurf, VS Code MCP extensions, etc.) speak text
|
||||
in your cloned voices, transcribe audio, and browse captures.
|
||||
|
||||
The server runs inside the same `uvicorn` process as the rest of Voicebox
|
||||
and is mounted at `/mcp` (Streamable HTTP transport).
|
||||
|
||||
## Install into your agent
|
||||
|
||||
Preferred — direct HTTP:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": { "X-Voicebox-Client-Id": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Fallback — stdio shim (when the client doesn't speak HTTP MCP). The
|
||||
`voicebox-mcp` binary ships inside the Voicebox.app bundle:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Claude Code one-liner:
|
||||
|
||||
```
|
||||
claude mcp add voicebox \
|
||||
--transport http \
|
||||
--url http://127.0.0.1:17493/mcp \
|
||||
--header "X-Voicebox-Client-Id: claude-code"
|
||||
```
|
||||
|
||||
## Tools
|
||||
|
||||
| Name | Purpose |
|
||||
|---|---|
|
||||
| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. |
|
||||
| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. |
|
||||
| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. |
|
||||
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||
|
||||
All tools resolve voice profiles in this precedence:
|
||||
|
||||
1. Explicit `profile` arg (name or id — case-insensitive)
|
||||
2. Per-client binding keyed by `X-Voicebox-Client-Id`
|
||||
3. `capture_settings.default_playback_voice_id` (global default)
|
||||
|
||||
Bindings are managed via `GET|PUT /mcp/bindings` or in the app under
|
||||
Settings → MCP.
|
||||
|
||||
## Debug with MCP Inspector
|
||||
|
||||
```
|
||||
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||
```
|
||||
|
||||
Point it at the URL, hit "List tools," call `voicebox.list_profiles`
|
||||
first to confirm wiring, then `voicebox.speak` for end-to-end.
|
||||
|
||||
## Non-MCP REST surface
|
||||
|
||||
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||
don't speak MCP (shell scripts, ACP, A2A):
|
||||
|
||||
```
|
||||
curl -X POST http://127.0.0.1:17493/speak \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'X-Voicebox-Client-Id: claude-code' \
|
||||
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||
```
|
||||
|
||||
## Code layout
|
||||
|
||||
```
|
||||
backend/mcp_server/
|
||||
├── __init__.py # re-export mount_into
|
||||
├── server.py # build_mcp_server() + mount_into(app)
|
||||
├── tools.py # @mcp.tool() implementations
|
||||
├── context.py # ClientIdMiddleware + current_client_id ContextVar
|
||||
├── resolve.py # profile resolution precedence
|
||||
├── events.py # pub/sub queue for /events/speak pill SSE
|
||||
└── README.md # you are here
|
||||
|
||||
backend/mcp_shim/ # stdio ↔ Streamable-HTTP proxy (see its README)
|
||||
```
|
||||
|
||||
The package is **`mcp_server`**, not `mcp`, to avoid shadowing the
|
||||
installed `mcp` PyPI package that FastMCP imports internally.
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Model Context Protocol server — exposes Voicebox tools to local AI agents.
|
||||
|
||||
Mounts a FastMCP instance at /mcp on the main FastAPI app (Streamable HTTP).
|
||||
A bundled stdio shim (backend/mcp_shim) forwards JSON-RPC into the same
|
||||
endpoint for MCP clients that only speak stdio.
|
||||
"""
|
||||
|
||||
from .server import mount_into
|
||||
|
||||
__all__ = ["mount_into"]
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Per-request client identity for MCP calls.
|
||||
|
||||
MCP clients identify themselves via an ``X-Voicebox-Client-Id`` HTTP header
|
||||
(direct-HTTP clients set it in their MCP config; the stdio shim forwards it
|
||||
from the ``VOICEBOX_CLIENT_ID`` env var). Middleware copies the value into a
|
||||
ContextVar so tool implementations can read it without plumbing the request
|
||||
object through every service call.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from contextvars import ContextVar
|
||||
from datetime import datetime
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||
|
||||
# Tool handlers read this to apply per-client voice bindings.
|
||||
current_client_id: ContextVar[str | None] = ContextVar(
|
||||
"current_client_id", default=None
|
||||
)
|
||||
|
||||
|
||||
class ClientIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Copy X-Voicebox-Client-Id into a ContextVar and stamp last_seen_at.
|
||||
|
||||
Only stamps on MCP-endpoint requests (anything under ``/mcp``) so
|
||||
unrelated REST traffic with the header set won't advance the
|
||||
last-seen timestamp — the Settings UI uses that to show when each
|
||||
client was last heard from.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
super().__init__(app)
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
client_id = request.headers.get(CLIENT_ID_HEADER)
|
||||
token = current_client_id.set(client_id)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
finally:
|
||||
current_client_id.reset(token)
|
||||
|
||||
if client_id and request.url.path.startswith("/mcp"):
|
||||
_stamp_last_seen(client_id)
|
||||
return response
|
||||
|
||||
|
||||
def _stamp_last_seen(client_id: str) -> None:
|
||||
"""Update or create the MCPClientBinding row for this client_id."""
|
||||
try:
|
||||
from ..database import get_db
|
||||
from ..database.models import MCPClientBinding
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
db = next(get_db())
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
row = MCPClientBinding(client_id=client_id)
|
||||
db.add(row)
|
||||
row.last_seen_at = datetime.utcnow()
|
||||
db.commit()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not stamp last_seen_at for %s", client_id, exc_info=True
|
||||
)
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""In-memory pub/sub for speaking-pill SSE broadcasts.
|
||||
|
||||
MCP ``voicebox.speak`` calls and the REST ``POST /speak`` route publish
|
||||
start/end events that DictateWindow subscribes to via /events/speak, so the
|
||||
floating pill surfaces whenever an agent is speaking.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
|
||||
# Each subscriber gets its own queue. Bounded to drop oldest if a client lags.
|
||||
_subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
|
||||
|
||||
def subscribe() -> asyncio.Queue[dict[str, Any]]:
|
||||
"""Register a new subscriber; caller must call unsubscribe() when done."""
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=64)
|
||||
_subscribers.add(queue)
|
||||
return queue
|
||||
|
||||
|
||||
def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
_subscribers.discard(queue)
|
||||
|
||||
|
||||
def publish(kind: str, payload: dict[str, Any]) -> None:
|
||||
"""Fan out to all current subscribers. Non-blocking; drops on full queue."""
|
||||
event = {"kind": kind, **payload}
|
||||
for queue in list(_subscribers):
|
||||
try:
|
||||
queue.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
# Slow subscriber — skip rather than block publishers.
|
||||
pass
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Voice profile resolution for MCP tool calls.
|
||||
|
||||
Precedence:
|
||||
1. Explicit tool arg (profile name or id)
|
||||
2. Per-client MCPClientBinding.profile_id
|
||||
3. CaptureSettings.default_playback_voice_id (global default)
|
||||
4. None — caller raises a helpful error
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import VoiceProfile as DBVoiceProfile, get_db
|
||||
from ..database.models import CaptureSettings
|
||||
from ..services.profiles import get_profile_orm_by_name_or_id as _lookup_profile
|
||||
|
||||
|
||||
def resolve_profile(
|
||||
explicit: str | None,
|
||||
client_id: str | None,
|
||||
db: Session,
|
||||
) -> DBVoiceProfile | None:
|
||||
"""Apply the full precedence chain and return the profile ORM row (or None)."""
|
||||
if explicit:
|
||||
profile = _lookup_profile(explicit, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
# Explicit but not found — return None so the caller can report it.
|
||||
return None
|
||||
|
||||
if client_id:
|
||||
# Per-client binding. Imported lazily so this module stays importable
|
||||
# even before the migration adds the table on first boot.
|
||||
from ..database.models import MCPClientBinding # noqa: WPS433
|
||||
|
||||
binding = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if binding and binding.profile_id:
|
||||
profile = _lookup_profile(binding.profile_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
# Global default from capture settings.
|
||||
settings = db.query(CaptureSettings).filter(CaptureSettings.id == 1).first()
|
||||
if settings and settings.default_playback_voice_id:
|
||||
profile = _lookup_profile(settings.default_playback_voice_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def with_db() -> Session:
|
||||
"""Utility for tool handlers that aren't managed by FastAPI's Depends."""
|
||||
return next(get_db())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Construct the FastMCP server and mount it on the FastAPI app.
|
||||
|
||||
The MCP endpoint lives at ``/mcp`` (Streamable HTTP transport). Modern MCP
|
||||
clients (Claude Code, Cursor, Windsurf, VS Code MCP extensions) connect
|
||||
directly via URL; older stdio-only clients use the ``voicebox-mcp`` shim
|
||||
binary bundled with the desktop app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .context import ClientIdMiddleware
|
||||
from .tools import register_tools
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_mcp_server() -> FastMCP:
|
||||
"""Create the FastMCP instance with Voicebox tools registered."""
|
||||
mcp = FastMCP(
|
||||
name="voicebox",
|
||||
instructions=(
|
||||
"Voicebox is a local voice I/O layer. Use `voicebox.speak` to "
|
||||
"play text in a voice profile, `voicebox.transcribe` for "
|
||||
"audio→text, and the `list_*` tools to discover profiles and "
|
||||
"captures."
|
||||
),
|
||||
)
|
||||
register_tools(mcp)
|
||||
return mcp
|
||||
|
||||
|
||||
def mount_into(
|
||||
app: FastAPI,
|
||||
*,
|
||||
extra_startup: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
"""Attach the MCP app to ``app`` at ``/mcp`` and install the client-id middleware.
|
||||
|
||||
``extra_startup`` — if provided, runs during the FastAPI lifespan. This
|
||||
is the hook that lets ``app.py`` keep its existing startup/shutdown
|
||||
bodies while also driving FastMCP's session manager.
|
||||
"""
|
||||
mcp = build_mcp_server()
|
||||
mcp_app = mcp.http_app(path="/", transport="http")
|
||||
|
||||
# ClientIdMiddleware must run before FastMCP so the ContextVar is set
|
||||
# by the time tool handlers execute. Starlette composes middlewares
|
||||
# outermost-first, so adding here on the parent app is correct.
|
||||
app.add_middleware(ClientIdMiddleware)
|
||||
app.mount("/mcp", mcp_app)
|
||||
app.state.mcp_lifespan = mcp_app.router.lifespan_context
|
||||
logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", ""))
|
||||
|
||||
|
||||
def compose_lifespan(*lifespans):
|
||||
"""Combine multiple async context managers into a single FastAPI lifespan.
|
||||
|
||||
Used by ``create_app`` to run the existing Voicebox startup/shutdown
|
||||
together with FastMCP's session manager (which MUST run in the
|
||||
ASGI lifespan for Streamable HTTP to work).
|
||||
"""
|
||||
|
||||
@asynccontextmanager
|
||||
async def _combined(app):
|
||||
async with AsyncExitStack() as stack:
|
||||
for cm_factory in lifespans:
|
||||
cm = cm_factory(app) if callable(cm_factory) else cm_factory
|
||||
await stack.enter_async_context(cm)
|
||||
yield
|
||||
|
||||
return _combined
|
||||
@@ -0,0 +1,321 @@
|
||||
"""Voicebox MCP tool implementations.
|
||||
|
||||
Thin wrappers over existing services/routes. Tools are registered with dotted
|
||||
names (``voicebox.speak`` etc.) so they look natural in agent logs —
|
||||
the Python function name stays snake_case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64 as b64
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..services import captures as captures_service
|
||||
from ..services import profiles as profiles_service
|
||||
from . import events as mcp_events
|
||||
from .context import current_client_id
|
||||
from .resolve import resolve_profile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Absolute-path transcribes are bounded to keep a bad client from
|
||||
# asking us to ingest a 20 GB file.
|
||||
MAX_TRANSCRIBE_BYTES = 200 * 1024 * 1024 # 200 MB
|
||||
|
||||
|
||||
def register_tools(mcp: FastMCP) -> None:
|
||||
"""Attach all Voicebox tools to the given FastMCP instance."""
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.speak",
|
||||
description=(
|
||||
"Speak text in a Voicebox voice profile. Returns a generation id "
|
||||
"the caller can poll at /generate/{id}/status. Audio plays on the "
|
||||
"user's speakers and is saved to the Captures / History tab."
|
||||
),
|
||||
)
|
||||
async def voicebox_speak(
|
||||
text: str,
|
||||
profile: str | None = None,
|
||||
engine: str | None = None,
|
||||
intent: Literal["respond", "rewrite", "compose"] | None = None,
|
||||
language: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Speak ``text`` in a voice profile.
|
||||
|
||||
``profile`` accepts a voice profile name (e.g. "Morgan") or id. If
|
||||
omitted, the server looks up the per-client binding for the calling
|
||||
MCP client, then falls back to the global default voice.
|
||||
|
||||
``intent`` only matters for profiles that have a personality prompt —
|
||||
when set, the text is first transformed by the LLM (respond to it,
|
||||
rewrite it in character, or compose a fresh utterance). Leave unset
|
||||
for plain TTS.
|
||||
"""
|
||||
db = next(get_db())
|
||||
try:
|
||||
vp = resolve_profile(profile, current_client_id.get(), db)
|
||||
if vp is None:
|
||||
raise ValueError(
|
||||
"No voice profile resolved. Pass `profile=` with a "
|
||||
"voice profile name or id, or set a default voice in "
|
||||
"Voicebox → Settings → MCP."
|
||||
)
|
||||
|
||||
# Persona path if intent requested and personality present.
|
||||
if intent is not None and vp.personality:
|
||||
return await _speak_with_persona(
|
||||
profile_id=vp.id,
|
||||
profile_name=vp.name,
|
||||
text=text,
|
||||
engine=engine,
|
||||
intent=intent,
|
||||
language=language,
|
||||
db=db,
|
||||
)
|
||||
|
||||
return await _speak_plain(
|
||||
profile_id=vp.id,
|
||||
profile_name=vp.name,
|
||||
text=text,
|
||||
engine=engine,
|
||||
language=language,
|
||||
db=db,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.transcribe",
|
||||
description=(
|
||||
"Transcribe an audio clip to text using Voicebox's local Whisper. "
|
||||
"Pass exactly one of `audio_base64` (bytes as base64) or "
|
||||
"`audio_path` (absolute local file path)."
|
||||
),
|
||||
)
|
||||
async def voicebox_transcribe(
|
||||
audio_base64: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if bool(audio_base64) == bool(audio_path):
|
||||
raise ValueError(
|
||||
"Pass exactly one of `audio_base64` or `audio_path`."
|
||||
)
|
||||
|
||||
# Absolute-path mode: validate and transcribe in place.
|
||||
if audio_path is not None:
|
||||
path = Path(audio_path)
|
||||
if not path.is_absolute():
|
||||
raise ValueError("`audio_path` must be absolute.")
|
||||
if not path.is_file():
|
||||
raise ValueError(f"File not found: {audio_path}")
|
||||
if path.stat().st_size > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"File exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
return await _transcribe_file(path, language, model)
|
||||
|
||||
# Base64 mode: decode into a temp file, transcribe, clean up.
|
||||
try:
|
||||
raw = b64.b64decode(audio_base64, validate=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Invalid audio_base64: {exc}") from exc
|
||||
if len(raw) > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"Audio exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".wav", delete=False
|
||||
) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
return await _transcribe_file(tmp_path, language, model)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_captures",
|
||||
description=(
|
||||
"List recent voice captures (dictations, recordings, uploads) "
|
||||
"with their transcripts. Most-recent first."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_captures(
|
||||
limit: int = 20, offset: int = 0
|
||||
) -> dict[str, Any]:
|
||||
if not (1 <= limit <= 200):
|
||||
raise ValueError("`limit` must be between 1 and 200.")
|
||||
if offset < 0:
|
||||
raise ValueError("`offset` must be >= 0.")
|
||||
db = next(get_db())
|
||||
try:
|
||||
items, total = captures_service.list_captures(
|
||||
db, limit=limit, offset=offset
|
||||
)
|
||||
return {
|
||||
"captures": [
|
||||
item.model_dump(mode="json") for item in items
|
||||
],
|
||||
"total": total,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_profiles",
|
||||
description=(
|
||||
"List available voice profiles (both cloned voices and presets). "
|
||||
"Use the returned `name` with voicebox.speak(profile=...)."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_profiles() -> dict[str, Any]:
|
||||
db = next(get_db())
|
||||
try:
|
||||
profiles = await profiles_service.list_profiles(db)
|
||||
return {
|
||||
"profiles": [
|
||||
{
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"voice_type": p.voice_type,
|
||||
"language": p.language,
|
||||
"has_personality": bool(getattr(p, "personality", None)),
|
||||
}
|
||||
for p in profiles
|
||||
]
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ─── Speak helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _speak_plain(
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_name: str,
|
||||
text: str,
|
||||
engine: str | None,
|
||||
language: str | None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""Plain TTS path — mirrors POST /generate. No LLM transform."""
|
||||
from ..routes.generations import generate_speech
|
||||
|
||||
req = models.GenerationRequest(
|
||||
profile_id=profile_id,
|
||||
text=text,
|
||||
language=language or "en",
|
||||
engine=engine or "qwen",
|
||||
)
|
||||
generation = await generate_speech(req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
|
||||
async def _speak_with_persona(
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_name: str,
|
||||
text: str,
|
||||
engine: str | None,
|
||||
intent: str,
|
||||
language: str | None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""LLM-transformed path — reuses POST /profiles/{id}/speak."""
|
||||
from ..routes.profiles import speak_in_character
|
||||
|
||||
req = models.PersonalitySpeakRequest(
|
||||
text=text,
|
||||
persist=True,
|
||||
language=language,
|
||||
engine=engine,
|
||||
intent=intent,
|
||||
)
|
||||
generation = await speak_in_character(profile_id, req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
|
||||
def _speak_response(
|
||||
generation, profile_name: str, *, source: str
|
||||
) -> dict[str, Any]:
|
||||
"""Normalize a GenerationResponse into the MCP tool's return shape.
|
||||
|
||||
Also fires a speak-start event so the DictateWindow pill surfaces
|
||||
the agent's speech. Speak-end is fired from run_generation's
|
||||
completion hook.
|
||||
"""
|
||||
payload = generation.model_dump(mode="json") if hasattr(
|
||||
generation, "model_dump"
|
||||
) else dict(generation)
|
||||
generation_id = payload.get("id")
|
||||
mcp_events.publish(
|
||||
"speak-start",
|
||||
{
|
||||
"generation_id": generation_id,
|
||||
"profile_name": profile_name,
|
||||
"source": source,
|
||||
"client_id": current_client_id.get(),
|
||||
},
|
||||
)
|
||||
return {
|
||||
"generation_id": generation_id,
|
||||
"status": payload.get("status"),
|
||||
"profile": profile_name,
|
||||
"source": source,
|
||||
"poll_url": f"/generate/{generation_id}/status"
|
||||
if generation_id
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
# ─── Transcribe helper ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _transcribe_file(
|
||||
path: Path, language: str | None, model: str | None
|
||||
) -> dict[str, Any]:
|
||||
from ..backends import WHISPER_HF_REPOS
|
||||
from ..services import transcribe as transcribe_service
|
||||
from ..utils.audio import load_audio
|
||||
|
||||
whisper = transcribe_service.get_whisper_model()
|
||||
model_size = model or whisper.model_size
|
||||
valid = list(WHISPER_HF_REPOS.keys())
|
||||
if model_size not in valid:
|
||||
raise ValueError(
|
||||
f"Invalid STT model '{model_size}'. Must be one of: {', '.join(valid)}"
|
||||
)
|
||||
|
||||
# load_audio is sync; keep the event loop responsive.
|
||||
audio, sr = await asyncio.to_thread(load_audio, str(path))
|
||||
duration = len(audio) / sr
|
||||
|
||||
if (
|
||||
not whisper.is_loaded() or whisper.model_size != model_size
|
||||
) and not whisper._is_model_cached(model_size):
|
||||
raise ValueError(
|
||||
f"Whisper model '{model_size}' is not yet downloaded. Open "
|
||||
"Voicebox → Settings → Models to download it first."
|
||||
)
|
||||
|
||||
text = await whisper.transcribe(str(path), language, model_size)
|
||||
return {
|
||||
"text": text,
|
||||
"duration": duration,
|
||||
"language": language,
|
||||
"model": model_size,
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Stdio → Streamable HTTP bridge for the Voicebox MCP server.
|
||||
|
||||
Some MCP clients only know how to spawn a subprocess and talk to it over
|
||||
stdin/stdout (the "stdio" transport). This package is a ~150-line adapter:
|
||||
the client spawns us as ``voicebox-mcp``; we proxy every JSON-RPC frame
|
||||
to http://127.0.0.1:17493/mcp/ and stream responses back out.
|
||||
|
||||
All the real work (tools, models, inference) lives in the Voicebox server
|
||||
process — this package contains no business logic.
|
||||
"""
|
||||
@@ -0,0 +1,197 @@
|
||||
"""voicebox-mcp — stdio ↔ Streamable-HTTP MCP proxy.
|
||||
|
||||
Some MCP clients only speak stdio. They spawn this binary, we pipe each
|
||||
JSON-RPC message to ``http://127.0.0.1:<port>/mcp/``, and stream the
|
||||
server's response back. The Voicebox server does all the real work.
|
||||
|
||||
Environment variables:
|
||||
VOICEBOX_PORT Voicebox server port (default 17493).
|
||||
VOICEBOX_HOST Host (default 127.0.0.1).
|
||||
VOICEBOX_CLIENT_ID Forwarded as X-Voicebox-Client-Id on every request.
|
||||
|
||||
Stdout is JSON-RPC only. Diagnostics go to stderr.
|
||||
Exit 0 on clean EOF, 1 on transport error, 2 if backend never answers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||
SESSION_HEADER = "mcp-session-id"
|
||||
HEALTH_TIMEOUT_S = 30.0
|
||||
DEFAULT_PORT = 17493
|
||||
|
||||
|
||||
def _err(msg: str) -> None:
|
||||
print(f"voicebox-mcp: {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _base_url() -> tuple[str, str]:
|
||||
host = os.environ.get("VOICEBOX_HOST", "127.0.0.1")
|
||||
port = int(os.environ.get("VOICEBOX_PORT", str(DEFAULT_PORT)))
|
||||
return f"http://{host}:{port}/mcp/", f"http://{host}:{port}/health"
|
||||
|
||||
|
||||
async def _wait_for_backend(client: httpx.AsyncClient, health_url: str) -> bool:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + HEALTH_TIMEOUT_S
|
||||
while loop.time() < deadline:
|
||||
try:
|
||||
r = await client.get(health_url, timeout=2.0)
|
||||
if r.status_code == 200:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
async def _read_stdin_line() -> str | None:
|
||||
"""Async-read a single line from stdin. Returns None on EOF."""
|
||||
loop = asyncio.get_running_loop()
|
||||
line = await loop.run_in_executor(None, sys.stdin.readline)
|
||||
if not line:
|
||||
return None
|
||||
return line
|
||||
|
||||
|
||||
def _write_stdout(obj: Any) -> None:
|
||||
"""Write a JSON object to stdout as one line, flushed."""
|
||||
sys.stdout.write(json.dumps(obj, separators=(",", ":")))
|
||||
sys.stdout.write("\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
async def _handle_request(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
raw: str,
|
||||
headers: dict[str, str],
|
||||
session_id: list[str | None],
|
||||
) -> None:
|
||||
"""Forward one JSON-RPC payload to the server and relay the response."""
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
_err(f"invalid JSON on stdin: {exc}")
|
||||
return
|
||||
|
||||
req_headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
**headers,
|
||||
}
|
||||
if session_id[0]:
|
||||
req_headers[SESSION_HEADER] = session_id[0]
|
||||
|
||||
# Notifications (no "id") don't expect a response body. Server returns
|
||||
# 202 Accepted and we stay quiet.
|
||||
is_notification = isinstance(message, dict) and "id" not in message
|
||||
|
||||
async with client.stream(
|
||||
"POST", url, headers=req_headers, content=raw.encode("utf-8")
|
||||
) as response:
|
||||
# Capture session id on initialize.
|
||||
if session_id[0] is None:
|
||||
sid = response.headers.get(SESSION_HEADER)
|
||||
if sid:
|
||||
session_id[0] = sid
|
||||
|
||||
if response.status_code == 202:
|
||||
return # notification acknowledged
|
||||
if response.status_code >= 400:
|
||||
body = await response.aread()
|
||||
_err(
|
||||
f"server {response.status_code}: "
|
||||
f"{body.decode('utf-8', errors='replace')[:400]}"
|
||||
)
|
||||
if is_notification:
|
||||
return
|
||||
_write_stdout(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"error": {
|
||||
"code": -32000,
|
||||
"message": (
|
||||
f"Voicebox MCP proxy got HTTP {response.status_code}"
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
ctype = response.headers.get("content-type", "")
|
||||
if "text/event-stream" in ctype:
|
||||
# SSE frames: lines prefixed "data: ..." contain the JSON-RPC msg.
|
||||
async for line in response.aiter_lines():
|
||||
if line.startswith("data:"):
|
||||
payload = line[5:].strip()
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
_write_stdout(json.loads(payload))
|
||||
except json.JSONDecodeError:
|
||||
_err(f"malformed SSE payload: {payload[:200]}")
|
||||
else:
|
||||
body = await response.aread()
|
||||
try:
|
||||
_write_stdout(json.loads(body))
|
||||
except json.JSONDecodeError:
|
||||
_err(
|
||||
f"non-JSON response ({ctype}): "
|
||||
f"{body.decode('utf-8', errors='replace')[:200]}"
|
||||
)
|
||||
|
||||
|
||||
async def _run() -> int:
|
||||
url, health_url = _base_url()
|
||||
forward_headers: dict[str, str] = {}
|
||||
client_id = os.environ.get("VOICEBOX_CLIENT_ID")
|
||||
if client_id:
|
||||
forward_headers[CLIENT_ID_HEADER] = client_id
|
||||
|
||||
session_id: list[str | None] = [None]
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(300.0)) as client:
|
||||
if not await _wait_for_backend(client, health_url):
|
||||
_err(
|
||||
f"timed out waiting for Voicebox at {health_url} — is the app open?"
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
while True:
|
||||
line = await _read_stdin_line()
|
||||
if line is None:
|
||||
return 0
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
await _handle_request(
|
||||
client, url, line, forward_headers, session_id
|
||||
)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
return 0
|
||||
except Exception as exc:
|
||||
_err(f"proxy failed: {exc!r}")
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return asyncio.run(_run())
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -248,6 +248,7 @@ class CaptureSettingsResponse(BaseModel):
|
||||
preserve_technical: bool = True
|
||||
allow_auto_paste: bool = True
|
||||
default_playback_voice_id: Optional[str] = None
|
||||
hotkey_enabled: bool = False
|
||||
chord_push_to_talk_keys: List[str] = Field(default_factory=lambda: ["MetaRight", "AltGr"])
|
||||
chord_toggle_to_talk_keys: List[str] = Field(
|
||||
default_factory=lambda: ["MetaRight", "AltGr", "Space"]
|
||||
@@ -269,6 +270,7 @@ class CaptureSettingsUpdate(BaseModel):
|
||||
preserve_technical: Optional[bool] = None
|
||||
allow_auto_paste: Optional[bool] = None
|
||||
default_playback_voice_id: Optional[str] = None
|
||||
hotkey_enabled: Optional[bool] = None
|
||||
chord_push_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||
chord_toggle_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||
|
||||
@@ -294,6 +296,68 @@ class GenerationSettingsUpdate(BaseModel):
|
||||
autoplay_on_generate: Optional[bool] = None
|
||||
|
||||
|
||||
class MCPClientBindingResponse(BaseModel):
|
||||
"""Per-MCP-client voice binding — what voice / engine / intent the server
|
||||
should use when a given client_id calls voicebox.speak without args."""
|
||||
|
||||
client_id: str
|
||||
label: Optional[str] = None
|
||||
profile_id: Optional[str] = None
|
||||
default_engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
default_intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
last_seen_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class MCPClientBindingUpsert(BaseModel):
|
||||
"""Create or update a binding. Matched by ``client_id``."""
|
||||
|
||||
client_id: str = Field(..., min_length=1, max_length=64)
|
||||
label: Optional[str] = Field(None, max_length=128)
|
||||
profile_id: Optional[str] = None
|
||||
default_engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
default_intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
|
||||
|
||||
class MCPClientBindingListResponse(BaseModel):
|
||||
items: List[MCPClientBindingResponse]
|
||||
|
||||
|
||||
class SpeakRequest(BaseModel):
|
||||
"""Body for POST /speak — non-MCP REST surface that mirrors voicebox.speak."""
|
||||
|
||||
text: str = Field(..., min_length=1, max_length=10000)
|
||||
profile: Optional[str] = Field(
|
||||
None,
|
||||
description="Voice profile name or id. Falls back to per-client binding, then default.",
|
||||
)
|
||||
engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
language: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$",
|
||||
)
|
||||
|
||||
|
||||
class LLMGenerateRequest(BaseModel):
|
||||
"""Request model for LLM text generation."""
|
||||
|
||||
|
||||
@@ -62,6 +62,11 @@ pedalboard>=0.9.0
|
||||
# HTTP client (for CUDA backend download)
|
||||
httpx>=0.27.0
|
||||
|
||||
# MCP server (Model Context Protocol) — lets local AI agents call
|
||||
# voicebox.speak / .transcribe / .list_captures / .list_profiles
|
||||
fastmcp>=3.0,<4.0
|
||||
sse-starlette>=2.0
|
||||
|
||||
# Utilities
|
||||
python-multipart>=0.0.6
|
||||
Pillow>=10.0.0
|
||||
|
||||
@@ -20,6 +20,9 @@ def register_routers(app: FastAPI) -> None:
|
||||
from .settings import router as settings_router
|
||||
from .tasks import router as tasks_router
|
||||
from .cuda import router as cuda_router
|
||||
from .speak import router as speak_router
|
||||
from .mcp_bindings import router as mcp_bindings_router
|
||||
from .events import router as events_router
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(profiles_router)
|
||||
@@ -36,3 +39,6 @@ def register_routers(app: FastAPI) -> None:
|
||||
app.include_router(settings_router)
|
||||
app.include_router(tasks_router)
|
||||
app.include_router(cuda_router)
|
||||
app.include_router(speak_router)
|
||||
app.include_router(mcp_bindings_router)
|
||||
app.include_router(events_router)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Server-Sent-Event streams the frontend subscribes to.
|
||||
|
||||
``GET /events/speak`` — broadcasts ``speak-start`` / ``speak-end`` events
|
||||
whenever an agent-initiated speak (MCP tool or POST /speak) runs. The
|
||||
DictateWindow uses them to show the floating pill in a `speaking` state.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from ..mcp_server import events as mcp_events
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/events/speak")
|
||||
async def speak_events(request: Request):
|
||||
"""SSE stream of speak-start / speak-end events."""
|
||||
|
||||
async def event_stream():
|
||||
queue = mcp_events.subscribe()
|
||||
try:
|
||||
# Immediate hello so EventSource knows the connection is live.
|
||||
yield {"event": "ready", "data": "{}"}
|
||||
while True:
|
||||
if await request.is_disconnected():
|
||||
return
|
||||
try:
|
||||
event = await asyncio.wait_for(queue.get(), timeout=15.0)
|
||||
except asyncio.TimeoutError:
|
||||
# Heartbeat so proxies don't reap idle streams.
|
||||
yield {"event": "ping", "data": "{}"}
|
||||
continue
|
||||
kind = event.pop("kind", "message")
|
||||
yield {"event": kind, "data": json.dumps(event)}
|
||||
finally:
|
||||
mcp_events.unsubscribe(queue)
|
||||
|
||||
return EventSourceResponse(event_stream())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""REST endpoints for per-MCP-client voice binding settings.
|
||||
|
||||
The Settings UI uses these to let users configure distinct voices per
|
||||
agent (Claude Code in Morgan, Cursor in Scarlett, ...). The ``client_id``
|
||||
column is the same value the MCP client sends in ``X-Voicebox-Client-Id``
|
||||
(or the stdio shim pulls from ``VOICEBOX_CLIENT_ID``).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..database.models import MCPClientBinding
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/mcp/bindings",
|
||||
response_model=models.MCPClientBindingListResponse,
|
||||
)
|
||||
async def list_mcp_bindings(db: Session = Depends(get_db)):
|
||||
rows = (
|
||||
db.query(MCPClientBinding)
|
||||
.order_by(MCPClientBinding.client_id)
|
||||
.all()
|
||||
)
|
||||
return models.MCPClientBindingListResponse(
|
||||
items=[models.MCPClientBindingResponse.model_validate(r) for r in rows]
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/mcp/bindings",
|
||||
response_model=models.MCPClientBindingResponse,
|
||||
)
|
||||
async def upsert_mcp_binding(
|
||||
data: models.MCPClientBindingUpsert,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Create-or-update a binding. Matches by client_id."""
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == data.client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
row = MCPClientBinding(client_id=data.client_id)
|
||||
db.add(row)
|
||||
|
||||
row.label = data.label
|
||||
row.profile_id = data.profile_id
|
||||
row.default_engine = data.default_engine
|
||||
row.default_intent = data.default_intent
|
||||
row.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return models.MCPClientBindingResponse.model_validate(row)
|
||||
|
||||
|
||||
@router.delete("/mcp/bindings/{client_id}")
|
||||
async def delete_mcp_binding(
|
||||
client_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Binding not found")
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"deleted": client_id}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""POST /speak — REST wrapper around voicebox.speak for non-MCP callers.
|
||||
|
||||
Shell scripts, ACP, A2A, or any agent that doesn't speak MCP can hit this
|
||||
endpoint to play text through a cloned voice. Uses the same profile
|
||||
resolution and generation pipeline as the MCP tool, so per-client
|
||||
bindings (via X-Voicebox-Client-Id) work identically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..mcp_server import events as mcp_events
|
||||
from ..mcp_server.resolve import resolve_profile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/speak", response_model=models.GenerationResponse)
|
||||
async def speak(
|
||||
data: models.SpeakRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Speak text in a voice profile. Mirrors voicebox.speak (MCP).
|
||||
|
||||
Response shape matches POST /generate — a ``GenerationResponse`` with
|
||||
``status="generating"`` and an ``id`` the caller polls at
|
||||
``GET /generate/{id}/status``.
|
||||
"""
|
||||
client_id = request.headers.get("X-Voicebox-Client-Id")
|
||||
profile = resolve_profile(data.profile, client_id, db)
|
||||
if profile is None:
|
||||
if data.profile:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Voice profile '{data.profile}' not found.",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"No voice profile resolved. Pass `profile` (name or id), "
|
||||
"or configure a default in Voicebox → Settings → MCP."
|
||||
),
|
||||
)
|
||||
|
||||
# Persona path if intent requested AND profile has a personality prompt.
|
||||
if data.intent is not None and profile.personality:
|
||||
from .profiles import speak_in_character
|
||||
|
||||
generation = await speak_in_character(
|
||||
profile.id,
|
||||
models.PersonalitySpeakRequest(
|
||||
text=data.text,
|
||||
persist=True,
|
||||
language=data.language,
|
||||
engine=data.engine,
|
||||
intent=data.intent,
|
||||
),
|
||||
db,
|
||||
)
|
||||
else:
|
||||
# Plain TTS path — matches POST /generate.
|
||||
from .generations import generate_speech
|
||||
|
||||
generation = await generate_speech(
|
||||
models.GenerationRequest(
|
||||
profile_id=profile.id,
|
||||
text=data.text,
|
||||
language=data.language or "en",
|
||||
engine=data.engine or "qwen",
|
||||
),
|
||||
db,
|
||||
)
|
||||
|
||||
mcp_events.publish(
|
||||
"speak-start",
|
||||
{
|
||||
"generation_id": getattr(generation, "id", None),
|
||||
"profile_name": profile.name,
|
||||
"source": "rest",
|
||||
"client_id": client_id,
|
||||
},
|
||||
)
|
||||
return generation
|
||||
@@ -134,6 +134,7 @@ async def run_generation(
|
||||
db=bg_db,
|
||||
error="Generation cancelled",
|
||||
)
|
||||
_notify_speak_end(generation_id, status="cancelled")
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
await history.update_generation_status(
|
||||
@@ -142,11 +143,28 @@ async def run_generation(
|
||||
db=bg_db,
|
||||
error=str(e),
|
||||
)
|
||||
_notify_speak_end(generation_id, status="failed")
|
||||
else:
|
||||
_notify_speak_end(generation_id, status="completed")
|
||||
finally:
|
||||
task_manager.complete_generation(generation_id)
|
||||
bg_db.close()
|
||||
|
||||
|
||||
def _notify_speak_end(generation_id: str, *, status: str) -> None:
|
||||
"""Publish a speak-end event; the frontend ignores unknown ids."""
|
||||
try:
|
||||
from ..mcp_server import events as mcp_events
|
||||
|
||||
mcp_events.publish(
|
||||
"speak-end",
|
||||
{"generation_id": generation_id, "status": status},
|
||||
)
|
||||
except Exception:
|
||||
# Never let event pub/sub break generation completion.
|
||||
pass
|
||||
|
||||
|
||||
def _save_generate(
|
||||
*,
|
||||
generation_id: str,
|
||||
|
||||
@@ -275,6 +275,27 @@ async def get_profile(
|
||||
return _profile_to_response(profile)
|
||||
|
||||
|
||||
def get_profile_orm_by_name_or_id(
|
||||
name_or_id: str,
|
||||
db: Session,
|
||||
) -> DBVoiceProfile | None:
|
||||
"""Resolve a profile from a user-supplied string that may be either id or name.
|
||||
|
||||
Id is tried first (fast path, matches UUIDs). Name fallback is
|
||||
case-insensitive so agents can say "Morgan" regardless of casing.
|
||||
"""
|
||||
if not name_or_id:
|
||||
return None
|
||||
row = db.query(DBVoiceProfile).filter(DBVoiceProfile.id == name_or_id).first()
|
||||
if row is not None:
|
||||
return row
|
||||
return (
|
||||
db.query(DBVoiceProfile)
|
||||
.filter(func.lower(DBVoiceProfile.name) == name_or_id.lower())
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
async def get_profile_samples(
|
||||
profile_id: str,
|
||||
db: Session,
|
||||
|
||||
@@ -46,6 +46,8 @@ tmp_ret = collect_all('espeakng_loader')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('en_core_web_sm')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('unidic_lite')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('mlx')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('mlx_audio')
|
||||
|
||||
+25
-16
@@ -1,23 +1,30 @@
|
||||
---
|
||||
title: "Voicebox Documentation"
|
||||
description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs."
|
||||
description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine."
|
||||
---
|
||||
|
||||
Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor.
|
||||
Voicebox is the **open-source, local-first AI voice studio** — a free
|
||||
alternative to ElevenLabs and WisprFlow in one app. Clone voices, generate
|
||||
speech across 7 TTS engines, dictate into any app with a global hotkey,
|
||||
compose multi-voice projects, and let any MCP-aware agent speak in a voice
|
||||
you own. Everything runs on your hardware.
|
||||
|
||||
|
||||
|
||||
- **Complete privacy** -- models and voice data stay on your machine
|
||||
- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro
|
||||
- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or 50+ curated preset voices via Kokoro and Qwen CustomVoice
|
||||
- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more
|
||||
- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters
|
||||
- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
||||
- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters
|
||||
- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives
|
||||
- **API-first** -- REST API for integrating voice synthesis into your own projects
|
||||
- **Native performance** -- built with Tauri (Rust), not Electron
|
||||
- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker
|
||||
- **Dictation** — hold a chord anywhere on your machine, speak, release; the transcript pastes into the focused field
|
||||
- **Captures tab** — paired audio + transcript archive, retranscribe / refine / play-as-voice
|
||||
- **Voice personalities** — per-profile compose / rewrite / respond, powered by a local LLM
|
||||
- **Agents speak back** — any MCP-aware agent can call Voicebox to speak in one of your cloned voices
|
||||
- **7 TTS engines** — Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, Kokoro
|
||||
- **Cloning and preset voices** — zero-shot cloning or 50+ curated preset voices
|
||||
- **23 languages** — from English to Arabic, Japanese, Hindi, Swahili
|
||||
- **Post-processing effects** — pitch shift, reverb, delay, chorus, compression, filters
|
||||
- **Expressive speech** — paralinguistic tags (`[laugh]`, `[sigh]`) and natural-language delivery control
|
||||
- **Unlimited length** — auto-chunking with crossfade for long scripts
|
||||
- **Stories editor** — multi-track timeline for conversations, podcasts, narratives
|
||||
- **API-first** — REST + WebSocket API, MCP server for agent integrations
|
||||
- **Complete privacy** — models, audio, transcripts, LLM output never leave your machine
|
||||
- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux (ROCm / CPU), Intel Arc, Docker
|
||||
|
||||
## Download
|
||||
|
||||
@@ -32,6 +39,8 @@ Voicebox is a **local-first voice cloning studio** -- a free and open-source alt
|
||||
|
||||
## Get Started
|
||||
|
||||
- [Installation](/overview/installation) -- download and install Voicebox
|
||||
- [Quick Start](/overview/quick-start) -- get up and running in 5 minutes
|
||||
- [API Reference](/api-reference) -- integrate voice synthesis into your apps
|
||||
- [Installation](/overview/installation) — download and install Voicebox
|
||||
- [Quick Start](/overview/quick-start) — get up and running in 5 minutes
|
||||
- [Dictation](/overview/dictation) — start talking to your computer
|
||||
- [Voice Personalities](/overview/voice-personalities) — compose, rewrite, respond in any profile
|
||||
- [API Reference](/api-reference) — integrate voice synthesis into your apps
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
---
|
||||
title: "Captures"
|
||||
description: "The paired audio + transcript archive — every dictation, recording, and uploaded audio file shows up here, replayable and retranscribable."
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
A **capture** is an audio clip paired with its transcript. The Captures tab
|
||||
is where every dictation, manual recording, and uploaded audio file lands,
|
||||
with the original audio kept alongside the text so you can replay, re-run
|
||||
transcription with a different model, refine the transcript, or send the
|
||||
content somewhere else — including generating it back as speech in any of
|
||||
your voice profiles.
|
||||
|
||||
<Callout type="info">
|
||||
The Captures tab shipped in **0.5.0**, alongside global dictation and the
|
||||
per-profile personality modes. If you've used earlier versions, note that
|
||||
the Audio tab moved into **Settings → Audio Channels** to make room for
|
||||
this one.
|
||||
</Callout>
|
||||
|
||||
## Where captures come from
|
||||
|
||||
| Source | How it shows up | Badge |
|
||||
|---|---|---|
|
||||
| **Dictation** | Triggered by the global hotkey (see [Dictation](/overview/dictation)). Auto-refined by default. | `dictation` |
|
||||
| **In-app recording** | Recorded directly in the Captures tab using the built-in mic. | `recording` |
|
||||
| **File upload** | Any audio file dropped into the Captures tab — `.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`. | `file` |
|
||||
|
||||
All three paths share the same backend pipeline, the same model picker, and
|
||||
the same refinement flags. The source badge is there so you can visually
|
||||
scan a long list.
|
||||
|
||||
## List view
|
||||
|
||||
The main Captures view is a chronological list. Each row shows:
|
||||
|
||||
- The transcript (raw or refined — the refined version wins if present)
|
||||
- Duration + timestamp
|
||||
- Source badge
|
||||
- A play button for the original audio
|
||||
- A meatballs menu with per-row actions
|
||||
|
||||
Filtering and search are a Tier-2 ask — ping if you need them.
|
||||
|
||||
## Detail view
|
||||
|
||||
Clicking into a capture opens the detail view:
|
||||
|
||||
- **Waveform player** for the original audio
|
||||
- **Transcript editor** — click in and edit. Changes save on blur.
|
||||
- **Refined vs. raw toggle** if refinement ran on this capture
|
||||
- **Per-capture action bar** — retranscribe, refine, play as voice, delete
|
||||
- **Settings snapshot** — STT model used, refinement flags at the time
|
||||
this capture was processed, and the voice model if any was played
|
||||
|
||||
## Retranscribe
|
||||
|
||||
Runs the capture's original audio through a different Whisper model without
|
||||
re-uploading or re-refining anything. Useful when:
|
||||
|
||||
- The default model mis-heard something and you want to try a larger model
|
||||
- You used Base for a noisy clip and want to rerun with Turbo
|
||||
- A non-English clip needs an explicit language hint
|
||||
|
||||
**Settings → Captures → Transcription** controls the default model and
|
||||
language lock for new captures. Retranscribe uses those defaults unless you
|
||||
override them per capture.
|
||||
|
||||
## Refine
|
||||
|
||||
Runs the raw transcript through the local LLM to produce a cleaned-up
|
||||
version. The flags on the capture are snapshotted when refinement first
|
||||
runs, so you can re-refine later with different flags without losing the raw
|
||||
transcript:
|
||||
|
||||
| Flag | Effect |
|
||||
|---|---|
|
||||
| **Smart cleanup** | Remove fillers (`um`, `uh`, `like`), tidy punctuation and capitalization. |
|
||||
| **Remove self-corrections** | Keep the final version when the speaker backtracks ("actually, no, on Tuesday"). |
|
||||
| **Preserve technical terms** | Leave identifiers (`handleSubmit`, `npm install`) untouched. |
|
||||
|
||||
See the Refinement section of [Dictation](/overview/dictation#refinement) for
|
||||
how Voicebox strips Whisper loop hallucinations *before* the LLM sees the
|
||||
transcript — a capture can be re-refined any number of times without
|
||||
re-introducing "thanks for watching thanks for watching" echoes.
|
||||
|
||||
The refinement model picker (three bundled Qwen3 sizes) lives in
|
||||
**Settings → Captures → Refinement**.
|
||||
|
||||
## Play as voice
|
||||
|
||||
This is the capability no one else in the dictation category ships: take any
|
||||
capture and play it back as speech in any of your voice profiles. One
|
||||
dropdown over every profile, one click, and the capture's text runs through
|
||||
`/generate` with the selected voice.
|
||||
|
||||
Use cases:
|
||||
|
||||
- Hear your own dictation back in a cloned voice of someone you like
|
||||
- Send a message you dictated as an audio reply in a specific character
|
||||
- Quickly prototype a line for a story without retyping
|
||||
|
||||
Playback uses whatever engine the selected profile is bound to — the same
|
||||
rules as the Generate tab. There's no LLM in this path; the transcript goes
|
||||
through unchanged. If you want the agent-style "transform the content before
|
||||
speaking" flow, that's what the
|
||||
[personality modes](/overview/voice-personalities) do — and the same
|
||||
primitive is exposed to MCP-aware agents via the
|
||||
[MCP Server](/overview/mcp-server) so Claude Code, Cursor, or Cline can speak
|
||||
in one of your voices on their own.
|
||||
|
||||
<Callout type="info">
|
||||
The default voice for the Captures tab's Play-as action is set in
|
||||
**Settings → Captures → Playback → Default voice**. You can still override
|
||||
it per capture.
|
||||
</Callout>
|
||||
|
||||
## Send-to menu
|
||||
|
||||
Each capture has a Send-to menu for moving its content into other parts of
|
||||
Voicebox:
|
||||
|
||||
- **Copy transcript** — to clipboard
|
||||
- **Use as voice sample…** — promote this capture to a sample on a voice
|
||||
profile of your choice. Opens a profile picker (with "+ New voice" for
|
||||
cold starts) and a reference-text confirm dialog, because cloning needs
|
||||
the `reference_text` to match the audio verbatim. Edit as needed and
|
||||
save — the capture stays in the Captures tab untouched; the sample is a
|
||||
copy, not a move.
|
||||
|
||||
## Storage and retention
|
||||
|
||||
**Settings → Captures → Storage** controls how long captures live on disk:
|
||||
|
||||
| Setting | Effect |
|
||||
|---|---|
|
||||
| **Retention: forever** | Never auto-delete. Default. |
|
||||
| **Retention: 90 days / 30 days / 7 days** | Captures older than the window are pruned on app start. |
|
||||
| **Clear all captures** | One-click nuke of every capture and its audio on disk. No undo. |
|
||||
|
||||
The original audio is always kept alongside the transcript — archival is on
|
||||
by default. Every capture's audio file and metadata row can be re-processed
|
||||
(retranscribe, refine, Play-as) as long as the audio file still exists.
|
||||
|
||||
## Short-recording guard
|
||||
|
||||
Audio clips under **300 ms** are short-circuited client-side and never
|
||||
uploaded. This prevents a fumbled chord tap from landing an empty capture.
|
||||
The threshold is tuned to filter accidents without cutting off intentional
|
||||
short dictations.
|
||||
|
||||
## Keyboard shortcuts
|
||||
|
||||
Inside the Captures tab:
|
||||
|
||||
| Keys | Action |
|
||||
|---|---|
|
||||
| `Space` | Play / pause the selected capture |
|
||||
| `↑` / `↓` | Previous / next capture in the list |
|
||||
| `Enter` | Open the selected capture in detail view |
|
||||
| `⌘ / Ctrl` + `C` (in detail view) | Copy the transcript |
|
||||
|
||||
## API surface
|
||||
|
||||
The Captures tab is backed by a small set of REST endpoints:
|
||||
|
||||
| Method | Endpoint | Use |
|
||||
|---|---|---|
|
||||
| `POST` | `/captures` | Upload audio + start the pipeline (STT, optional refinement, archival). |
|
||||
| `GET` | `/captures` | List captures. |
|
||||
| `GET` | `/captures/{id}` | Fetch one capture. |
|
||||
| `POST` | `/captures/{id}/retranscribe` | Rerun STT with a chosen model. |
|
||||
| `POST` | `/captures/{id}/refine` | Rerun refinement with chosen flags. |
|
||||
| `POST` | `/profiles/{id}/samples/from-capture/{capture_id}` | Promote a capture to a voice profile sample. |
|
||||
|
||||
These endpoints are stable and usable from your own scripts — see
|
||||
[Remote Mode](/overview/remote-mode) for running Voicebox as a server the rest
|
||||
of your machine can talk to.
|
||||
|
||||
## Next steps
|
||||
|
||||
<Cards>
|
||||
<Card title="Dictation" href="/overview/dictation">
|
||||
The global hotkey flow that feeds most captures.
|
||||
</Card>
|
||||
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||
Per-profile compose / rewrite / respond modes for captures you want to
|
||||
transform, not just transcribe.
|
||||
</Card>
|
||||
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||
Promote a capture into a voice sample on a profile.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -0,0 +1,208 @@
|
||||
---
|
||||
title: "Dictation"
|
||||
description: "Hold a key anywhere on your machine, speak, release — the transcript lands in whatever text field you had focused."
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Dictation lets you turn speech into clean text anywhere on your computer. Hold
|
||||
a chord, talk, release — Voicebox transcribes what you said with Whisper,
|
||||
optionally cleans it up with a local LLM, and pastes the result into the text
|
||||
field you had focused when you started.
|
||||
|
||||
Everything happens on your hardware. No cloud, no accounts, no audio leaving
|
||||
the machine.
|
||||
|
||||
<Callout type="info">
|
||||
Dictation was introduced in **0.5.0** alongside the Captures tab and the
|
||||
per-profile personality modes. It's the "input" half of Voicebox's voice I/O
|
||||
loop — cloning and TTS are still the "output" half.
|
||||
</Callout>
|
||||
|
||||
## The flow
|
||||
|
||||
<Steps>
|
||||
<Step title="Hold the chord">
|
||||
Hold the push-to-talk chord anywhere on your machine. A small pill fades
|
||||
in over your current app.
|
||||
</Step>
|
||||
<Step title="Speak">
|
||||
The pill shows `Recording` with a live waveform and an elapsed-time
|
||||
counter. Speak naturally — you don't have to wait for anything.
|
||||
</Step>
|
||||
<Step title="Release">
|
||||
On release, the pill flips to `Transcribing`, then `Refining` if
|
||||
auto-refine is on, then disappears.
|
||||
</Step>
|
||||
<Step title="Text lands in your app">
|
||||
If auto-paste is enabled and Voicebox has Accessibility permission, the
|
||||
transcript pastes into the text field you had focused when you started
|
||||
talking — not wherever focus drifted while you were speaking.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
Either way, every capture also appears in the **Captures tab** with the
|
||||
original audio and the transcript paired together. See
|
||||
[Captures](/overview/captures) for what you can do with them after the fact.
|
||||
|
||||
## Push-to-talk and toggle modes
|
||||
|
||||
Voicebox ships two chord behaviors out of the box:
|
||||
|
||||
| Mode | Default (macOS) | Default (Windows) | Behavior |
|
||||
|---|---|---|---|
|
||||
| **Push-to-talk** | Right `⌘` + Right `⌥` | Right `Ctrl` + Right `Shift` | Recording stops when you release the chord. |
|
||||
| **Toggle-to-talk** | Push-to-talk + `Space` | Push-to-talk + `Space` | Recording keeps going until you tap the chord again. |
|
||||
|
||||
**Holding PTT and tapping `Space` mid-hold upgrades a hold into a toggled
|
||||
session** without a gap in the audio. This is the single most useful detail of
|
||||
the chord system — short bursts feel fast, long-form narration feels
|
||||
hands-free, and there's no decision up front about which mode you wanted.
|
||||
|
||||
## The on-screen pill
|
||||
|
||||
While you're dictating, a floating pill appears over the current app. It walks
|
||||
through the states of the capture cycle and shows live signals for each:
|
||||
|
||||
| State | What it shows |
|
||||
|---|---|
|
||||
| `Recording` | Live waveform + elapsed time. |
|
||||
| `Transcribing` | Thinking waveform while Whisper runs. |
|
||||
| `Refining` | Same thinking waveform while the LLM cleans up the transcript (only if auto-refine is on). |
|
||||
| Error | Red tint. Click the pill to copy the error to your clipboard. Auto-dismisses. |
|
||||
|
||||
The pill is transparent, always-on-top, and pre-created hidden at app start —
|
||||
so it appears instantly when you hit the chord, with no window flash.
|
||||
|
||||
## Customizing the chord
|
||||
|
||||
Open **Settings → Captures → Dictation** to change either chord.
|
||||
|
||||
- **Left vs right modifier badges.** When you hold keys into the chord
|
||||
picker, Voicebox records whether each modifier is the left or right variant.
|
||||
That means you can bind to just the right `⌥` while leaving the left `⌥`
|
||||
alone — useful if you want dictation on one hand and keep your
|
||||
other-hand shortcuts intact.
|
||||
- **Chord defaults are picked to stay out of your way.** On macOS, the
|
||||
defaults deliberately avoid left-hand `Cmd+Option` chords so
|
||||
`Cmd+Option+I` (devtools), `Cmd+Option+Esc` (force quit), and
|
||||
`Cmd+Option+Space` (Spotlight) all remain yours. On Windows, the defaults
|
||||
route around AltGr collisions on German / French / Spanish layouts where
|
||||
`Ctrl+Alt` synthesizes AltGr.
|
||||
- **Live reload.** Changing a chord in Settings takes effect immediately —
|
||||
no restart, no tab reload.
|
||||
|
||||
## Auto-paste into the focused app
|
||||
|
||||
Once transcription finishes, Voicebox can synthesize a native paste into
|
||||
whatever text field had focus when you started the chord. Your clipboard is
|
||||
saved before and restored after, so nothing you had copied goes missing.
|
||||
|
||||
| Platform | Mechanism |
|
||||
|---|---|
|
||||
| macOS | `CGEventPost` at the HID tap with a full `⌘V` key sequence, preceded by reactivating the original app via `NSRunningApplication`. |
|
||||
| Windows | `SendInput` with correct scan codes, plus a `SetForegroundWindow` + `AttachThreadInput` handshake to defeat foreground-lock when pasting into a window that wasn't frontmost at chord-start. |
|
||||
|
||||
**Focus is snapshotted at chord-start.** The paste targets the original field
|
||||
even if focus drifts during transcribe / refine — that's the "pastes where you
|
||||
were talking *from*, not where you're looking *now*" behavior.
|
||||
|
||||
<Callout type="info">
|
||||
Auto-paste is optional. If Accessibility permission isn't granted (macOS),
|
||||
or you prefer to keep synthetic input off, dictation still runs — transcripts
|
||||
land in the Captures tab and you can copy them manually. The setting lives
|
||||
inline next to the Accessibility prompt in Settings → Captures → Dictation,
|
||||
not as a global banner.
|
||||
</Callout>
|
||||
|
||||
## Refinement
|
||||
|
||||
If auto-refine is on, a local LLM cleans up the raw Whisper transcript
|
||||
before it's pasted. The goal is to remove verbal clutter without rewriting
|
||||
what you actually said.
|
||||
|
||||
What refinement typically fixes:
|
||||
|
||||
- Filler words (`um`, `uh`, `like` used as pauses, `you know`)
|
||||
- Self-corrections — the LLM keeps the final version and drops earlier
|
||||
attempts (`could you uh run the migration real quick, and then, yeah,
|
||||
check the logs` → `Could you run the migration, then check the logs?`)
|
||||
- Basic punctuation and capitalization
|
||||
- Whisper loop hallucinations — Voicebox strips repeated tokens (six or
|
||||
more identical tokens in a row, case-insensitive) *before* the LLM
|
||||
sees the transcript, so a small refinement model can't echo them back
|
||||
|
||||
What refinement deliberately preserves:
|
||||
|
||||
- Technical terms and code identifiers (`npm install`, `handleSubmit`)
|
||||
- Legitimate repetition (`no, no, no, no, no` has fewer than six identical
|
||||
tokens, so it survives)
|
||||
- Your intent — refinement is cleanup, not rewriting
|
||||
|
||||
Flags are snapshotted per capture, so you can re-refine the same raw
|
||||
transcript later with different flags without losing the original. The
|
||||
refinement model picker (**Settings → Captures → Refinement**) offers three
|
||||
bundled Qwen3 sizes:
|
||||
|
||||
| Model | Size | Best for |
|
||||
|---|---|---|
|
||||
| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual dictation. |
|
||||
| Qwen3 1.7B | ~1.1 GB | Sweet spot when transcripts contain code identifiers. |
|
||||
| Qwen3 4B | ~2.5 GB | Full quality, slowest. |
|
||||
|
||||
This is the same local LLM used by the per-profile personality modes — one
|
||||
LLM in the app, not two. See [Voice Personalities](/overview/voice-personalities).
|
||||
|
||||
## Platform notes
|
||||
|
||||
### macOS
|
||||
|
||||
- **Accessibility permission** is required for auto-paste. The prompt lives
|
||||
inline next to the toggle in **Settings → Captures → Dictation**, with a
|
||||
deep link to **System Settings → Privacy & Security → Accessibility**.
|
||||
- **TSM crash mitigation.** The global hotkey listener runs on a background
|
||||
thread with `set_is_main_thread(false)` to sidestep a known
|
||||
macOS 14+ crash in the `rdev` library. If you hit an unexpected dictation
|
||||
failure on macOS, check the logs for TSM-related messages.
|
||||
|
||||
### Windows
|
||||
|
||||
- **UAC / UIPI caveat.** Synthetic paste into an *elevated* window from a
|
||||
non-elevated Voicebox is blocked by Windows itself. Run Voicebox elevated
|
||||
if you regularly dictate into elevated apps (e.g. an elevated terminal or
|
||||
Task Manager).
|
||||
- **Right-hand default chord** (`Ctrl+Shift`) avoids AltGr collisions on
|
||||
keyboard layouts where `Ctrl+Alt` is the compose key (German, French,
|
||||
Spanish, some others).
|
||||
|
||||
### Linux
|
||||
|
||||
- **Not yet in this release.** The Rust shim ships the macOS and Windows
|
||||
paths in 0.5.0. Linux `uinput` / AT-SPI support and the Wayland paste
|
||||
story are tracked in `docs/plans/VOICE_IO.md`.
|
||||
|
||||
## When auto-paste skips itself
|
||||
|
||||
A few cases where Voicebox deliberately does *not* synthesize a paste:
|
||||
|
||||
- **Focus was inside Voicebox** when the chord started. The transcript goes
|
||||
to the Captures tab so a dictation-into-Voicebox round-trip doesn't
|
||||
accidentally paste into the generate box.
|
||||
- **No text focus detected.** The transcript still lands in the Captures
|
||||
tab; copy it from there with one click.
|
||||
- **Accessibility permission not granted** on macOS. Same — Captures tab
|
||||
only.
|
||||
|
||||
## Next steps
|
||||
|
||||
<Cards>
|
||||
<Card title="Captures" href="/overview/captures">
|
||||
The paired audio + transcript archive every dictation lands in.
|
||||
</Card>
|
||||
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||
The same local LLM doubles as per-profile compose / rewrite / respond.
|
||||
</Card>
|
||||
<Card title="Transcription" href="/developer/transcription">
|
||||
Developer-level details on Whisper, Whisper Turbo, and the STT backend.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -1,23 +1,48 @@
|
||||
---
|
||||
title: "Introduction"
|
||||
description: "Voicebox is a local-first voice cloning studio -- a free and open-source alternative to ElevenLabs."
|
||||
description: "Voicebox is the open-source, local-first AI voice studio — a free alternative to ElevenLabs and WisprFlow, running entirely on your machine."
|
||||
---
|
||||
|
||||
## What is Voicebox?
|
||||
|
||||
Voicebox is a **local-first voice cloning studio** -- a free and open-source alternative to ElevenLabs. Clone voices from a few seconds of audio or pick from 50+ preset voices, generate speech in 23 languages across 7 TTS engines, apply post-processing effects, and compose multi-voice projects with a timeline editor.
|
||||
Voicebox is the **open-source, local-first AI voice studio**. It closes the
|
||||
voice I/O loop in both directions on one machine, with no cloud and no
|
||||
accounts:
|
||||
|
||||
- **Complete privacy** -- models and voice data stay on your machine
|
||||
- **7 TTS engines** -- Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox Multilingual, Chatterbox Turbo, HumeAI TADA, and Kokoro
|
||||
- **Cloning and preset voices** -- zero-shot cloning from a reference sample, or curated preset voices via Kokoro (50 voices) and Qwen CustomVoice (9 voices)
|
||||
- **23 languages** -- from English to Arabic, Japanese, Hindi, Swahili, and more
|
||||
- **Post-processing effects** -- pitch shift, reverb, delay, chorus, compression, and filters
|
||||
- **Expressive speech** -- paralinguistic tags like `[laugh]`, `[sigh]`, `[gasp]` via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
||||
- **Unlimited length** -- auto-chunking with crossfade for scripts, articles, and chapters
|
||||
- **Stories editor** -- multi-track timeline for conversations, podcasts, and narratives
|
||||
- **API-first** -- REST API for integrating voice synthesis into your own projects
|
||||
- **Native performance** -- built with Tauri (Rust), not Electron
|
||||
- **Runs everywhere** -- macOS (MLX/Metal), Windows (CUDA), Linux, AMD ROCm, Intel Arc, Docker
|
||||
- **Humans talk** — hold a chord anywhere on your machine and your
|
||||
dictation lands as clean text in whatever text field you had focused
|
||||
- **Agents talk back** — any MCP-aware agent can call Voicebox to speak in
|
||||
one of your cloned voices
|
||||
- **Voices speak for themselves** — voice profiles can carry a personality
|
||||
that compose, rewrite, or respond to text before it's spoken
|
||||
|
||||
It's the free, local alternative to both ElevenLabs (voice cloning and TTS)
|
||||
and WisprFlow (voice dictation for agents and power users) — covering both
|
||||
sides of the same loop in one app, with a single model directory and LLM
|
||||
shared between input and output.
|
||||
|
||||
## What's in the app
|
||||
|
||||
- **Dictation** — global hotkey, push-to-talk and toggle modes, auto-paste
|
||||
into the focused field on macOS and Windows (see [Dictation](/overview/dictation))
|
||||
- **Captures tab** — paired audio + transcript archive, retranscribe,
|
||||
refine, play-as-voice, promote-to-sample (see [Captures](/overview/captures))
|
||||
- **Voice cloning** — 5 cloning engines covering 23 languages. Zero-shot
|
||||
cloning from a reference sample (see [Voice Cloning](/overview/voice-cloning))
|
||||
- **Preset voices** — 50+ curated voices via Kokoro and Qwen CustomVoice
|
||||
for when you don't want to clone (see [Preset Voices](/overview/preset-voices))
|
||||
- **Voice personalities** — optional free-form personality on any profile
|
||||
plus compose / rewrite / respond modes powered by a local LLM (see
|
||||
[Voice Personalities](/overview/voice-personalities))
|
||||
- **Post-processing effects** — pitch shift, reverb, delay, chorus,
|
||||
compression, filters (Spotify's Pedalboard)
|
||||
- **Expressive speech** — paralinguistic tags like `[laugh]` and `[sigh]`
|
||||
via Chatterbox Turbo; natural-language delivery control via Qwen CustomVoice
|
||||
- **Unlimited length** — auto-chunking with crossfade for long scripts
|
||||
- **Stories editor** — multi-track timeline for conversations and podcasts
|
||||
- **API-first** — REST + WebSocket API; MCP server for agent integrations
|
||||
- **Runs everywhere** — macOS (MLX/Metal), Windows (CUDA / DirectML), Linux
|
||||
(ROCm / CPU), Intel Arc, Docker
|
||||
|
||||
## TTS Engines
|
||||
|
||||
@@ -30,9 +55,21 @@ Seven engines with different strengths, switchable per-generation:
|
||||
| **LuxTTS** | Cloned | English | Lightweight (~1GB VRAM), 48kHz output, 150x realtime on CPU |
|
||||
| **Chatterbox Multilingual** | Cloned | 23 | Broadest language coverage |
|
||||
| **Chatterbox Turbo** | Cloned | English | Fast 350M model with paralinguistic emotion/sound tags |
|
||||
| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model -- 700s+ coherent audio |
|
||||
| **TADA** (1B / 3B) | Cloned | 10 | HumeAI speech-language model — 700s+ coherent audio |
|
||||
| **Kokoro** | Preset (50 voices) | 9 | 82M parameters, CPU realtime, lowest VRAM of any engine |
|
||||
|
||||
## STT and local LLM
|
||||
|
||||
Voicebox also runs a full speech recognition and local LLM stack, shared
|
||||
between dictation, the Captures tab, and per-profile personality modes:
|
||||
|
||||
| Layer | Models |
|
||||
|---|---|
|
||||
| **STT** | Whisper Base / Small / Medium / Large / Turbo (PyTorch or MLX) |
|
||||
| **LLM** | Qwen3 0.6B / 1.7B / 4B (refinement + per-profile compose / rewrite / respond) |
|
||||
|
||||
No cloud fallback, no bring-your-own-API-key. Local is the product.
|
||||
|
||||
## GPU Support
|
||||
|
||||
| Platform | Backend | Notes |
|
||||
@@ -46,11 +83,13 @@ Seven engines with different strengths, switchable per-generation:
|
||||
|
||||
## Use Cases
|
||||
|
||||
- **Game development** -- generate dynamic dialogue for characters
|
||||
- **Content creation** -- produce podcasts and video voiceovers
|
||||
- **Accessibility** -- build text-to-speech tools for users who need them
|
||||
- **Voice assistants** -- create custom voice interfaces
|
||||
- **Production pipelines** -- automate voiceover workflows via the REST API
|
||||
- **Dictation for humans and agents** — speak instead of type, in any app
|
||||
- **Agent voice output** — any MCP-aware agent can speak in a cloned voice
|
||||
- **Game development** — generate dynamic dialogue for characters
|
||||
- **Content creation** — podcasts, video voiceovers, audiobooks
|
||||
- **Accessibility** — speech-to-text for any field, TTS with a voice you own
|
||||
- **Voice assistants** — custom voice interfaces without a cloud bill
|
||||
- **Production pipelines** — automate voice workflows via the REST API
|
||||
|
||||
## Tech Stack
|
||||
|
||||
@@ -61,8 +100,9 @@ Seven engines with different strengths, switchable per-generation:
|
||||
| State | Zustand, React Query |
|
||||
| Backend | FastAPI (Python) |
|
||||
| TTS Engines | Qwen3-TTS, Qwen CustomVoice, LuxTTS, Chatterbox, Chatterbox Turbo, TADA, Kokoro |
|
||||
| STT | Whisper / Whisper Turbo (PyTorch or MLX) |
|
||||
| Local LLM | Qwen3 0.6B / 1.7B / 4B (MLX or PyTorch) |
|
||||
| Effects | Pedalboard (Spotify) |
|
||||
| Transcription | Whisper / Whisper Turbo (PyTorch or MLX) |
|
||||
| Inference | MLX (Apple Silicon) / PyTorch (CUDA/ROCm/XPU/CPU) |
|
||||
| Database | SQLite |
|
||||
| Audio | WaveSurfer.js, librosa |
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
---
|
||||
title: "MCP Server"
|
||||
description: "Let Claude Code, Cursor, Cline, or any MCP-aware agent speak in one of your cloned voices — locally, with no cloud."
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Voicebox ships a built-in **Model Context Protocol** server so local AI
|
||||
agents can call your Voicebox install directly: speak text in a voice
|
||||
profile, transcribe audio, and list captures or profiles. The server runs
|
||||
inside the same process as the rest of Voicebox and is mounted at `/mcp`
|
||||
over Streamable HTTP.
|
||||
|
||||
Agent asks to speak → Voicebox plays audio on your speakers → an on-screen
|
||||
pill surfaces the voice name for the whole duration so you always see what's
|
||||
coming out of your machine.
|
||||
|
||||
<Callout type="info">
|
||||
MCP shipped in **0.5.0** alongside [Dictation](/overview/dictation) and
|
||||
[Voice Personalities](/overview/voice-personalities). The design goal is
|
||||
"local voice layer for every agent on your machine" — the same app that
|
||||
captures your voice can generate a response in any voice profile you've
|
||||
cloned.
|
||||
</Callout>
|
||||
|
||||
## Quick install
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
claude mcp add voicebox \
|
||||
--transport http \
|
||||
--url http://127.0.0.1:17493/mcp \
|
||||
--header "X-Voicebox-Client-Id: claude-code"
|
||||
```
|
||||
|
||||
### Cursor / Windsurf / VS Code MCP / any HTTP MCP client
|
||||
|
||||
Drop this into the client's MCP config (usually `.mcp.json` or a Settings UI):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": { "X-Voicebox-Client-Id": "cursor" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Change `cursor` to whatever name you want the binding to show up as in
|
||||
Voicebox → Settings → MCP. The value is just an identifier for the
|
||||
per-client voice binding — not a secret, not a credential.
|
||||
|
||||
### Clients that only speak stdio
|
||||
|
||||
A stdio shim binary `voicebox-mcp` is bundled with the desktop app. Point
|
||||
the client at that binary's absolute path:
|
||||
|
||||
<Tabs items={["macOS", "Windows", "Linux"]}>
|
||||
<Tab value="macOS">
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab value="Windows">
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "C:\\Program Files\\Voicebox\\voicebox-mcp.exe",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab value="Linux">
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "/opt/voicebox/voicebox-mcp",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-desktop" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
The shim waits up to 30 seconds for the Voicebox backend to come up, then
|
||||
proxies JSON-RPC from stdio over Streamable HTTP. Voicebox must be running
|
||||
for the shim to connect.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Use |
|
||||
|---|---|
|
||||
| `voicebox.speak` | Speak text in a voice profile. Returns a `generation_id` to poll. |
|
||||
| `voicebox.transcribe` | Whisper transcription of base64 audio or an absolute local path. |
|
||||
| `voicebox.list_captures` | Recent captures with transcripts, paginated. |
|
||||
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||
|
||||
### `voicebox.speak`
|
||||
|
||||
```ts
|
||||
voicebox.speak({
|
||||
text: "Deploy complete.",
|
||||
profile?: "Morgan", // name or id; falls back to per-client binding, then default
|
||||
engine?: "qwen", // qwen | qwen_custom_voice | luxtts | chatterbox | chatterbox_turbo | tada | kokoro
|
||||
intent?: "respond", // respond | rewrite | compose — only if the profile has a personality
|
||||
language?: "en",
|
||||
})
|
||||
```
|
||||
|
||||
Returns:
|
||||
|
||||
```json
|
||||
{
|
||||
"generation_id": "…",
|
||||
"status": "generating",
|
||||
"profile": "Morgan",
|
||||
"source": "mcp",
|
||||
"poll_url": "/generate/<id>/status"
|
||||
}
|
||||
```
|
||||
|
||||
- **Plain TTS** — omit `intent`. Text is spoken as-is.
|
||||
- **Persona mode** — pass `intent` and the profile must have a personality set.
|
||||
The LLM transforms the text (respond to it, rewrite it in character, or
|
||||
compose a fresh utterance) before TTS. See [Voice Personalities](/overview/voice-personalities).
|
||||
|
||||
### `voicebox.transcribe`
|
||||
|
||||
```ts
|
||||
voicebox.transcribe({
|
||||
audio_base64?: "<base64>", // exactly one of these two
|
||||
audio_path?: "/absolute/path/to/file.wav",
|
||||
language?: "en",
|
||||
model?: "turbo", // base | small | medium | large | turbo
|
||||
})
|
||||
```
|
||||
|
||||
Returns `{ text, duration, language, model }`. 200 MB ceiling on either path.
|
||||
|
||||
### `voicebox.list_captures`
|
||||
|
||||
`{ limit?: 20, offset?: 0 }` → `{ captures: [...], total }`. `limit` is
|
||||
clamped to `1..=200`.
|
||||
|
||||
### `voicebox.list_profiles`
|
||||
|
||||
No args → `{ profiles: [{ id, name, voice_type, language, has_personality }] }`.
|
||||
|
||||
## Voice resolution
|
||||
|
||||
Every call to `voicebox.speak` (and `POST /speak`) resolves the voice profile
|
||||
in this order:
|
||||
|
||||
<Steps>
|
||||
<Step title="Explicit `profile` arg">
|
||||
Passed as a name (case-insensitive) or id. If the name/id doesn't match,
|
||||
the call errors — the server doesn't silently fall back.
|
||||
</Step>
|
||||
<Step title="Per-client binding">
|
||||
Looked up by the `X-Voicebox-Client-Id` header. Managed in
|
||||
**Voicebox → Settings → MCP**. Lets you pin Claude Code to Morgan,
|
||||
Cursor to Scarlett, etc.
|
||||
</Step>
|
||||
<Step title="Global default">
|
||||
`capture_settings.default_playback_voice_id` — same default voice the
|
||||
Captures tab's "Play as voice" action uses.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
If none of the three produce a profile the tool returns a helpful error
|
||||
pointing at Settings.
|
||||
|
||||
## Per-client bindings
|
||||
|
||||
Voicebox → Settings → MCP shows one row per `client_id` Voicebox has heard
|
||||
from, plus the config snippets you can copy into each agent. Each row
|
||||
carries:
|
||||
|
||||
| Field | Purpose |
|
||||
|---|---|
|
||||
| `label` | Display name in the Settings UI (e.g. "Claude Code"). |
|
||||
| `profile_id` | The voice this client uses when `profile` isn't passed. |
|
||||
| `default_engine` | Override the TTS engine for this client. |
|
||||
| `default_intent` | Default persona mode (`respond` / `rewrite` / `compose`). |
|
||||
| `last_seen_at` | Last time the server saw a request from this client. |
|
||||
|
||||
`last_seen_at` is stamped automatically by middleware on every `/mcp/*`
|
||||
request — useful when you're not sure whether your config took.
|
||||
|
||||
## The speaking pill
|
||||
|
||||
Every agent-initiated speak surfaces the floating pill the same way
|
||||
[Dictation](/overview/dictation) does, in a new `Speaking` state showing the
|
||||
profile name and an elapsed timer. The pill is intentionally unmissable —
|
||||
silent background TTS is a trust hazard, so Voicebox always shows what's
|
||||
being spoken and in what voice.
|
||||
|
||||
Behind the scenes, the backend broadcasts `speak-start` and `speak-end`
|
||||
events on `GET /events/speak`, which `DictateWindow` subscribes to via SSE.
|
||||
The pill overrides the capture session when both would render — you can't
|
||||
hear two pills at once.
|
||||
|
||||
## Non-MCP REST surface
|
||||
|
||||
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||
don't speak MCP — shell scripts, ACP, A2A, GitHub Actions, whatever.
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:17493/speak \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'X-Voicebox-Client-Id: ci' \
|
||||
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||
```
|
||||
|
||||
Body fields match the MCP tool: `text`, optional `profile`, `engine`,
|
||||
`intent`, `language`. Returns a `GenerationResponse` — the same shape as
|
||||
`POST /generate`.
|
||||
|
||||
## Debugging
|
||||
|
||||
Use the MCP Inspector to poke tools directly without plumbing through an
|
||||
agent:
|
||||
|
||||
```
|
||||
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||
```
|
||||
|
||||
Start with `voicebox.list_profiles` to confirm wiring, then
|
||||
`voicebox.speak` for end-to-end — you should hear audio and see the
|
||||
generation land in the Captures tab.
|
||||
|
||||
<Callout type="info">
|
||||
If an agent can't reach the server, the first thing to check is that
|
||||
Voicebox is running — the backend only listens while the desktop app is
|
||||
open. The stdio shim surfaces this as a JSON-RPC error on the client
|
||||
side after its 30-second health-wait window elapses.
|
||||
</Callout>
|
||||
|
||||
## Security
|
||||
|
||||
- **Localhost only.** The server binds to `127.0.0.1`. If you ever point
|
||||
Voicebox at a non-loopback interface (e.g. remote-mode over a trusted
|
||||
network), add a bearer token — it's on the roadmap but not in 0.5.0.
|
||||
- **No auth today.** Any process that can connect to your loopback can
|
||||
call MCP. That's the same trust boundary as the rest of Voicebox's REST
|
||||
API and is appropriate for a single-user local tool.
|
||||
- **`audio_path` reads are unrestricted** against the same trust
|
||||
boundary. If you're scripting against a shared host, prefer
|
||||
`audio_base64` so you don't have to think about path sandboxing.
|
||||
- **Voice cloning consent applies.** See [Voice Cloning](/overview/voice-cloning#limitations)
|
||||
— an agent being able to call `voicebox.speak` in someone's voice
|
||||
doesn't change the ethics of whose voices you clone.
|
||||
|
||||
## Implementation notes
|
||||
|
||||
- **Transport:** Streamable HTTP (Nov-2025 MCP spec, post-SSE). Claude
|
||||
Code, Cursor, Windsurf, and VS Code MCP extensions all support it.
|
||||
- **Package naming:** the backend package is `backend/mcp_server/`, not
|
||||
`mcp`, to avoid shadowing the PyPI `mcp` package FastMCP imports
|
||||
internally.
|
||||
- **Dependencies:** `fastmcp>=3.0,<4.0`, `sse-starlette>=2.0`.
|
||||
- **Lifespan:** mounting FastMCP requires the `lifespan=` kwarg on
|
||||
`FastAPI()` — the startup/shutdown event decorators are incompatible
|
||||
with FastMCP's Streamable HTTP session manager. The Voicebox app.py
|
||||
composes both into one async context manager.
|
||||
|
||||
For the full developer-facing tour of the code layout, see
|
||||
`backend/mcp_server/README.md` in the repo.
|
||||
|
||||
## Next steps
|
||||
|
||||
<Cards>
|
||||
<Card title="Voice Personalities" href="/overview/voice-personalities">
|
||||
Persona mode (`intent=respond/rewrite/compose`) for agents that should
|
||||
transform text in-character before speaking.
|
||||
</Card>
|
||||
<Card title="Dictation" href="/overview/dictation">
|
||||
The pill that surfaces agent speech is the same one that surfaces
|
||||
your dictations — one mental model for both directions of the loop.
|
||||
</Card>
|
||||
<Card title="Captures" href="/overview/captures">
|
||||
Every agent-initiated speak lands in the Captures tab with its
|
||||
generated audio — replay, download, repurpose.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -7,8 +7,12 @@
|
||||
"docker",
|
||||
"quick-start",
|
||||
"gpu-acceleration",
|
||||
"dictation",
|
||||
"captures",
|
||||
"voice-cloning",
|
||||
"preset-voices",
|
||||
"voice-personalities",
|
||||
"mcp-server",
|
||||
"stories-editor",
|
||||
"recording-transcription",
|
||||
"generation-history",
|
||||
|
||||
@@ -1,64 +1,106 @@
|
||||
---
|
||||
title: "Recording & Transcription"
|
||||
description: "Record audio and transcribe speech with Whisper"
|
||||
description: "A map of the three places you can record and transcribe audio in Voicebox — dictation, captures, and voice-profile samples."
|
||||
---
|
||||
|
||||
## Recording
|
||||
## Overview
|
||||
|
||||
Voicebox includes built-in recording capabilities for creating voice samples and capturing audio.
|
||||
Voicebox records and transcribes audio in three different contexts, each
|
||||
feeding a different surface in the app. This page is a map; follow the links
|
||||
for the detail.
|
||||
|
||||
### Features
|
||||
| Goal | Where | Docs |
|
||||
|---|---|---|
|
||||
| Speak and have your words land in another app | Global hotkey → Captures tab + auto-paste | [Dictation](/overview/dictation) |
|
||||
| Record a thought, a meeting, or a voice memo inside Voicebox | Captures tab | [Captures](/overview/captures) |
|
||||
| Record a clip to clone a voice from | Voices tab → profile samples | [Creating Voice Profiles](/overview/creating-voice-profiles) |
|
||||
|
||||
- **Microphone input** - Record from any audio input device
|
||||
- **System audio capture** - Record desktop audio (macOS/Windows)
|
||||
- **Waveform visualization** - See audio levels in real-time
|
||||
- **Multiple formats** - Export as WAV, MP3, or M4A
|
||||
All three paths share the same STT backend — it's the surrounding workflow
|
||||
that differs.
|
||||
|
||||
### How to Record
|
||||
## Dictation
|
||||
|
||||
<Steps>
|
||||
<Step title="Select Input">
|
||||
Choose your microphone or system audio
|
||||
</Step>
|
||||
<Step title="Start Recording">
|
||||
Click the record button and speak clearly
|
||||
</Step>
|
||||
<Step title="Stop & Save">
|
||||
Click stop when finished
|
||||
</Step>
|
||||
<Step title="Use or Export">
|
||||
Use as voice sample or export to file
|
||||
</Step>
|
||||
</Steps>
|
||||
The 0.5.0 headline feature. Hold a chord anywhere on your machine, speak,
|
||||
release. The transcript lands in whatever text field you had focused,
|
||||
cleaned up by a local LLM if auto-refine is on. Captures accumulate in the
|
||||
Captures tab for later replay or re-transcription.
|
||||
|
||||
## Transcription
|
||||
Covered end-to-end in [Dictation](/overview/dictation).
|
||||
|
||||
Automatic speech-to-text powered by OpenAI's Whisper model.
|
||||
## Captures tab
|
||||
|
||||
### Features
|
||||
When you don't need to paste into another app — you just want a clean
|
||||
transcript of some audio — the Captures tab is the home. Record in-app,
|
||||
drop in a file (`.wav`, `.mp3`, `.m4a`, `.webm`, `.opus`, `.flac`), or dig
|
||||
through dictations that already landed there. Every capture keeps its
|
||||
original audio, can be retranscribed with a different model, and can be
|
||||
played back through any voice profile you have.
|
||||
|
||||
- **High accuracy** - Industry-leading speech recognition
|
||||
- **Multiple languages** - Supports 50+ languages
|
||||
- **Automatic detection** - Language auto-detection
|
||||
- **Timestamps** - Word-level timing information
|
||||
Covered in [Captures](/overview/captures).
|
||||
|
||||
### How to Transcribe
|
||||
## Voice profile samples
|
||||
|
||||
<Steps>
|
||||
<Step title="Select Audio">
|
||||
Choose a recording or upload an audio file
|
||||
</Step>
|
||||
<Step title="Choose Language">
|
||||
Select language or use auto-detect
|
||||
</Step>
|
||||
<Step title="Transcribe">
|
||||
Click transcribe and wait for processing
|
||||
</Step>
|
||||
<Step title="Review & Export">
|
||||
Review text and export as needed
|
||||
</Step>
|
||||
</Steps>
|
||||
A separate flow, in the Voices tab. When you're creating a profile from an
|
||||
audio clip, the sample is what the cloning engine actually learns from —
|
||||
the `reference_text` on a sample must match the audio *verbatim*, which is
|
||||
why samples are a different data model from captures.
|
||||
|
||||
You can promote a capture to a sample from the Captures tab's Send-to menu
|
||||
("Use as voice sample…"), which opens a reference-text confirm dialog so
|
||||
you can correct the last ~10% of transcript accuracy before saving.
|
||||
|
||||
Covered in [Creating Voice Profiles](/overview/creating-voice-profiles).
|
||||
|
||||
## Transcription models
|
||||
|
||||
All three paths share the same Whisper models. Pick a default in
|
||||
**Settings → Captures → Transcription**; override per capture if you need
|
||||
to.
|
||||
|
||||
| Model | Size | When to pick it |
|
||||
|---|---|---|
|
||||
| Whisper Base | ~300 MB | Fast. Default. Good for clean speech. |
|
||||
| Whisper Small | ~500 MB | Better quality, still fast. |
|
||||
| Whisper Medium | ~1.5 GB | High quality. |
|
||||
| Whisper Large | ~3 GB | Best quality, slow on CPU. |
|
||||
| Whisper Turbo | ~1.5 GB | Large-tier quality, ~5× faster than Large. |
|
||||
|
||||
On Apple Silicon the model runs through **MLX-Whisper** (~8× faster than
|
||||
PyTorch). Everywhere else it runs through PyTorch `transformers`. The
|
||||
backend picks the right one — you don't configure it.
|
||||
|
||||
<Callout type="info">
|
||||
Transcription is useful for creating voice samples from existing audio or generating subtitles.
|
||||
For noisy clips, prefer **Turbo** or **Large**. Base can hallucinate on
|
||||
hard inputs — most famously the "thanks for watching" loop. Voicebox
|
||||
strips those loops deterministically before LLM refinement runs, so a
|
||||
capture can be cleanly re-refined even if the raw transcript has them.
|
||||
</Callout>
|
||||
|
||||
## Language
|
||||
|
||||
You can pass a language hint for short clips (under ~5 seconds) where
|
||||
Whisper's auto-detect is unreliable. Set a default language lock in
|
||||
**Settings → Captures → Transcription → Language**, or override per capture.
|
||||
|
||||
## Transcription API
|
||||
|
||||
Developer-level detail on the STT backend, model loading, preprocessing, and
|
||||
the `/transcribe` endpoint lives in the
|
||||
[Transcription developer guide](/developer/transcription). The Captures
|
||||
pipeline also exposes `/captures` as a higher-level endpoint that wraps
|
||||
STT + archival + optional refinement in one call — see
|
||||
[Captures](/overview/captures#api-surface).
|
||||
|
||||
## Next steps
|
||||
|
||||
<Cards>
|
||||
<Card title="Dictation" href="/overview/dictation">
|
||||
Hold a chord anywhere on your machine, speak, release.
|
||||
</Card>
|
||||
<Card title="Captures" href="/overview/captures">
|
||||
The paired audio + transcript archive.
|
||||
</Card>
|
||||
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||
Record or upload samples for voice cloning.
|
||||
</Card>
|
||||
</Cards>
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
---
|
||||
title: "Voice Personalities"
|
||||
description: "Attach a personality to a voice profile and use Compose, Rewrite, or Respond to generate in-character speech — all powered by a local LLM."
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
A **personality** is an optional free-form description attached to a voice
|
||||
profile — who this voice is, how they speak, what they care about. Set one
|
||||
and three new actions appear on the profile, each powered by a bundled
|
||||
Qwen3 LLM running entirely locally:
|
||||
|
||||
- **Compose** — generate a fresh utterance in this character's voice
|
||||
- **Rewrite** — restate your text in their voice while preserving every idea
|
||||
- **Respond** — treat your text as a prompt and produce the character's reply
|
||||
|
||||
The LLM produces the text. The voice profile speaks it. No cloud round-trip,
|
||||
no external API — the whole loop runs on your hardware.
|
||||
|
||||
<Callout type="info">
|
||||
Personalities shipped in **0.5.0**. The same local LLM doubles as the
|
||||
refinement model for [Dictation](/overview/dictation) — one LLM in the app,
|
||||
not two, sharing one model cache and one GPU-memory footprint.
|
||||
</Callout>
|
||||
|
||||
## Setting a personality
|
||||
|
||||
Open a voice profile's edit view. The **Personality** field is free-form text
|
||||
up to **2,000 characters**. Describe the voice however helps you — past
|
||||
lines they'd say, speech patterns, tone, boundaries.
|
||||
|
||||
Good descriptions tend to include:
|
||||
|
||||
- A one-line identity (who they are)
|
||||
- Speech patterns (rhythm, vocabulary, what they avoid)
|
||||
- Representative phrases — example lines show the LLM the target tone
|
||||
better than adjectives
|
||||
- What the character *wouldn't* do (they don't explain, they don't
|
||||
apologize, they refuse to break character, etc.)
|
||||
|
||||
You can set a personality on any voice profile type — cloned or preset. The
|
||||
three modes work identically regardless of engine.
|
||||
|
||||
## The three modes
|
||||
|
||||
Each mode is tuned for a specific job and the LLM temperature is adjusted
|
||||
to match.
|
||||
|
||||
### Compose
|
||||
|
||||
Generate a fresh utterance in the character's voice, with no seed text.
|
||||
Click again to get a different take.
|
||||
|
||||
- **When to use:** prototyping, sampling a character's voice, brainstorming
|
||||
a line without typing one first
|
||||
- **Temperature:** hot — variety is the point
|
||||
- **Typical output:** a short, punchy line that fits the character's
|
||||
register
|
||||
|
||||
### Rewrite
|
||||
|
||||
Take your input text and restate it in the character's voice while
|
||||
preserving every idea. High-fidelity mode — the content doesn't change, only
|
||||
the voice does.
|
||||
|
||||
- **When to use:** turning a dictated memo into in-character speech; lifting
|
||||
a plain-English script into a specific voice without editing by hand
|
||||
- **Temperature:** cold — faithfulness wins
|
||||
- **Typical output:** same ideas, same order, different phrasing and cadence
|
||||
|
||||
### Respond
|
||||
|
||||
Treat your input as a prompt and produce the character's reply — as if
|
||||
you'd said it *to* them.
|
||||
|
||||
- **When to use:** spoken-input agents; Q&A with a specific voice;
|
||||
interactive character experiences
|
||||
- **Temperature:** balanced — creative but grounded
|
||||
- **Typical output:** a reply to your prompt, written in-character
|
||||
|
||||
## Speech-only framing
|
||||
|
||||
All three modes enforce **speech-only** output. The LLM is prompted to
|
||||
produce things a person would actually say out loud — no narration, no
|
||||
action tags (`*sighs*`, `[laughs]`), no meta-commentary, no markdown
|
||||
formatting, no stage directions.
|
||||
|
||||
This is deliberate: the output is going straight into TTS, and anything that
|
||||
isn't speakable ends up either ignored or read literally. The speech-only
|
||||
framing also makes the output land cleanly inside dialogue, so you can drop
|
||||
a Respond result straight into a Story.
|
||||
|
||||
## The local LLM
|
||||
|
||||
The bundled LLM is **Qwen3**, available in three sizes:
|
||||
|
||||
| Model | Download size | Best for |
|
||||
|---|---|---|
|
||||
| Qwen3 0.6B | ~400 MB | Default. Very fast, good for casual use. |
|
||||
| Qwen3 1.7B | ~1.1 GB | Sweet spot for character personalities with specific phrasing. |
|
||||
| Qwen3 4B | ~2.5 GB | Full quality. Slowest. Useful for very particular tone. |
|
||||
|
||||
The model runs through the same backend split Voicebox already uses for TTS
|
||||
— **MLX** (4-bit community quants) on Apple Silicon, **PyTorch** (transformers
|
||||
`AutoModelForCausalLM`) everywhere else. Downloads go through the same cache
|
||||
and model-management UI as TTS models.
|
||||
|
||||
Pick a size in **Settings → Captures → Refinement → Refinement model** — the
|
||||
personality modes reuse it. If you switch models, both refinement and
|
||||
personality output pick up the change on the next call.
|
||||
|
||||
## Using the modes
|
||||
|
||||
The three actions appear as buttons on the profile when a personality is
|
||||
set. For each:
|
||||
|
||||
<Steps>
|
||||
<Step title="Type or dictate your input">
|
||||
Rewrite and Respond need input text. Compose doesn't.
|
||||
</Step>
|
||||
<Step title="Click the mode">
|
||||
The LLM runs, then the result fills the generate box.
|
||||
</Step>
|
||||
<Step title="Hit generate">
|
||||
The TTS engine speaks the LLM output in the profile's voice. The result
|
||||
lands in generation history as a normal generation.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
Each button also has an inline regenerate affordance — click again to
|
||||
resample. Compose will give you something totally different; Rewrite and
|
||||
Respond will give you a variation on the same content.
|
||||
|
||||
## Use cases
|
||||
|
||||
- **Agents that speak in a voice you own.** Combine Respond with the
|
||||
built-in [MCP Server](/overview/mcp-server) so Claude Code, Cursor,
|
||||
Cline, or any MCP-aware agent can talk back through a profile with a
|
||||
personality. The agent calls `voicebox.speak({ text, profile, intent:
|
||||
"respond" })` and Voicebox produces in-character speech in your cloned
|
||||
voice.
|
||||
- **Interactive characters.** Games, narrative tools, accessibility
|
||||
experiences. A character with a personality description plus a cloned
|
||||
voice becomes a reusable prop.
|
||||
- **Accessibility.** People who can't speak in their original voice can
|
||||
keep a personality description of how they used to sound and use Rewrite
|
||||
to turn typed input into in-character speech.
|
||||
- **Creative drafting.** Write a plain outline, Rewrite line-by-line into
|
||||
the character's voice, drop the audio into a Story.
|
||||
|
||||
## API surface
|
||||
|
||||
Personalities and the three modes are accessible via REST:
|
||||
|
||||
| Method | Endpoint | Body |
|
||||
|---|---|---|
|
||||
| `PUT` | `/profiles/{id}` | Include a `personality` field up to 2,000 chars to set it. |
|
||||
| `POST` | `/profiles/{id}/speak` | Runs the LLM + TTS in one shot. Body includes `text`, `intent` (`compose`, `rewrite`, or `respond`), optional `engine`, `language`. |
|
||||
|
||||
The `/profiles/{id}/speak` endpoint is the same primitive MCP's
|
||||
`voicebox.speak` tool calls when that ships. Scripts and agents can use it
|
||||
directly today.
|
||||
|
||||
## Limits and gotchas
|
||||
|
||||
- **The personality is a prompt, not a fine-tune.** The LLM will sometimes
|
||||
drift out of character, especially on Compose at high temperature. Click
|
||||
again for another take.
|
||||
- **Long personalities are not always better.** 2,000 chars is a ceiling,
|
||||
not a goal. A sharp 300-char description with two example lines
|
||||
typically outperforms a long one.
|
||||
- **Speech-only framing is enforced, but not bulletproof.** Very large
|
||||
prompts or unusual inputs can sneak an action tag through. If you see
|
||||
`[laughs]` in TTS output, it's usually a personality-field hint the
|
||||
model anchored onto — remove it from the description.
|
||||
- **Rewrite is stricter than Respond.** If the output is changing your
|
||||
meaning, you probably want Respond (or a wholesale Compose with context
|
||||
in the input), not Rewrite.
|
||||
|
||||
## Next steps
|
||||
|
||||
<Cards>
|
||||
<Card title="Dictation" href="/overview/dictation">
|
||||
Dictate the input for Rewrite or Respond from anywhere on your machine.
|
||||
</Card>
|
||||
<Card title="Captures" href="/overview/captures">
|
||||
Captures feed personalities naturally — dictate a memo, rewrite it in
|
||||
a character voice, generate speech.
|
||||
</Card>
|
||||
<Card title="Creating Voice Profiles" href="/overview/creating-voice-profiles">
|
||||
Add a personality to an existing profile.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -0,0 +1,344 @@
|
||||
# MCP Server — Voicebox Speed Run
|
||||
|
||||
**Status:** v1 shipped — HTTP transport, all 4 tools, per-client bindings, `POST /speak`, stdio shim (binary built, bundled into Tauri sidecar), Settings UI, speak-pill via SSE with Rust-side `dictate:show` handler so agent-initiated speech surfaces the pill on screen. `cargo check` clean, `tsc` clean, full Inspector round-trip verified.
|
||||
**Last reviewed:** 2026-04-23
|
||||
|
||||
## Status
|
||||
|
||||
### Shipped (backend)
|
||||
- **`fastmcp` + `sse-starlette`** pinned in `backend/requirements.txt`.
|
||||
- **`backend/mcp_server/`** package with `server.py`, `tools.py`, `context.py`, `resolve.py`, `events.py`, `README.md`. Named `mcp_server` (not `mcp`) to sidestep a shadowing conflict with the installed `mcp` PyPI package that FastMCP imports internally.
|
||||
- **Streamable HTTP mount at `/mcp`** via FastMCP's `http_app(transport='http')`. Sub-app lifespan composed with Voicebox's own startup/shutdown through an `@asynccontextmanager lifespan=` in `backend/app.py` (migrated away from the deprecated `@app.on_event` handlers).
|
||||
- **Four MCP tools**, dot-named to match the landing and ecosystem convention:
|
||||
- `voicebox.speak(text, profile?, engine?, intent?, language?)`
|
||||
- `voicebox.transcribe(audio_base64?, audio_path?, language?, model?)`
|
||||
- `voicebox.list_captures(limit, offset)`
|
||||
- `voicebox.list_profiles()`
|
||||
- **`ClientIdMiddleware`** pulls `X-Voicebox-Client-Id` into a `ContextVar` on every `/mcp*` request; auto-stamps `MCPClientBinding.last_seen_at`, auto-creating the row if the client is new.
|
||||
- **Profile resolution precedence** `explicit → per-client binding → capture_settings.default_playback_voice_id → error`. `services/profiles.get_profile_orm_by_name_or_id()` lets agents pass a voice by name ("Morgan") instead of UUID.
|
||||
- **`MCPClientBinding` table** (new) via `Base.metadata.create_all` — no migration needed.
|
||||
- **Bindings REST:** `GET|PUT /mcp/bindings`, `DELETE /mcp/bindings/{client_id}`.
|
||||
- **`POST /speak`** REST wrapper for non-MCP callers (shell / ACP / A2A). Same `resolve_profile` precedence, same code path as the MCP tool.
|
||||
- **Stdio shim** at `backend/mcp_shim/__main__.py` — ~200 lines of `httpx` proxy; reads env (`VOICEBOX_PORT`, `VOICEBOX_HOST`, `VOICEBOX_CLIENT_ID`), waits for `/health`, then streams JSON-RPC ↔ SSE. Rolled our own after the `mcp` SDK's session-management helpers mis-shook-hands. Smoke-tested: `initialize`, `tools/list`, and `tools/call` all round-trip cleanly.
|
||||
- **Pill SSE:** `GET /events/speak` (`sse-starlette`) emits `speak-start` from the MCP tool and `POST /speak`, `speak-end` from `services/generation.run_generation`'s finally block.
|
||||
- **PyInstaller:**
|
||||
- `backend/build_binary.py` `--shim` flag builds a minimal `voicebox-mcp` binary (torch/transformers/mlx/etc. explicitly excluded, target <20 MB).
|
||||
- The main server spec picks up `fastmcp`, `mcp`, `sse_starlette`, and `backend.mcp_server.*` via `--collect-all` / `--hidden-import`.
|
||||
- **`backend/mcp_server/README.md`** quickstart (Inspector, `.mcp.json` snippets, tool reference).
|
||||
|
||||
### Shipped (frontend)
|
||||
- **`Settings → MCP`** page (`app/src/components/ServerTab/MCPPage.tsx`):
|
||||
- Three copy-paste snippets auto-filled with the detected `serverUrl`: HTTP (recommended), Claude Code CLI one-liner, stdio fallback.
|
||||
- Default voice picker (bound to `capture_settings.default_playback_voice_id`, shared with Captures-tab "Play as voice").
|
||||
- Per-client bindings table with inline profile picker, remove button, and a connection-status indicator that refreshes every 10 s.
|
||||
- Add-binding form with client_id / label / profile dropdown.
|
||||
- **`useMCPBindings`** TanStack hook (optimistic delete, invalidate on upsert).
|
||||
- **`useSpeakEvents`** hook — auto-reconnecting `EventSource('/events/speak')`, tracks the active generation_id, exposes an elapsed-ms timer that ticks so the pill's clock advances.
|
||||
- **`CapturePill`** has a new `'speaking'` state + "Speaking" label + playing-bars mode.
|
||||
- **`DictateWindow`** subscribes to speak events and overrides `pillState` when an agent is speaking. Emits `dictate:show` on speak-start so the Rust side can surface the pill window.
|
||||
- Router + `ServerTab` tab bar wired to `/settings/mcp`.
|
||||
|
||||
### Shipped (native shell)
|
||||
- **`tauri.conf.json`** — `voicebox-mcp` added to `externalBin` (alongside `voicebox-server`).
|
||||
- **`dictate:show` listener** in `tauri/src-tauri/src/main.rs` — invokes a new `show_dictate_window(app_handle)` helper that mirrors the hotkey-monitor's position+show logic (undo click-through, reposition to top-center of the current monitor, show). Agent-initiated speech now pops the pill visible on screen.
|
||||
|
||||
### Validated end-to-end (this session, via curl)
|
||||
- `/mcp/` init → `tools/list` → `tools/call voicebox.speak` → actual audio plays (Jarvis, 1.68 s).
|
||||
- `POST /speak` with `X-Voicebox-Client-Id: claude-code` resolves to the bound Jarvis profile without passing `profile`.
|
||||
- `/events/speak` emits `ready`, `speak-start`, `speak-end` in order, generation_id threads through both.
|
||||
- Stdio shim: `echo {…} | python -m backend.mcp_shim` returns valid JSON-RPC for all 4 methods.
|
||||
- `last_seen_at` auto-stamps on first call; binding row auto-creates.
|
||||
- Frontend `tsc --noEmit`: clean.
|
||||
- `cargo check` on the Tauri crate: clean.
|
||||
|
||||
### Outstanding (must-do before release)
|
||||
- **CI build for shim on Windows/Linux** — `python backend/build_binary.py --shim` is wired up and built cleanly for `aarch64-apple-darwin` (18 MB, installed at `tauri/src-tauri/binaries/voicebox-mcp-aarch64-apple-darwin`, Tauri `cargo check` green). The Windows and Linux triples (`x86_64-pc-windows-msvc`, `x86_64-unknown-linux-gnu`) need the same build in their respective CI runners and artifacts dropped alongside the macOS binary.
|
||||
- **Windows/Linux paths in the stdio snippet** — the Settings page hardcodes the macOS path (`/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp`). Needs a per-OS switch (`%LOCALAPPDATA%\Programs\Voicebox\voicebox-mcp.exe`, Linux bundled-path), ideally with the Tauri shell resolving its own app path at runtime and injecting it into the snippet.
|
||||
|
||||
### Nice-to-have (follow-up passes)
|
||||
- **One-click install buttons** — write/merge into `~/.claude/settings.json`, `~/.cursor/mcp.json`, etc. via a Tauri command. Copy-paste works today; this is pure ergonomics.
|
||||
- **`.mcpb` desktop extension** for Claude Desktop (single file, double-click to install). Claude Desktop-only, so lower priority than the agent-harness crowd.
|
||||
- **Refactor the hotkey_monitor.rs show-logic** to call `show_dictate_window()` instead of duplicating the position+show block. Skipped at ship to avoid regressing the well-tested chord path.
|
||||
- **Source attribution on `Generation.source`** — currently `"manual" | "personality_speak"`; adding `"mcp"` / `"rest"` would let the Captures tab filter by MCP-originated rows.
|
||||
|
||||
## Context
|
||||
|
||||
Voicebox already ships the I/O surface (Captures, Generate, personality-driven `/profiles/{id}/speak`), but local AI agents can't reach any of it. This plan adds a Model Context Protocol server so Claude Code / Cursor / Cline can call `voicebox.speak`, `voicebox.transcribe`, `voicebox.list_captures`, and `voicebox.list_profiles` — turning Voicebox into the local voice layer for every agent on the user's machine (Phase 5 of `docs/plans/VOICE_IO.md`).
|
||||
|
||||
The shortest path to "Claude Code speaks in a cloned voice": mount **FastMCP** inside the existing FastAPI/uvicorn process at `/mcp` (Streamable HTTP), and users install it as a URL (`{"url": "http://127.0.0.1:17493/mcp"}`) — the ecosystem-idiomatic shape for a long-running local service. Per-client voice binding via a new `mcp_client_bindings` table + Settings UI, resolved from an `X-Voicebox-Client-Id` header. A **stdio shim binary** `voicebox-mcp` is bundled as a fallback sidecar for clients that can't speak HTTP MCP. A public `POST /speak` REST wrapper covers non-MCP callers (shell scripts, ACP, A2A). A `speaking` pill state gives agent-initiated audio visibility — trust-critical, non-negotiable.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Claude Code / Cursor / Windsurf / VS Code MCP
|
||||
│
|
||||
├─ HTTP (primary) ────────────────────┐
|
||||
│ {"url": ".../mcp"} │
|
||||
│ │
|
||||
└─ stdio (fallback) ───────────────▶ [voicebox-mcp shim binary]
|
||||
{"command": "/abs/path/voicebox-mcp"} (absolute path;
|
||||
│ Settings page
|
||||
│ copies it for you)
|
||||
▼
|
||||
uvicorn + FastAPI (port 17493)
|
||||
├─ /mcp (FastMCP, Streamable HTTP)
|
||||
└─ /speak (REST wrapper for non-MCP callers)
|
||||
└─ tools call existing services
|
||||
```
|
||||
|
||||
- **Transport:** Streamable HTTP as primary (Nov-2025 spec, post-SSE). Claude Code, Cursor, Windsurf, and the VS Code MCP extensions all support HTTP — it's the idiomatic shape for a long-running local service, which Voicebox already is.
|
||||
- **Stdio fallback:** `voicebox-mcp` binary bundled inside the app for clients that can't speak HTTP MCP. The Settings page renders the exact snippet with the detected absolute path — user copies, pastes, done. No PATH manipulation, no custom CLI wrapper.
|
||||
- **Identity:** HTTP clients set `X-Voicebox-Client-Id` header in their MCP config's `headers` block. Stdio clients set `VOICEBOX_CLIENT_ID` env var, which the shim forwards as the same HTTP header. Server reads it into a `ContextVar`.
|
||||
- **Profile resolution precedence:** explicit tool arg → per-client `MCPClientBinding.profile_id` → `capture_settings.default_playback_voice_id` → error.
|
||||
- **Port:** `17493`, matching `tauri/src-tauri/src/main.rs:63` (`SERVER_PORT` constant). Shim default with `VOICEBOX_PORT` env override.
|
||||
- **Non-MCP access:** `POST /speak` is a thin REST wrapper around the same tool path — one endpoint for shell scripts, ACP, A2A, and anything that isn't MCP-native.
|
||||
|
||||
## Library choice
|
||||
|
||||
- **`fastmcp`** (PyPI — verify on install whether the canonical import is `fastmcp` standalone or `mcp.server.fastmcp` from the consolidated `mcp` package; the API is identical).
|
||||
- **`sse-starlette`** for the `/events/speak` pill-state broadcast.
|
||||
- **`httpx` + `anyio`** already present — used by the shim.
|
||||
|
||||
## Data model
|
||||
|
||||
New table, **one row per client_id** (not a singleton — scales to unknown clients, maps 1:1 to the Settings UI list):
|
||||
|
||||
```python
|
||||
# backend/database/models.py
|
||||
class MCPClientBinding(Base):
|
||||
__tablename__ = "mcp_client_bindings"
|
||||
client_id = Column(String, primary_key=True) # "claude-code", "cursor", ...
|
||||
label = Column(String, nullable=True)
|
||||
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
|
||||
default_engine = Column(String, nullable=True)
|
||||
default_intent = Column(String, nullable=True) # "respond" | "rewrite" | "compose"
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
```
|
||||
|
||||
Global default stays in `capture_settings.default_playback_voice_id` — no duplication. Migration: new `_migrate_mcp_client_bindings()` in `backend/database/migrations.py` using `CREATE TABLE IF NOT EXISTS`, mirroring the existing idempotent-add-column pattern.
|
||||
|
||||
## File plan
|
||||
|
||||
### Backend — new
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `backend/mcp/__init__.py` | Package marker |
|
||||
| `backend/mcp/server.py` | `build_mcp_server()` + `mount_into(app)`; registers tools, middleware, mount at `/mcp` |
|
||||
| `backend/mcp/tools.py` | The 4 `@mcp.tool()` functions — thin wrappers over existing services |
|
||||
| `backend/mcp/context.py` | `current_client_id: ContextVar[str \| None]` + Starlette middleware |
|
||||
| `backend/mcp/resolve.py` | `resolve_profile(explicit, client_id, db) -> VoiceProfile \| None` |
|
||||
| `backend/mcp/events.py` | In-memory `asyncio.Queue` pub/sub for speak-start / speak-end |
|
||||
| `backend/mcp/README.md` | MCP Inspector quickstart + `.mcp.json` snippets |
|
||||
| `backend/mcp_shim/__init__.py`, `__main__.py` | Stdio ↔ Streamable HTTP proxy (~150 lines) |
|
||||
| `backend/voicebox-mcp.spec` | PyInstaller spec for the shim (strips torch/transformers from `hiddenimports`) |
|
||||
| `backend/routes/speak.py` | `POST /speak {text, profile?, engine?, intent?, language?}` — REST wrapper around `resolve_profile()` + `speak_in_character()` for non-MCP agents |
|
||||
|
||||
### Backend — modified
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `backend/app.py` | Migrate `@app.on_event("startup"/"shutdown")` (lines 185, 268) to `lifespan=` kwarg on `FastAPI()` using `AsyncExitStack`; call `mount_into(application)` after `register_routers`. Register `ClientIdMiddleware`. |
|
||||
| `backend/routes/profiles.py` | In `speak_in_character` (line 453): `events.publish("speak-start", {...})` on entry; completion hook publishes `speak-end`. Accept optional `source="mcp"` marker. |
|
||||
| `backend/services/generation.py` | `run_generation` completion path publishes `speak-end`. |
|
||||
| `backend/services/profiles.py` | New `async def get_profile_by_name_or_id(name_or_id, db)` — id lookup first, case-insensitive name fallback. |
|
||||
| `backend/database/models.py` | Add `MCPClientBinding`. |
|
||||
| `backend/database/migrations.py` | Add `_migrate_mcp_client_bindings`. |
|
||||
| `backend/models.py` | Add `MCPClientBindingResponse`, `MCPClientBindingUpdate`. |
|
||||
| `backend/routes/__init__.py` | Register `mcp_bindings_router`, `speak_router`, `events_router`. |
|
||||
| `backend/routes/mcp_bindings.py` (new) | REST CRUD for bindings (list, upsert, delete). |
|
||||
| `backend/routes/events.py` (new) | `GET /events/speak` — `EventSourceResponse` subscribed to the events queue. |
|
||||
| `backend/requirements.txt` | `+ fastmcp` (or `mcp>=1.0`), `+ sse-starlette` |
|
||||
| `backend/voicebox-server.spec` | `hiddenimports += ['mcp', 'mcp.server', 'fastmcp']` |
|
||||
| `backend/build_binary.py` | Second PyInstaller invocation for `voicebox-mcp.spec`; copy to `tauri/src-tauri/binaries/` with target-triple suffix |
|
||||
|
||||
### Frontend — new
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `app/src/components/ServerSettings/MCPBindings.tsx` | Settings section — default voice + per-client binding rows + `.mcp.json` copy-paste cheatsheet |
|
||||
| `app/src/lib/hooks/useMCPBindings.ts` | TanStack Query mirror of `useCaptureSettings` |
|
||||
| `app/src/lib/api/mcp.ts` | `listMCPBindings` / `upsertMCPBinding` / `deleteMCPBinding` |
|
||||
|
||||
### Frontend — modified
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `app/src/components/DictateWindow/DictateWindow.tsx` | Open `EventSource('/events/speak')`; on `speak-start` set pill to `speaking` with profile name; dismiss on `speak-end`. |
|
||||
| `app/src/components/CapturePill/CapturePill.tsx` | Add `speaking` branch — reuse the active waveform, swap status label to profile name. |
|
||||
| `app/src/lib/hooks/useCaptureRecordingSession.ts` | Union a `speaking` injection into the derived pill state. |
|
||||
| `app/src/lib/api/types.ts` | `MCPClientBinding`, `MCPClientBindingUpdate` types. |
|
||||
| `app/src/components/ServerSettings/index.tsx` | Register the new MCP section in the tab aggregator. |
|
||||
|
||||
### Tauri
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `tauri/src-tauri/tauri.conf.json` | `"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"]` |
|
||||
| `tauri/src-tauri/binaries/voicebox-mcp-<triple>` | Build artifact from PyInstaller |
|
||||
|
||||
## Tool signatures
|
||||
|
||||
All tools read `current_client_id.get()` (from middleware). Return JSON-serializable dicts.
|
||||
|
||||
Tools are registered with **dotted names** (`voicebox.speak`, etc.) to match the landing page and the industry convention (`filesystem.read_file`, `github.create_issue`). Python function names stay snake_case; the dot goes in the `name=` kwarg.
|
||||
|
||||
```python
|
||||
# backend/mcp/tools.py
|
||||
|
||||
@mcp.tool(name="voicebox.speak")
|
||||
async def speak(text: str,
|
||||
profile: str | None = None, # name OR id
|
||||
engine: str | None = None,
|
||||
intent: str = "respond", # "respond" | "rewrite" | "compose"
|
||||
language: str | None = None) -> dict:
|
||||
"""Speak text in a voice profile. Returns {generation_id, status, profile, poll}."""
|
||||
# resolve profile via precedence, call speak_in_character (profiles.py:453)
|
||||
# with persist=True so it lands in history.
|
||||
|
||||
@mcp.tool(name="voicebox.transcribe")
|
||||
async def transcribe(audio_base64: str | None = None,
|
||||
audio_path: str | None = None, # absolute local path
|
||||
language: str | None = None,
|
||||
model: str | None = None) -> dict:
|
||||
"""Transcribe audio. Exactly one of audio_base64/audio_path. Returns {text, duration, language}."""
|
||||
# validate path readable, size < 200 MB, then call services.transcribe.transcribe_bytes
|
||||
|
||||
@mcp.tool(name="voicebox.list_captures")
|
||||
async def list_captures(limit: int = 20, offset: int = 0) -> dict:
|
||||
"""Recent captures with transcripts. Returns {captures: [...]}"""
|
||||
|
||||
@mcp.tool(name="voicebox.list_profiles")
|
||||
async def list_profiles() -> dict:
|
||||
"""Available voice profiles. Returns {profiles: [{id, name, voice_type, has_personality}]}"""
|
||||
```
|
||||
|
||||
### `POST /speak` (non-MCP REST wrapper)
|
||||
|
||||
```python
|
||||
# backend/routes/speak.py
|
||||
@router.post("/speak", response_model=GenerationResponse)
|
||||
async def speak(data: SpeakRequest, request: Request, db: Session = Depends(get_db)):
|
||||
"""Same behavior as the MCP tool — for shell scripts, ACP, A2A, or anything non-MCP."""
|
||||
client_id = request.headers.get("X-Voicebox-Client-Id")
|
||||
profile = resolve_profile(data.profile, client_id, db)
|
||||
if profile is None: raise HTTPException(400, "No voice profile resolved.")
|
||||
persist_req = PersonalitySpeakRequest(text=data.text, persist=True, language=data.language,
|
||||
engine=data.engine, intent=data.intent or "respond")
|
||||
return await speak_in_character(profile.id, persist_req, db)
|
||||
```
|
||||
|
||||
`SpeakRequest`: `{ text: str, profile: str | None, engine: str | None, intent: str | None, language: str | None }`. Accepts name OR id for `profile` (via `resolve_profile`), and resolves via the same precedence as the MCP tool so the two surfaces behave identically.
|
||||
|
||||
## Mount point (`backend/app.py`)
|
||||
|
||||
```python
|
||||
# After register_routers(application):
|
||||
from .mcp.server import mount_into
|
||||
mount_into(application)
|
||||
```
|
||||
|
||||
`mount_into` installs `ClientIdMiddleware` and calls `app.mount("/mcp", mcp.streamable_http_app())`.
|
||||
|
||||
**Lifespan migration is load-bearing** — FastMCP's session manager requires the `lifespan=` kwarg, not `@app.on_event`. Wrap the existing startup/shutdown bodies in an `@asynccontextmanager` using `contextlib.AsyncExitStack` so both Voicebox's init and FastMCP's session manager run. Verify dev + packaged build after the migration.
|
||||
|
||||
## Stdio shim (`backend/mcp_shim/__main__.py`)
|
||||
|
||||
1. Port: `int(os.environ.get("VOICEBOX_PORT", "17493"))`.
|
||||
2. Client id: `os.environ.get("VOICEBOX_CLIENT_ID", "unknown")`.
|
||||
3. Health probe `GET /health` with 30 s tolerance (torch imports slowly). On failure, emit JSON-RPC error on stdout, exit 1.
|
||||
4. Connect Streamable HTTP MCP client to `http://127.0.0.1:{port}/mcp` with `X-Voicebox-Client-Id: {client_id}` header.
|
||||
5. Proxy JSON-RPC bidirectionally — stdin → HTTP, SSE → stdout. Use `mcp` SDK's built-in stdio↔HTTP bridge if available; otherwise ~40 lines of asyncio.
|
||||
6. Stdout = JSON-RPC only. All logs to stderr.
|
||||
|
||||
PyInstaller spec keeps only `mcp`, `httpx`, `anyio`, `click` — target binary <20 MB.
|
||||
|
||||
## Pill `speaking` state
|
||||
|
||||
- `backend/mcp/events.py`: module-level `_subscribers: list[asyncio.Queue]` + `publish(kind, payload)` + `subscribe() -> Queue`.
|
||||
- `speak_in_character` publishes `speak-start` with `{generation_id, profile_id, profile_name, source}` immediately after `task_manager.start_generation`; `run_generation`'s completion path publishes `speak-end`.
|
||||
- `/events/speak` → `EventSourceResponse`.
|
||||
- `DictateWindow` opens `EventSource` next to existing `dictate:*` listeners, maps `speak-start/end` → pill `speaking` mode with profile name.
|
||||
- Optional filter: only show pill when `source === "mcp"` (avoids pill churn during manual speak flows). Settings toggle later.
|
||||
|
||||
## Settings UI (`MCPBindings.tsx`)
|
||||
|
||||
- **Global default voice** picker bound to `capture_settings.default_playback_voice_id` (reuses `useCaptureSettings`).
|
||||
- **Per-client table** — add/edit/remove rows of `{client_id, label, profile_id, default_engine, default_intent}`. Uses `useMCPBindings`.
|
||||
- **Connection cheatsheet** — two tabs, HTTP (default) and Stdio (fallback), with copy-to-clipboard snippets per known client:
|
||||
|
||||
HTTP form (primary):
|
||||
```json
|
||||
{"mcpServers": {"voicebox": {
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": {"X-Voicebox-Client-Id": "claude-code"}
|
||||
}}}
|
||||
```
|
||||
|
||||
Stdio form (fallback, absolute path auto-filled from detected app location):
|
||||
```json
|
||||
{"mcpServers": {"voicebox": {
|
||||
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||
"env": {"VOICEBOX_CLIENT_ID": "claude-code"}
|
||||
}}}
|
||||
```
|
||||
|
||||
Plus the Claude-Code-specific one-liner:
|
||||
```
|
||||
claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code"
|
||||
```
|
||||
- **One-click install buttons** for known clients (v1: Claude Code via `claude mcp add` invocation, and a config-file writer for Cursor/Windsurf whose config locations are known). Each has a matching "Remove" button. Hide buttons for clients not detected on disk.
|
||||
- **Connection status** — small indicator next to each binding showing the last time that `client_id` actually called the server (rolling timestamp recorded by middleware), so users can tell their install worked.
|
||||
|
||||
## Ordered task list (shortest path first)
|
||||
|
||||
1. `fastmcp` + `sse-starlette` → `backend/requirements.txt`; install.
|
||||
2. Add `backend/mcp/{server,tools,context,resolve}.py` with the 4 tools registered as `voicebox.speak` etc. (no middleware yet — global default profile only).
|
||||
3. Migrate `app.py` to `lifespan=`; mount FastMCP at `/mcp`.
|
||||
4. **Milestone:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp` — call `voicebox.speak`, hear audio.
|
||||
5. Add `get_profile_by_name_or_id`; wire the tool's `profile` arg.
|
||||
6. `MCPClientBinding` model + migration; middleware; full `resolve_profile` precedence.
|
||||
7. `backend/routes/speak.py` — `POST /speak` REST wrapper, reusing `resolve_profile` + `speak_in_character`.
|
||||
8. `/mcp/bindings` REST + `MCPBindings.tsx` UI with HTTP and stdio copy-snippets, one-click install for detected clients, and connection-status indicators. **Users can install Voicebox as an MCP server after this step.**
|
||||
9. `backend/mcp_shim/__main__.py` + PyInstaller spec + `build_binary.py` second pass; register `voicebox-mcp` as a Tauri sidecar. (Fallback path goes live.)
|
||||
10. Events queue + `/events/speak` SSE + `DictateWindow` `speaking` pill state.
|
||||
11. `backend/mcp/README.md` quickstart.
|
||||
|
||||
Claude Code can call `voicebox.speak` after step 4 (direct HTTP, manual config). Step 8 makes that a one-click experience. Step 9 adds the stdio fallback for clients that don't speak HTTP MCP.
|
||||
|
||||
## Verification
|
||||
|
||||
- **Step 4 smoke:** `npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp`. Call `voicebox.list_profiles`, then `voicebox.speak(text="hello from mcp")`. Audio plays; generation appears in History with `source="personality_speak"` (or new `source="mcp"` if we add one).
|
||||
- **REST wrapper:** `curl -X POST http://127.0.0.1:17493/speak -d '{"text":"hi","profile":"Morgan"}'` — same behavior, same pill surface.
|
||||
- **Per-client:** open two Inspector sessions with different `X-Voicebox-Client-Id` headers, bind each to a different profile in Settings, verify distinct voices without `profile` arg.
|
||||
- **Claude Code end-to-end (HTTP):** `claude mcp add voicebox --transport http --url http://127.0.0.1:17493/mcp --header "X-Voicebox-Client-Id: claude-code"`, then ask Claude Code to speak. Pill shows `speaking: <profile>`, audio plays, capture appears in history.
|
||||
- **Stdio fallback:** manually paste the stdio snippet from Settings into a client's config, verify same behavior. `VOICEBOX_CLIENT_ID=claude-code python -m backend.mcp_shim` while backend is up; pipe a tools/list JSON-RPC in, verify response over stdout.
|
||||
- **Transcribe:** point at `/tmp/test.wav`; diff against `POST /transcribe` response.
|
||||
- **Failure modes:** kill backend mid-speak — shim must surface a JSON-RPC error, not deadlock. When backend isn't running, HTTP clients should get a clear connection-refused surfaced by the client.
|
||||
|
||||
## Risks / open decisions
|
||||
|
||||
- **`fastmcp` vs `mcp` package name** — confirm on `pip install`; APIs are near-identical, adjust imports.
|
||||
- **Lifespan migration** touches critical path (DB init, task queue, watchdog). Dev + packaged build both need a smoke after.
|
||||
- **Shim binary size** — if `mcp` pulls in enough dep weight that PyInstaller output is awkward, fall back to a Rust shim (Tauri shell is already Rust; JSON-RPC framing is trivial).
|
||||
- **Source attribution** — consider `source="mcp"` on the `Generation` model, or a dedicated `originator_client` column, if the Captures tab should filter MCP-originated generations.
|
||||
- **`audio_path` in `voicebox_transcribe`** — local-only today, but if the server ever binds beyond 127.0.0.1 we need to restrict reads to `data_dir` + user-whitelist.
|
||||
- **Auth** — none for now (127.0.0.1 only). If we bind outside, bearer token via `~/.voicebox/secret` + plumb through shim.
|
||||
- **HTTP MCP client support** — the plan leads with direct HTTP. Claude Code, Cursor, Windsurf, and VS Code MCP extensions all support it as of 2026, but if we discover an important client is stdio-only we still have the shim fallback ready.
|
||||
- **`.mcpb` desktop extension for Claude Desktop** (v2 polish) — Claude Desktop supports a double-clickable extension bundle format. Worth revisiting after v1 ships for an even cleaner install; skipped for now since Claude Desktop isn't the primary user (Claude Code + IDE users are).
|
||||
|
||||
## Critical files
|
||||
|
||||
- `backend/app.py`
|
||||
- `backend/routes/profiles.py`
|
||||
- `backend/routes/speak.py` (new)
|
||||
- `backend/database/models.py`
|
||||
- `backend/database/migrations.py`
|
||||
- `backend/services/generation.py`
|
||||
- `backend/build_binary.py`
|
||||
- `tauri/src-tauri/tauri.conf.json`
|
||||
- `tauri/src-tauri/src/main.rs` (port constant — no change, just reference)
|
||||
- `app/src/components/DictateWindow/DictateWindow.tsx`
|
||||
- `app/src/components/CapturePill/CapturePill.tsx`
|
||||
- `app/src/components/ServerSettings/`
|
||||
@@ -60,18 +60,7 @@ const TONE_CLASSES: Record<Scenario['log'][number]['tone'], string> = {
|
||||
|
||||
// ─── Console mockup ─────────────────────────────────────────────────────────
|
||||
|
||||
function AgentConsole() {
|
||||
const [idx, setIdx] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const iv = window.setInterval(() => {
|
||||
setIdx((i) => (i + 1) % SCENARIOS.length);
|
||||
}, 4200);
|
||||
return () => window.clearInterval(iv);
|
||||
}, []);
|
||||
|
||||
const scenario = SCENARIOS[idx];
|
||||
|
||||
function AgentConsole({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) {
|
||||
return (
|
||||
<div className="rounded-xl border border-app-line bg-app-darkerBox overflow-hidden shadow-[0_20px_60px_rgba(0,0,0,0.35)]">
|
||||
{/* Titlebar */}
|
||||
@@ -88,12 +77,11 @@ function AgentConsole() {
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div className="p-5 font-mono text-[12px] leading-relaxed min-h-[280px] flex flex-col">
|
||||
{/* Log lines */}
|
||||
<div className="space-y-1.5 mb-5">
|
||||
<div className="p-5 font-mono text-[12px] leading-relaxed min-h-[220px] flex flex-col">
|
||||
<div className="space-y-1.5">
|
||||
{scenario.log.map((line, i) => (
|
||||
<motion.div
|
||||
key={`${idx}-line-${i}`}
|
||||
key={`${cycleKey}-line-${i}`}
|
||||
className="flex items-start gap-2"
|
||||
initial={{ opacity: 0, y: 2 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
@@ -111,13 +99,57 @@ function AgentConsole() {
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* The pill in speaking state — the payoff */}
|
||||
{/* Idle cursor so the terminal doesn't feel empty */}
|
||||
<div className="mt-auto flex items-center gap-2 pt-4">
|
||||
<span className="text-ink-faint/50">$</span>
|
||||
<span className="inline-block h-3.5 w-[7px] bg-ink-faint/40 animate-pulse" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Desktop-floating pill stage ────────────────────────────────────────────
|
||||
|
||||
function AgentSpeakStage({ scenario, cycleKey }: { scenario: Scenario; cycleKey: number }) {
|
||||
return (
|
||||
<div
|
||||
className="relative rounded-xl border border-app-line bg-app-darkerBox/60 overflow-hidden min-h-[180px] flex-1"
|
||||
style={{
|
||||
backgroundImage: `
|
||||
linear-gradient(to right, hsl(30 10% 94% / 0.04) 1px, transparent 1px),
|
||||
linear-gradient(to bottom, hsl(30 10% 94% / 0.04) 1px, transparent 1px)
|
||||
`,
|
||||
backgroundSize: '28px 28px',
|
||||
}}
|
||||
>
|
||||
{/* Caption in the corner — "this is on the desktop, not in a terminal" */}
|
||||
<div className="absolute top-3 left-4 text-[9px] font-mono uppercase tracking-[0.22em] text-ink-faint/50">
|
||||
On your desktop
|
||||
</div>
|
||||
|
||||
{/* Voice-tinted glow behind the pill */}
|
||||
<div className="pointer-events-none absolute inset-0 flex items-center justify-center">
|
||||
<motion.div
|
||||
key={`pill-${idx}`}
|
||||
className="mt-auto self-start inline-flex items-center gap-2.5 px-3 h-9 rounded-full bg-black/55 backdrop-blur-sm shadow-[0_6px_20px_rgba(0,0,0,0.4)]"
|
||||
initial={{ opacity: 0, y: 6 }}
|
||||
key={`glow-${cycleKey}`}
|
||||
className="w-[320px] h-[140px] rounded-full blur-[70px]"
|
||||
style={{
|
||||
background: `linear-gradient(135deg, ${scenario.voiceGradient[0]}, ${scenario.voiceGradient[1]})`,
|
||||
}}
|
||||
initial={{ opacity: 0 }}
|
||||
animate={{ opacity: 0.3 }}
|
||||
transition={{ duration: 0.5 }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Pill + utterance caption */}
|
||||
<div className="absolute inset-0 flex flex-col items-center justify-center gap-3 px-6">
|
||||
<motion.div
|
||||
key={`pill-${cycleKey}`}
|
||||
className="inline-flex items-center gap-3 px-4 h-11 rounded-full bg-black/55 backdrop-blur-md shadow-[0_12px_40px_rgba(0,0,0,0.45)]"
|
||||
initial={{ opacity: 0, y: 8 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{ duration: 0.35, delay: 0.6 }}
|
||||
transition={{ duration: 0.4, delay: 0.6 }}
|
||||
>
|
||||
<div
|
||||
className="h-4 w-4 rounded-full shrink-0 ring-1 ring-white/10"
|
||||
@@ -125,19 +157,19 @@ function AgentConsole() {
|
||||
background: `linear-gradient(135deg, ${scenario.voiceGradient[0]}, ${scenario.voiceGradient[1]})`,
|
||||
}}
|
||||
/>
|
||||
<span className="text-[11px] font-medium text-foreground/90">
|
||||
<span className="text-[12px] font-medium text-foreground/90 shrink-0">
|
||||
Speaking · <span className="text-accent">{scenario.voice}</span>
|
||||
</span>
|
||||
<div className="flex items-center gap-[2px] h-4">
|
||||
<div className="flex items-center gap-[2.5px] h-5 shrink-0">
|
||||
{[0, 1, 2, 3, 4, 5].map((i) => (
|
||||
<motion.div
|
||||
key={`bar-${scenario.voice}-${i}`}
|
||||
className="w-[2px] rounded-full bg-accent"
|
||||
animate={{ height: ['4px', '12px', '6px', '10px', '4px'] }}
|
||||
className="w-[2.5px] rounded-full bg-accent"
|
||||
animate={{ height: ['5px', '14px', '7px', '12px', '5px'] }}
|
||||
transition={{
|
||||
duration: 0.9,
|
||||
duration: 1.0,
|
||||
repeat: Infinity,
|
||||
delay: i * 0.08,
|
||||
delay: i * 0.09,
|
||||
ease: 'easeInOut',
|
||||
}}
|
||||
/>
|
||||
@@ -145,13 +177,12 @@ function AgentConsole() {
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
{/* The utterance — what the agent said */}
|
||||
<motion.div
|
||||
key={`utter-${idx}`}
|
||||
className="mt-3 text-[11px] text-ink-dull"
|
||||
key={`utter-${cycleKey}`}
|
||||
className="text-[12px] text-ink-dull/80 italic text-center max-w-sm"
|
||||
initial={{ opacity: 0 }}
|
||||
animate={{ opacity: 1 }}
|
||||
transition={{ duration: 0.4, delay: 0.8 }}
|
||||
transition={{ duration: 0.4, delay: 0.9 }}
|
||||
>
|
||||
“{scenario.utterance}”
|
||||
</motion.div>
|
||||
@@ -165,8 +196,7 @@ function AgentConsole() {
|
||||
const MCP_CONFIG = `{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "voicebox",
|
||||
"args": ["mcp"]
|
||||
"url": "http://127.0.0.1:17493/mcp"
|
||||
}
|
||||
}
|
||||
}`;
|
||||
@@ -246,13 +276,24 @@ const BULLETS = [
|
||||
// ─── Section ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function AgentIntegration() {
|
||||
const [idx, setIdx] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const iv = window.setInterval(() => {
|
||||
setIdx((i) => (i + 1) % SCENARIOS.length);
|
||||
}, 4200);
|
||||
return () => window.clearInterval(iv);
|
||||
}, []);
|
||||
|
||||
const scenario = SCENARIOS[idx];
|
||||
|
||||
return (
|
||||
<section id="agents" className="border-t border-border py-24">
|
||||
<section id="mcp" className="border-t border-border py-24">
|
||||
<div className="mx-auto max-w-6xl px-6">
|
||||
{/* Header */}
|
||||
<div className="max-w-3xl mx-auto text-center mb-14">
|
||||
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-accent mb-4">
|
||||
Agents
|
||||
MCP
|
||||
</div>
|
||||
<h2 className="text-4xl md:text-5xl font-semibold tracking-tight text-foreground mb-5">
|
||||
Every agent gets a voice.
|
||||
@@ -265,10 +306,13 @@ export function AgentIntegration() {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Code + console split */}
|
||||
<div className="grid md:grid-cols-2 gap-6 mb-12">
|
||||
{/* Code (left) + console with pill stage stacked underneath (right) */}
|
||||
<div className="grid md:grid-cols-2 gap-6 mb-12 items-stretch">
|
||||
<CodePanel />
|
||||
<AgentConsole />
|
||||
<div className="flex flex-col gap-4">
|
||||
<AgentConsole scenario={scenario} cycleKey={idx} />
|
||||
<AgentSpeakStage scenario={scenario} cycleKey={idx} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Bullets */}
|
||||
|
||||
@@ -40,8 +40,28 @@ export function Footer() {
|
||||
<h4 className="text-sm font-semibold mb-3">Product</h4>
|
||||
<ul className="space-y-2 text-sm text-muted-foreground">
|
||||
<li>
|
||||
<a href="#features" className="hover:text-foreground transition-colors">
|
||||
Features
|
||||
<a href="/#features" className="hover:text-foreground transition-colors">
|
||||
Clone
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="/capture" className="hover:text-foreground transition-colors">
|
||||
Capture
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="/#mcp" className="hover:text-foreground transition-colors">
|
||||
MCP
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="/#about" className="hover:text-foreground transition-colors">
|
||||
Models
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="/#api" className="hover:text-foreground transition-colors">
|
||||
API
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
@@ -49,11 +69,6 @@ export function Footer() {
|
||||
Download
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#about" className="hover:text-foreground transition-colors">
|
||||
About
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ export function Navbar() {
|
||||
href="/#features"
|
||||
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
||||
>
|
||||
Features
|
||||
Clone
|
||||
</a>
|
||||
<a
|
||||
href="/capture"
|
||||
@@ -63,10 +63,10 @@ export function Navbar() {
|
||||
</span>
|
||||
</a>
|
||||
<a
|
||||
href="/#agents"
|
||||
href="/#mcp"
|
||||
className="rounded-md px-3 py-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground"
|
||||
>
|
||||
Agents
|
||||
MCP
|
||||
</a>
|
||||
<a
|
||||
href="/#about"
|
||||
|
||||
@@ -166,48 +166,13 @@ impl Chord {
|
||||
// Monitor
|
||||
// ========================================================================
|
||||
|
||||
/// Hardcoded Pass 1 defaults. Two right-hand modifiers so the usual left-hand
|
||||
/// shortcuts pass through unaffected. Replaced in Pass 2 by reading from the
|
||||
/// server-side `capture_settings` table via a Tauri command the frontend
|
||||
/// invokes whenever `useCaptureSettings` resolves.
|
||||
///
|
||||
/// - **macOS:** `MetaRight + AltGr` — right Command + right Option. (rdev
|
||||
/// labels right-Option as `AltGr` for Linux-convention symmetry; on macOS
|
||||
/// it's the physical right-option key.)
|
||||
/// - **Windows / Linux:** `ControlRight + ShiftRight` — right Ctrl + right
|
||||
/// Shift. Deliberately avoids `AltGr`: on international Windows layouts
|
||||
/// the OS synthesises `AltGr` as `Ctrl+Alt`, so any `AltGr`-involving
|
||||
/// default would fire on every `@`, `€`, `\` keypress on German / French
|
||||
/// / Spanish keyboards.
|
||||
pub fn default_bindings() -> Bindings {
|
||||
#[cfg(target_os = "macos")]
|
||||
let (m1, m2) = (Key::MetaRight, Key::AltGr);
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let (m1, m2) = (Key::ControlRight, Key::ShiftRight);
|
||||
|
||||
let mut b = Bindings::new();
|
||||
b.insert(ChordAction::PushToTalk, {
|
||||
let mut s = HashSet::new();
|
||||
s.insert(m1);
|
||||
s.insert(m2);
|
||||
s
|
||||
});
|
||||
b.insert(ChordAction::ToggleToTalk, {
|
||||
let mut s = HashSet::new();
|
||||
s.insert(m1);
|
||||
s.insert(m2);
|
||||
s.insert(Key::Space);
|
||||
s
|
||||
});
|
||||
b
|
||||
}
|
||||
|
||||
pub struct HotkeyMonitor {
|
||||
chord: Arc<Mutex<Chord>>,
|
||||
}
|
||||
|
||||
impl HotkeyMonitor {
|
||||
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
||||
eprintln!("[HotkeyMonitor] spawn() called with {} bindings", bindings.len());
|
||||
let chord = Arc::new(Mutex::new(Chord::new(bindings)));
|
||||
let chord_for_thread = chord.clone();
|
||||
let app_for_thread = app.clone();
|
||||
@@ -219,7 +184,9 @@ impl HotkeyMonitor {
|
||||
#[cfg(target_os = "macos")]
|
||||
rdev::set_is_main_thread(false);
|
||||
|
||||
eprintln!("[HotkeyMonitor] background thread entering rdev::listen");
|
||||
let result = listen(move |event| {
|
||||
eprintln!("[HotkeyMonitor] rdev event: {:?}", event.event_type);
|
||||
let input = match event.event_type {
|
||||
EventType::KeyPress(k) => KeyEvent::Down(k),
|
||||
EventType::KeyRelease(k) => KeyEvent::Up(k),
|
||||
@@ -231,11 +198,17 @@ impl HotkeyMonitor {
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
if !effects.is_empty() {
|
||||
eprintln!("[HotkeyMonitor] chord matched, effects: {:?}", effects);
|
||||
}
|
||||
|
||||
for effect in effects {
|
||||
apply_effect(&app_for_thread, effect);
|
||||
}
|
||||
});
|
||||
|
||||
// listen() blocks forever on success; reaching here means it errored.
|
||||
eprintln!("[HotkeyMonitor] rdev::listen returned (this only happens on error): {:?}", result);
|
||||
if let Err(err) = result {
|
||||
eprintln!(
|
||||
"HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.",
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
//! Platform permission gate for the global keyboard tap.
|
||||
//!
|
||||
//! On macOS 10.15+, creating a CGEventTap that observes keyboard events
|
||||
//! requires the host process to be listed under System Settings → Privacy &
|
||||
//! Security → Input Monitoring. Without that trust, `rdev::listen` returns
|
||||
//! immediately and no key events ever flow through the chord engine.
|
||||
//!
|
||||
//! The relevant TCC pair lives in IOKit, mirroring `AXIsProcessTrusted` /
|
||||
//! `AXIsProcessTrustedWithOptions` on the Accessibility side:
|
||||
//!
|
||||
//! - `IOHIDCheckAccess(kIOHIDRequestTypeListenEvent)` — read the current
|
||||
//! grant without prompting. We call this from the Captures settings UI
|
||||
//! so the row can show "granted" / "missing" without surprising the user.
|
||||
//! - `IOHIDRequestAccess(kIOHIDRequestTypeListenEvent)` — fire the
|
||||
//! "Voicebox would like to receive keystrokes from any application"
|
||||
//! dialog and add Voicebox to the Input Monitoring pane (toggle off).
|
||||
//! Returns true when access is already granted; otherwise returns false
|
||||
//! and queues the prompt. The user still has to flip the toggle on; this
|
||||
//! just gets us into the list.
|
||||
//!
|
||||
//! `enable_hotkey` calls `request` on first invocation so the prompt fires
|
||||
//! from a deterministic, user-initiated point (the Captures toggle) instead
|
||||
//! of as a side-effect of `rdev::listen` creating its CGEventTap.
|
||||
//!
|
||||
//! Windows / Linux don't gate keyboard taps behind a TCC-style permission,
|
||||
//! so those branches return `true`.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod ffi {
|
||||
use std::os::raw::c_uint;
|
||||
|
||||
/// `kIOHIDRequestTypeListenEvent` from `<IOKit/hidsystem/IOHIDLib.h>` —
|
||||
/// the request-type discriminator for "I want to read keyboard / mouse
|
||||
/// events created by other processes."
|
||||
pub const REQUEST_TYPE_LISTEN_EVENT: c_uint = 1;
|
||||
|
||||
/// `kIOHIDAccessTypeGranted` from `IOHIDLib.h`. The other values are
|
||||
/// `Denied = 1` and `Unknown = 2`; we only ever care about the granted
|
||||
/// case so they don't get their own constants.
|
||||
pub const ACCESS_TYPE_GRANTED: c_uint = 0;
|
||||
|
||||
#[link(name = "IOKit", kind = "framework")]
|
||||
extern "C" {
|
||||
/// Returns the current access state as an `IOHIDAccessType` enum
|
||||
/// (Granted=0, Denied=1, Unknown=2). No prompt side-effect.
|
||||
///
|
||||
/// Declared as `c_uint` rather than `bool`: the C signature returns
|
||||
/// the full enum, and reading a 3-valued enum into Rust's 1-bit
|
||||
/// `bool` is undefined behaviour that silently inverts our gate.
|
||||
pub fn IOHIDCheckAccess(request_type: c_uint) -> c_uint;
|
||||
|
||||
/// Returns true when access is already granted; otherwise queues
|
||||
/// the system prompt and returns false synchronously. Safe to call
|
||||
/// repeatedly — once the entry exists in the Input Monitoring pane
|
||||
/// macOS won't re-prompt. Real `Boolean` (UInt8) return on the C
|
||||
/// side, so `bool` here is correct.
|
||||
pub fn IOHIDRequestAccess(request_type: c_uint) -> bool;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn is_trusted() -> bool {
|
||||
unsafe { ffi::IOHIDCheckAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) == ffi::ACCESS_TYPE_GRANTED }
|
||||
}
|
||||
|
||||
/// Fire the Input Monitoring prompt if not already granted. Returns the
|
||||
/// current grant state; a `false` here means the prompt was queued and the
|
||||
/// user needs to flip the toggle in System Settings before key events flow.
|
||||
#[cfg(target_os = "macos")]
|
||||
pub fn request() -> bool {
|
||||
unsafe { ffi::IOHIDRequestAccess(ffi::REQUEST_TYPE_LISTEN_EVENT) }
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn is_trusted() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
pub fn request() -> bool {
|
||||
true
|
||||
}
|
||||
+193
-30
@@ -8,6 +8,7 @@ mod clipboard;
|
||||
mod focus_capture;
|
||||
#[cfg(desktop)]
|
||||
mod hotkey_monitor;
|
||||
mod input_monitoring;
|
||||
#[cfg(desktop)]
|
||||
mod key_codes;
|
||||
mod synthetic_keys;
|
||||
@@ -57,6 +58,39 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewW
|
||||
Ok(window)
|
||||
}
|
||||
|
||||
/// Position, undo click-through, and show the dictate pill window.
|
||||
///
|
||||
/// The hide path parks the window at (-10_000, -10_000) and toggles
|
||||
/// `ignore_cursor_events(true)` so invisible click targets don't leak; we
|
||||
/// undo both here. Mirrors the logic the hotkey_monitor's
|
||||
/// `Effect::StartRecording` path runs, minus the focus snapshot — this is
|
||||
/// for agent-initiated speech, not dictation, so there's no focused text
|
||||
/// field to paste into.
|
||||
#[cfg(desktop)]
|
||||
pub fn show_dictate_window(app: &tauri::AppHandle) {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
// current_monitor() returns None when the window has been parked
|
||||
// off any display by the hide path; fall back to the primary.
|
||||
let monitor = window
|
||||
.current_monitor()
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| window.primary_monitor().ok().flatten());
|
||||
if let Some(monitor) = monitor {
|
||||
let monitor_pos = monitor.position();
|
||||
let monitor_size = monitor.size();
|
||||
if let Ok(win_size) = window.outer_size() {
|
||||
let x = monitor_pos.x
|
||||
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||
let _ = window.set_position(PhysicalPosition::new(x, y));
|
||||
}
|
||||
}
|
||||
let _ = window.set_ignore_cursor_events(false);
|
||||
let _ = window.show();
|
||||
}
|
||||
}
|
||||
|
||||
const LEGACY_PORT: u16 = 8000;
|
||||
const SERVER_PORT: u16 = 17493;
|
||||
|
||||
@@ -791,22 +825,34 @@ fn check_accessibility_permission() -> bool {
|
||||
accessibility::is_trusted()
|
||||
}
|
||||
|
||||
/// Push a new chord configuration into the running `HotkeyMonitor`. The
|
||||
/// frontend calls this both at startup (replaying the saved chord from
|
||||
/// capture_settings) and any time the user edits the chord in the picker —
|
||||
/// no app restart needed because the engine swap is atomic under the
|
||||
/// monitor's mutex.
|
||||
///
|
||||
/// Returns an error when a key name doesn't map to an `rdev::Key`, so the
|
||||
/// picker UI can surface "this key isn't supported" instead of silently
|
||||
/// dropping it from the chord.
|
||||
#[cfg(desktop)]
|
||||
/// Reports whether the process can observe global keyboard events. Read by
|
||||
/// the Captures settings UI to surface a "missing — open Settings" hint
|
||||
/// beside the hotkey toggle. No prompt side-effect.
|
||||
#[command]
|
||||
fn update_chord_bindings(
|
||||
monitor: State<'_, hotkey_monitor::HotkeyMonitor>,
|
||||
push_to_talk: Vec<String>,
|
||||
toggle_to_talk: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
fn check_input_monitoring_permission() -> bool {
|
||||
input_monitoring::is_trusted()
|
||||
}
|
||||
|
||||
/// Holds the lazily-spawned global hotkey monitor. The monitor is `None`
|
||||
/// until the user opts in via the Captures settings toggle — that opt-in is
|
||||
/// what triggers the macOS Input Monitoring TCC prompt, so a fresh-install
|
||||
/// user who never enables the hotkey never sees the prompt.
|
||||
///
|
||||
/// Once spawned, the monitor stays alive for the rest of the process: rdev's
|
||||
/// `listen` blocks forever and offers no stop signal. "Disable" therefore
|
||||
/// swaps the chord engine to empty bindings (matches nothing, fires nothing)
|
||||
/// rather than tearing down the CGEventTap.
|
||||
#[cfg(desktop)]
|
||||
#[derive(Default)]
|
||||
pub struct HotkeyState {
|
||||
monitor: Mutex<Option<hotkey_monitor::HotkeyMonitor>>,
|
||||
}
|
||||
|
||||
#[cfg(desktop)]
|
||||
fn build_chord_bindings(
|
||||
push_to_talk: &[String],
|
||||
toggle_to_talk: &[String],
|
||||
) -> Result<hotkey_monitor::Bindings, String> {
|
||||
use hotkey_monitor::{Bindings, ChordAction};
|
||||
use rdev::Key;
|
||||
use std::collections::HashSet;
|
||||
@@ -824,14 +870,103 @@ fn update_chord_bindings(
|
||||
Ok(chord)
|
||||
}
|
||||
|
||||
let push_chord = build_chord("push-to-talk", &push_to_talk)?;
|
||||
let toggle_chord = build_chord("toggle-to-talk", &toggle_to_talk)?;
|
||||
let push_chord = build_chord("push-to-talk", push_to_talk)?;
|
||||
let toggle_chord = build_chord("toggle-to-talk", toggle_to_talk)?;
|
||||
|
||||
let mut bindings = Bindings::new();
|
||||
bindings.insert(ChordAction::PushToTalk, push_chord);
|
||||
bindings.insert(ChordAction::ToggleToTalk, toggle_chord);
|
||||
Ok(bindings)
|
||||
}
|
||||
|
||||
monitor.update_bindings(bindings);
|
||||
/// Spawn the global hotkey monitor on first call; subsequent calls just push
|
||||
/// the new bindings into the existing monitor. Idempotent on purpose — the
|
||||
/// frontend invokes this both at startup (when `capture_settings.hotkey_enabled`
|
||||
/// is true) and from the settings toggle.
|
||||
///
|
||||
/// On macOS this is the call that triggers the "Voicebox would like to receive
|
||||
/// keystrokes from any application" TCC prompt, since `rdev::listen` creates
|
||||
/// the CGEventTap inside `HotkeyMonitor::spawn`.
|
||||
#[cfg(desktop)]
|
||||
#[command]
|
||||
fn enable_hotkey(
|
||||
app: tauri::AppHandle,
|
||||
state: State<'_, HotkeyState>,
|
||||
push_to_talk: Vec<String>,
|
||||
toggle_to_talk: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("[enable_hotkey] called: push={:?}, toggle={:?}", push_to_talk, toggle_to_talk);
|
||||
let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?;
|
||||
|
||||
// Fire the Input Monitoring TCC prompt explicitly from the user's
|
||||
// toggle click, before rdev::listen would do it implicitly via
|
||||
// CGEventTap creation. Two reasons: (1) the prompt timing becomes
|
||||
// deterministic — it appears in response to a click instead of as a
|
||||
// mysterious side-effect of "the app started"; (2) on subsequent
|
||||
// launches we can short-circuit the spawn entirely if the user
|
||||
// revoked the grant, instead of leaning on rdev silently failing.
|
||||
// The call returns the current grant state; we ignore it because
|
||||
// rdev::listen will surface its own error via stderr, and the
|
||||
// settings UI polls `check_input_monitoring_permission` separately.
|
||||
let granted = input_monitoring::request();
|
||||
eprintln!("[enable_hotkey] IOHIDRequestAccess returned granted={}", granted);
|
||||
eprintln!("[enable_hotkey] IOHIDCheckAccess says trusted={}", input_monitoring::is_trusted());
|
||||
|
||||
// The dictate pill webview must exist before the first chord fires so it
|
||||
// can subscribe to `dictate:start`. Build it here (idempotent — Tauri
|
||||
// returns the existing window when one with this label already exists).
|
||||
if app.get_webview_window(DICTATE_WINDOW_LABEL).is_none() {
|
||||
if let Err(e) = build_dictate_window(&app) {
|
||||
eprintln!("Failed to build dictate window: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
let mut slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||
match slot.as_ref() {
|
||||
Some(monitor) => monitor.update_bindings(bindings),
|
||||
None => {
|
||||
*slot = Some(hotkey_monitor::HotkeyMonitor::spawn(app, bindings));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Quiet the global hotkey by swapping the chord engine to empty bindings.
|
||||
/// The CGEventTap stays alive (rdev::listen has no stop) but the chord state
|
||||
/// machine matches nothing, so no `dictate:*` events fire and the dictate
|
||||
/// pill never shows. A subsequent `enable_hotkey` call re-arms it without
|
||||
/// re-prompting for permission.
|
||||
#[cfg(desktop)]
|
||||
#[command]
|
||||
fn disable_hotkey(state: State<'_, HotkeyState>) -> Result<(), String> {
|
||||
let slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||
if let Some(monitor) = slot.as_ref() {
|
||||
monitor.update_bindings(hotkey_monitor::Bindings::new());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Push a new chord configuration into the running `HotkeyMonitor`. Called
|
||||
/// by the chord-picker UI when the user edits the chord. No-ops when the
|
||||
/// monitor isn't spawned — the picker is gated behind the enable toggle, so
|
||||
/// this can only happen if the frontend races; the next `enable_hotkey` will
|
||||
/// pick up the saved chords.
|
||||
///
|
||||
/// Returns an error when a key name doesn't map to an `rdev::Key`, so the
|
||||
/// picker UI can surface "this key isn't supported" instead of silently
|
||||
/// dropping it from the chord.
|
||||
#[cfg(desktop)]
|
||||
#[command]
|
||||
fn update_chord_bindings(
|
||||
state: State<'_, HotkeyState>,
|
||||
push_to_talk: Vec<String>,
|
||||
toggle_to_talk: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?;
|
||||
let slot = state.monitor.lock().map_err(|e| e.to_string())?;
|
||||
if let Some(monitor) = slot.as_ref() {
|
||||
monitor.update_bindings(bindings);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -855,6 +990,26 @@ fn open_accessibility_settings(app: tauri::AppHandle) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Open the Privacy & Security → Input Monitoring pane in System Settings.
|
||||
/// Used by the Captures settings UI when the toggle is on but the grant
|
||||
/// is missing, so the user can flip the system toggle without hunting.
|
||||
#[command]
|
||||
fn open_input_monitoring_settings(app: tauri::AppHandle) -> Result<(), String> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let url = "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent";
|
||||
app.shell()
|
||||
.open(url, None)
|
||||
.map_err(|e| format!("Failed to open Input Monitoring settings: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
let _ = app;
|
||||
Err("Input Monitoring settings pane is only implemented on macOS".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Deliver `text` into the UI that had focus when the chord fired.
|
||||
///
|
||||
/// Pipeline: activate the captured PID → settle → save the user's
|
||||
@@ -1044,18 +1199,12 @@ pub fn run() {
|
||||
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
|
||||
app.handle().plugin(tauri_plugin_process::init())?;
|
||||
|
||||
if let Err(e) = build_dictate_window(app.handle()) {
|
||||
eprintln!("Failed to pre-create dictate window: {}", e);
|
||||
}
|
||||
|
||||
let monitor = hotkey_monitor::HotkeyMonitor::spawn(
|
||||
app.handle().clone(),
|
||||
hotkey_monitor::default_bindings(),
|
||||
);
|
||||
// Stored as state so the chord-picker UI can call
|
||||
// `update_chord_bindings` to live-swap the engine's chords
|
||||
// without restarting the listener thread.
|
||||
app.manage(monitor);
|
||||
// HotkeyMonitor is spawned lazily via the `enable_hotkey`
|
||||
// command — see HotkeyState. The dictate pill webview is
|
||||
// built in the same lazy path so we don't pay setup cost
|
||||
// (and don't trigger the macOS Input Monitoring TCC prompt)
|
||||
// for users who never enable the global hotkey.
|
||||
app.manage(HotkeyState::default());
|
||||
|
||||
// The frontend emits `dictate:hide` whenever the pill cycle
|
||||
// finishes (rest-fade → hidden). `hide()` alone has been
|
||||
@@ -1073,6 +1222,16 @@ pub fn run() {
|
||||
let _ = window.hide();
|
||||
}
|
||||
});
|
||||
|
||||
// Agent-initiated speech (voicebox.speak over MCP or POST /speak)
|
||||
// pops the pill up so the user can see what's coming out of their
|
||||
// machine. The DictateWindow subscribes to /events/speak via SSE
|
||||
// and emits `dictate:show` on speak-start; we repeat the same
|
||||
// position+show dance the hotkey path uses.
|
||||
let handle_for_show = app.handle().clone();
|
||||
app.handle().listen("dictate:show", move |_event| {
|
||||
show_dictate_window(&handle_for_show);
|
||||
});
|
||||
}
|
||||
|
||||
// Hide title bar icon on Windows
|
||||
@@ -1148,8 +1307,12 @@ pub fn run() {
|
||||
debug_capture_focus,
|
||||
debug_focus_roundtrip,
|
||||
check_accessibility_permission,
|
||||
check_input_monitoring_permission,
|
||||
open_accessibility_settings,
|
||||
open_input_monitoring_settings,
|
||||
paste_final_text,
|
||||
enable_hotkey,
|
||||
disable_hotkey,
|
||||
update_chord_bindings
|
||||
])
|
||||
.on_window_event({
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"active": true,
|
||||
"targets": "all",
|
||||
"createUpdaterArtifacts": "v1Compatible",
|
||||
"externalBin": ["binaries/voicebox-server"],
|
||||
"externalBin": ["binaries/voicebox-server", "binaries/voicebox-mcp"],
|
||||
"icon": [
|
||||
"icons/32x32.png",
|
||||
"icons/128x128.png",
|
||||
|
||||
Reference in New Issue
Block a user