feat(mcp): local MCP server exposes voicebox.* tools to AI agents

Mounts FastMCP at /mcp (Streamable HTTP) so Claude Code, Cursor,
Windsurf, and the VS Code MCP extensions can call voicebox.speak,
voicebox.transcribe, voicebox.list_captures, and voicebox.list_profiles
against the running Voicebox server.

Backend
- new backend/mcp_server package (tools, middleware, profile resolve,
  pub/sub events); named mcp_server to avoid shadowing the installed mcp
  PyPI package FastMCP imports internally
- app.py migrated from @app.on_event to lifespan= so FastMCP's session
  manager cohabits with Voicebox's startup/shutdown
- new MCPClientBinding table + /mcp/bindings CRUD; ClientIdMiddleware
  reads X-Voicebox-Client-Id into a ContextVar and stamps last_seen_at
- profile resolution precedence: explicit -> per-client binding ->
  capture_settings.default_playback_voice_id
- POST /speak REST wrapper for non-MCP callers (shell, ACP, A2A)
- GET /events/speak SSE broadcasts speak-start / speak-end so the pill
  surfaces agent-initiated speech
- backend/mcp_shim proxy (plain httpx) for stdio-only MCP clients
- PyInstaller spec updates + new --shim build target (~18 MB)

Frontend
- Settings -> MCP page with HTTP / stdio / claude-mcp-add copy snippets,
  default voice picker, per-client bindings table, connection status
- useMCPBindings, useSpeakEvents hooks
- CapturePill gains 'speaking' state; DictateWindow subscribes to SSE
  and emits dictate:show so the Rust side surfaces the pill window

Native
- tauri.conf.json externalBin now includes voicebox-mcp
- show_dictate_window helper + dictate:show listener in main.rs
- (also in this commit: InputMonitoringGate UX, hotkey_monitor tweaks,
  landing footer/navbar updates, new overview docs for captures /
  dictation / mcp-server / voice-personalities)

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
2026-04-22 22:05:30 -07:00
co-authored by Claude Opus 4.7
parent 87c582ad54
commit 0cef2c9fe1
52 changed files with 4094 additions and 336 deletions
+120 -2
View File
@@ -295,6 +295,28 @@ def build_server(cuda=False):
"unidic_lite",
"--hidden-import",
"loguru",
# MCP server — Streamable-HTTP endpoint and the 4 voicebox.* tools.
# FastMCP pulls in a chain of deps (mcp, cyclopts, openapi-pydantic,
# etc.) that don't auto-discover cleanly under PyInstaller, so we
# collect them whole. Small compared to torch.
"--hidden-import",
"backend.mcp_server",
"--hidden-import",
"backend.mcp_server.server",
"--hidden-import",
"backend.mcp_server.tools",
"--hidden-import",
"backend.mcp_server.context",
"--hidden-import",
"backend.mcp_server.resolve",
"--hidden-import",
"backend.mcp_server.events",
"--collect-all",
"fastmcp",
"--collect-all",
"mcp",
"--hidden-import",
"sse_starlette",
]
)
@@ -447,12 +469,108 @@ def build_server(cuda=False):
logger.info("Binary built in %s", backend_dir / "dist" / binary_name)
def build_shim():
"""Build the voicebox-mcp stdio shim as a tiny standalone binary.
This is the bridge for MCP clients that only speak stdio — it proxies
JSON-RPC to the main voicebox-server's /mcp endpoint. Keep it small: no
torch, no ML deps, just httpx + asyncio.
"""
backend_dir = Path(__file__).parent
args = [
"mcp_shim/__main__.py",
"--onefile",
"--name",
"voicebox-mcp",
# Stdio-only — no console hiding needed on Windows since the parent
# MCP client is spawning this as a child process and wants stdio.
"--hidden-import",
"backend.mcp_shim",
"--hidden-import",
"backend.mcp_shim.__main__",
"--hidden-import",
"httpx",
"--hidden-import",
"httpx._transports.default",
"--hidden-import",
"anyio",
# Exclude everything heavy that httpx/asyncio don't actually need so
# the binary stays tiny (~15 MB instead of ~400 MB).
"--exclude-module",
"torch",
"--exclude-module",
"transformers",
"--exclude-module",
"mlx",
"--exclude-module",
"mlx_audio",
"--exclude-module",
"qwen_tts",
"--exclude-module",
"chatterbox",
"--exclude-module",
"zipvoice",
"--exclude-module",
"tada",
"--exclude-module",
"kokoro",
"--exclude-module",
"misaki",
"--exclude-module",
"spacy",
"--exclude-module",
"librosa",
"--exclude-module",
"numba",
"--exclude-module",
"numpy",
"--exclude-module",
"pedalboard",
"--exclude-module",
"fastapi",
"--exclude-module",
"uvicorn",
"--exclude-module",
"sqlalchemy",
"--exclude-module",
"fastmcp",
"--exclude-module",
"mcp",
]
dist_dir = str(backend_dir / "dist")
build_dir = str(backend_dir / "build")
args.extend(
[
"--distpath",
dist_dir,
"--workpath",
build_dir,
"--noconfirm",
"--clean",
]
)
os.chdir(backend_dir)
PyInstaller.__main__.run(args)
logger.info("Shim built: %s", backend_dir / "dist" / "voicebox-mcp")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Build voicebox-server binary")
parser = argparse.ArgumentParser(description="Build voicebox binaries")
parser.add_argument(
"--cuda",
action="store_true",
help="Build CUDA-enabled binary (voicebox-server-cuda)",
)
parser.add_argument(
"--shim",
action="store_true",
help="Build the voicebox-mcp stdio shim binary instead of the server",
)
cli_args = parser.parse_args()
build_server(cuda=cli_args.cuda)
if cli_args.shim:
build_shim()
else:
build_server(cuda=cli_args.cuda)