mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-19 06:40:38 -07:00
feat(mcp): local MCP server exposes voicebox.* tools to AI agents
Mounts FastMCP at /mcp (Streamable HTTP) so Claude Code, Cursor, Windsurf, and the VS Code MCP extensions can call voicebox.speak, voicebox.transcribe, voicebox.list_captures, and voicebox.list_profiles against the running Voicebox server. Backend - new backend/mcp_server package (tools, middleware, profile resolve, pub/sub events); named mcp_server to avoid shadowing the installed mcp PyPI package FastMCP imports internally - app.py migrated from @app.on_event to lifespan= so FastMCP's session manager cohabits with Voicebox's startup/shutdown - new MCPClientBinding table + /mcp/bindings CRUD; ClientIdMiddleware reads X-Voicebox-Client-Id into a ContextVar and stamps last_seen_at - profile resolution precedence: explicit -> per-client binding -> capture_settings.default_playback_voice_id - POST /speak REST wrapper for non-MCP callers (shell, ACP, A2A) - GET /events/speak SSE broadcasts speak-start / speak-end so the pill surfaces agent-initiated speech - backend/mcp_shim proxy (plain httpx) for stdio-only MCP clients - PyInstaller spec updates + new --shim build target (~18 MB) Frontend - Settings -> MCP page with HTTP / stdio / claude-mcp-add copy snippets, default voice picker, per-client bindings table, connection status - useMCPBindings, useSpeakEvents hooks - CapturePill gains 'speaking' state; DictateWindow subscribes to SSE and emits dictate:show so the Rust side surfaces the pill window Native - tauri.conf.json externalBin now includes voicebox-mcp - show_dictate_window helper + dictate:show listener in main.rs - (also in this commit: InputMonitoringGate UX, hotkey_monitor tweaks, landing footer/navbar updates, new overview docs for captures / dictation / mcp-server / voice-personalities) Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
87c582ad54
commit
0cef2c9fe1
+100
-81
@@ -4,6 +4,7 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -68,15 +69,36 @@ def safe_content_disposition(disposition_type: str, filename: str) -> str:
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application."""
|
||||
from .mcp_server.server import build_mcp_server
|
||||
from .mcp_server.context import ClientIdMiddleware
|
||||
|
||||
# Build the MCP app up-front so we can wire its lifespan into FastAPI's —
|
||||
# FastMCP's Streamable HTTP transport only works if its session manager
|
||||
# runs inside the parent ASGI lifespan.
|
||||
mcp = build_mcp_server()
|
||||
mcp_app = mcp.http_app(path="/", transport="http")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
await _run_startup(app)
|
||||
async with mcp_app.router.lifespan_context(app):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await _run_shutdown()
|
||||
|
||||
application = FastAPI(
|
||||
title="voicebox API",
|
||||
description="Production-quality Qwen3-TTS voice cloning API",
|
||||
version=__version__,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
_configure_cors(application)
|
||||
application.add_middleware(ClientIdMiddleware)
|
||||
register_routers(application)
|
||||
_register_lifecycle(application)
|
||||
application.mount("/mcp", mcp_app)
|
||||
logger.info("MCP: mounted at /mcp")
|
||||
_mount_frontend(application)
|
||||
|
||||
return application
|
||||
@@ -179,107 +201,104 @@ def _get_gpu_status() -> str:
|
||||
return "None (CPU only)"
|
||||
|
||||
|
||||
def _register_lifecycle(application: FastAPI) -> None:
|
||||
"""Attach startup and shutdown event handlers."""
|
||||
async def _run_startup(application: FastAPI) -> None:
|
||||
"""Database init, warnings, model-cache prep. Runs on lifespan entry."""
|
||||
import platform
|
||||
import sys
|
||||
|
||||
@application.on_event("startup")
|
||||
async def startup_event():
|
||||
import platform
|
||||
import sys
|
||||
logger.info("Voicebox v%s starting up", __version__)
|
||||
logger.info(
|
||||
"Python %s on %s %s (%s)",
|
||||
sys.version.split()[0],
|
||||
platform.system(),
|
||||
platform.release(),
|
||||
platform.machine(),
|
||||
)
|
||||
|
||||
logger.info("Voicebox v%s starting up", __version__)
|
||||
logger.info(
|
||||
"Python %s on %s %s (%s)",
|
||||
sys.version.split()[0],
|
||||
platform.system(),
|
||||
platform.release(),
|
||||
platform.machine(),
|
||||
)
|
||||
database.init_db()
|
||||
|
||||
database.init_db()
|
||||
from .database.session import _db_path
|
||||
|
||||
from .database.session import _db_path
|
||||
logger.info("Database: %s", _db_path)
|
||||
logger.info("Data directory: %s", config.get_data_dir())
|
||||
|
||||
logger.info("Database: %s", _db_path)
|
||||
logger.info("Data directory: %s", config.get_data_dir())
|
||||
init_queue()
|
||||
|
||||
init_queue()
|
||||
# Mark stale "generating" records as failed -- leftovers from a killed process
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
# Mark stale "generating" records as failed -- leftovers from a killed process
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
db = next(get_db())
|
||||
try:
|
||||
result = db.execute(
|
||||
sa_text(
|
||||
"UPDATE generations SET status = 'failed', "
|
||||
"error = 'Server was shut down during generation' "
|
||||
"WHERE status IN ('generating', 'loading_model')"
|
||||
)
|
||||
db = next(get_db())
|
||||
try:
|
||||
result = db.execute(
|
||||
sa_text(
|
||||
"UPDATE generations SET status = 'failed', "
|
||||
"error = 'Server was shut down during generation' "
|
||||
"WHERE status IN ('generating', 'loading_model')"
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
logger.info("Marked %d stale generation(s) as failed", result.rowcount)
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
logger.info("Marked %d stale generation(s) as failed", result.rowcount)
|
||||
|
||||
from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration
|
||||
from .database import VoiceProfile as DBVoiceProfile, Generation as DBGeneration
|
||||
|
||||
profile_count = db.query(DBVoiceProfile).count()
|
||||
generation_count = db.query(DBGeneration).count()
|
||||
logger.info("Profiles: %d, Generations: %d", profile_count, generation_count)
|
||||
profile_count = db.query(DBVoiceProfile).count()
|
||||
generation_count = db.query(DBGeneration).count()
|
||||
logger.info("Profiles: %d, Generations: %d", profile_count, generation_count)
|
||||
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.warning("Could not clean up stale generations: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.warning("Could not clean up stale generations: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
backend_type = get_backend_type()
|
||||
logger.info("Backend: %s", backend_type.upper())
|
||||
logger.info("GPU: %s", _get_gpu_status())
|
||||
backend_type = get_backend_type()
|
||||
logger.info("Backend: %s", backend_type.upper())
|
||||
logger.info("GPU: %s", _get_gpu_status())
|
||||
|
||||
# Warn if GPU architecture is not supported by this PyTorch build
|
||||
from .backends.base import check_cuda_compatibility
|
||||
from .backends.base import check_cuda_compatibility
|
||||
|
||||
_compatible, _cuda_warning = check_cuda_compatibility()
|
||||
if not _compatible:
|
||||
logger.warning("GPU COMPATIBILITY: %s", _cuda_warning)
|
||||
_compatible, _cuda_warning = check_cuda_compatibility()
|
||||
if not _compatible:
|
||||
logger.warning("GPU COMPATIBILITY: %s", _cuda_warning)
|
||||
|
||||
from .services.cuda import check_and_update_cuda_binary
|
||||
from .services.cuda import check_and_update_cuda_binary
|
||||
|
||||
create_background_task(check_and_update_cuda_binary())
|
||||
create_background_task(check_and_update_cuda_binary())
|
||||
|
||||
try:
|
||||
progress_manager = get_progress_manager()
|
||||
progress_manager._set_main_loop(asyncio.get_running_loop())
|
||||
except Exception as e:
|
||||
logger.warning("Could not initialize progress manager event loop: %s", e)
|
||||
try:
|
||||
progress_manager = get_progress_manager()
|
||||
progress_manager._set_main_loop(asyncio.get_running_loop())
|
||||
except Exception as e:
|
||||
logger.warning("Could not initialize progress manager event loop: %s", e)
|
||||
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
try:
|
||||
from huggingface_hub import constants as hf_constants
|
||||
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Model cache: %s", cache_dir)
|
||||
except Exception as e:
|
||||
logger.warning("Could not create HuggingFace cache directory: %s", e)
|
||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Model cache: %s", cache_dir)
|
||||
except Exception as e:
|
||||
logger.warning("Could not create HuggingFace cache directory: %s", e)
|
||||
|
||||
logger.info("Ready")
|
||||
logger.info("Ready")
|
||||
|
||||
@application.on_event("shutdown")
|
||||
async def shutdown_event():
|
||||
logger.info("Voicebox server shutting down...")
|
||||
try:
|
||||
tts.unload_tts_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload TTS model")
|
||||
try:
|
||||
transcribe.unload_whisper_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload Whisper model")
|
||||
try:
|
||||
llm.unload_llm_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload LLM model")
|
||||
|
||||
async def _run_shutdown() -> None:
|
||||
"""Unload models on lifespan exit."""
|
||||
logger.info("Voicebox server shutting down...")
|
||||
try:
|
||||
tts.unload_tts_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload TTS model")
|
||||
try:
|
||||
transcribe.unload_whisper_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload Whisper model")
|
||||
try:
|
||||
llm.unload_llm_model()
|
||||
except Exception:
|
||||
logger.exception("Failed to unload LLM model")
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
+120
-2
@@ -295,6 +295,28 @@ def build_server(cuda=False):
|
||||
"unidic_lite",
|
||||
"--hidden-import",
|
||||
"loguru",
|
||||
# MCP server — Streamable-HTTP endpoint and the 4 voicebox.* tools.
|
||||
# FastMCP pulls in a chain of deps (mcp, cyclopts, openapi-pydantic,
|
||||
# etc.) that don't auto-discover cleanly under PyInstaller, so we
|
||||
# collect them whole. Small compared to torch.
|
||||
"--hidden-import",
|
||||
"backend.mcp_server",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.server",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.tools",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.context",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.resolve",
|
||||
"--hidden-import",
|
||||
"backend.mcp_server.events",
|
||||
"--collect-all",
|
||||
"fastmcp",
|
||||
"--collect-all",
|
||||
"mcp",
|
||||
"--hidden-import",
|
||||
"sse_starlette",
|
||||
]
|
||||
)
|
||||
|
||||
@@ -447,12 +469,108 @@ def build_server(cuda=False):
|
||||
logger.info("Binary built in %s", backend_dir / "dist" / binary_name)
|
||||
|
||||
|
||||
def build_shim():
|
||||
"""Build the voicebox-mcp stdio shim as a tiny standalone binary.
|
||||
|
||||
This is the bridge for MCP clients that only speak stdio — it proxies
|
||||
JSON-RPC to the main voicebox-server's /mcp endpoint. Keep it small: no
|
||||
torch, no ML deps, just httpx + asyncio.
|
||||
"""
|
||||
backend_dir = Path(__file__).parent
|
||||
|
||||
args = [
|
||||
"mcp_shim/__main__.py",
|
||||
"--onefile",
|
||||
"--name",
|
||||
"voicebox-mcp",
|
||||
# Stdio-only — no console hiding needed on Windows since the parent
|
||||
# MCP client is spawning this as a child process and wants stdio.
|
||||
"--hidden-import",
|
||||
"backend.mcp_shim",
|
||||
"--hidden-import",
|
||||
"backend.mcp_shim.__main__",
|
||||
"--hidden-import",
|
||||
"httpx",
|
||||
"--hidden-import",
|
||||
"httpx._transports.default",
|
||||
"--hidden-import",
|
||||
"anyio",
|
||||
# Exclude everything heavy that httpx/asyncio don't actually need so
|
||||
# the binary stays tiny (~15 MB instead of ~400 MB).
|
||||
"--exclude-module",
|
||||
"torch",
|
||||
"--exclude-module",
|
||||
"transformers",
|
||||
"--exclude-module",
|
||||
"mlx",
|
||||
"--exclude-module",
|
||||
"mlx_audio",
|
||||
"--exclude-module",
|
||||
"qwen_tts",
|
||||
"--exclude-module",
|
||||
"chatterbox",
|
||||
"--exclude-module",
|
||||
"zipvoice",
|
||||
"--exclude-module",
|
||||
"tada",
|
||||
"--exclude-module",
|
||||
"kokoro",
|
||||
"--exclude-module",
|
||||
"misaki",
|
||||
"--exclude-module",
|
||||
"spacy",
|
||||
"--exclude-module",
|
||||
"librosa",
|
||||
"--exclude-module",
|
||||
"numba",
|
||||
"--exclude-module",
|
||||
"numpy",
|
||||
"--exclude-module",
|
||||
"pedalboard",
|
||||
"--exclude-module",
|
||||
"fastapi",
|
||||
"--exclude-module",
|
||||
"uvicorn",
|
||||
"--exclude-module",
|
||||
"sqlalchemy",
|
||||
"--exclude-module",
|
||||
"fastmcp",
|
||||
"--exclude-module",
|
||||
"mcp",
|
||||
]
|
||||
|
||||
dist_dir = str(backend_dir / "dist")
|
||||
build_dir = str(backend_dir / "build")
|
||||
args.extend(
|
||||
[
|
||||
"--distpath",
|
||||
dist_dir,
|
||||
"--workpath",
|
||||
build_dir,
|
||||
"--noconfirm",
|
||||
"--clean",
|
||||
]
|
||||
)
|
||||
|
||||
os.chdir(backend_dir)
|
||||
PyInstaller.__main__.run(args)
|
||||
logger.info("Shim built: %s", backend_dir / "dist" / "voicebox-mcp")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Build voicebox-server binary")
|
||||
parser = argparse.ArgumentParser(description="Build voicebox binaries")
|
||||
parser.add_argument(
|
||||
"--cuda",
|
||||
action="store_true",
|
||||
help="Build CUDA-enabled binary (voicebox-server-cuda)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shim",
|
||||
action="store_true",
|
||||
help="Build the voicebox-mcp stdio shim binary instead of the server",
|
||||
)
|
||||
cli_args = parser.parse_args()
|
||||
build_server(cuda=cli_args.cuda)
|
||||
if cli_args.shim:
|
||||
build_shim()
|
||||
else:
|
||||
build_server(cuda=cli_args.cuda)
|
||||
|
||||
@@ -15,6 +15,7 @@ from .models import (
|
||||
Generation,
|
||||
GenerationSettings,
|
||||
GenerationVersion,
|
||||
MCPClientBinding,
|
||||
ProfileChannelMapping,
|
||||
ProfileSample,
|
||||
Project,
|
||||
@@ -35,6 +36,7 @@ __all__ = [
|
||||
"Generation",
|
||||
"GenerationSettings",
|
||||
"GenerationVersion",
|
||||
"MCPClientBinding",
|
||||
"ProfileChannelMapping",
|
||||
"ProfileSample",
|
||||
"Project",
|
||||
|
||||
@@ -224,6 +224,13 @@ def _migrate_capture_settings(engine, inspector, tables: set[str]) -> None:
|
||||
"chord_toggle_to_talk_keys TEXT NOT NULL DEFAULT '[\"MetaRight\",\"AltGr\",\"Space\"]'",
|
||||
"chord_toggle_to_talk_keys",
|
||||
)
|
||||
if "hotkey_enabled" not in columns:
|
||||
_add_column(
|
||||
engine,
|
||||
"capture_settings",
|
||||
"hotkey_enabled BOOLEAN NOT NULL DEFAULT 0",
|
||||
"hotkey_enabled",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_storage_paths(engine, tables: set[str]) -> None:
|
||||
|
||||
@@ -196,6 +196,12 @@ class CaptureSettings(Base):
|
||||
preserve_technical = Column(Boolean, nullable=False, default=True)
|
||||
allow_auto_paste = Column(Boolean, nullable=False, default=True)
|
||||
default_playback_voice_id = Column(String, nullable=True)
|
||||
# Default OFF — opting in is what triggers the macOS Input Monitoring TCC
|
||||
# prompt. We deliberately don't spawn the global keyboard tap until the
|
||||
# user flips this on so a fresh-install user doesn't see a scary
|
||||
# "Voicebox would like to receive keystrokes from any application" dialog
|
||||
# before they've even opened the Captures tab.
|
||||
hotkey_enabled = Column(Boolean, nullable=False, default=False)
|
||||
# Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand
|
||||
# modifiers by default so they don't collide with left-hand system
|
||||
# shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit).
|
||||
@@ -221,6 +227,29 @@ class GenerationSettings(Base):
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class MCPClientBinding(Base):
|
||||
"""Per-MCP-client settings (voice profile, engine, intent).
|
||||
|
||||
Lets users bind distinct voices to distinct agents — e.g. Claude Code
|
||||
speaks in "Morgan," Cursor in "Scarlett." The MCP client identifies
|
||||
itself via the ``X-Voicebox-Client-Id`` HTTP header; direct-HTTP
|
||||
clients set it in their MCP config's ``headers`` block, the stdio
|
||||
shim forwards it from the ``VOICEBOX_CLIENT_ID`` env var.
|
||||
"""
|
||||
|
||||
__tablename__ = "mcp_client_bindings"
|
||||
|
||||
client_id = Column(String, primary_key=True)
|
||||
label = Column(String, nullable=True) # display name
|
||||
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
|
||||
default_engine = Column(String, nullable=True)
|
||||
# "respond" | "rewrite" | "compose" — null means plain TTS (no LLM transform).
|
||||
default_intent = Column(String, nullable=True)
|
||||
last_seen_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class Capture(Base):
|
||||
"""A single voice input capture (dictation, recording, or uploaded file).
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Voicebox MCP server
|
||||
|
||||
Local **Model Context Protocol** server — lets any MCP-aware agent
|
||||
(Claude Code, Cursor, Windsurf, VS Code MCP extensions, etc.) speak text
|
||||
in your cloned voices, transcribe audio, and browse captures.
|
||||
|
||||
The server runs inside the same `uvicorn` process as the rest of Voicebox
|
||||
and is mounted at `/mcp` (Streamable HTTP transport).
|
||||
|
||||
## Install into your agent
|
||||
|
||||
Preferred — direct HTTP:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": { "X-Voicebox-Client-Id": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Fallback — stdio shim (when the client doesn't speak HTTP MCP). The
|
||||
`voicebox-mcp` binary ships inside the Voicebox.app bundle:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Claude Code one-liner:
|
||||
|
||||
```
|
||||
claude mcp add voicebox \
|
||||
--transport http \
|
||||
--url http://127.0.0.1:17493/mcp \
|
||||
--header "X-Voicebox-Client-Id: claude-code"
|
||||
```
|
||||
|
||||
## Tools
|
||||
|
||||
| Name | Purpose |
|
||||
|---|---|
|
||||
| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. |
|
||||
| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. |
|
||||
| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. |
|
||||
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||
|
||||
All tools resolve voice profiles in this precedence:
|
||||
|
||||
1. Explicit `profile` arg (name or id — case-insensitive)
|
||||
2. Per-client binding keyed by `X-Voicebox-Client-Id`
|
||||
3. `capture_settings.default_playback_voice_id` (global default)
|
||||
|
||||
Bindings are managed via `GET|PUT /mcp/bindings` or in the app under
|
||||
Settings → MCP.
|
||||
|
||||
## Debug with MCP Inspector
|
||||
|
||||
```
|
||||
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||
```
|
||||
|
||||
Point it at the URL, hit "List tools," call `voicebox.list_profiles`
|
||||
first to confirm wiring, then `voicebox.speak` for end-to-end.
|
||||
|
||||
## Non-MCP REST surface
|
||||
|
||||
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||
don't speak MCP (shell scripts, ACP, A2A):
|
||||
|
||||
```
|
||||
curl -X POST http://127.0.0.1:17493/speak \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'X-Voicebox-Client-Id: claude-code' \
|
||||
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||
```
|
||||
|
||||
## Code layout
|
||||
|
||||
```
|
||||
backend/mcp_server/
|
||||
├── __init__.py # re-export mount_into
|
||||
├── server.py # build_mcp_server() + mount_into(app)
|
||||
├── tools.py # @mcp.tool() implementations
|
||||
├── context.py # ClientIdMiddleware + current_client_id ContextVar
|
||||
├── resolve.py # profile resolution precedence
|
||||
├── events.py # pub/sub queue for /events/speak pill SSE
|
||||
└── README.md # you are here
|
||||
|
||||
backend/mcp_shim/ # stdio ↔ Streamable-HTTP proxy (see its README)
|
||||
```
|
||||
|
||||
The package is **`mcp_server`**, not `mcp`, to avoid shadowing the
|
||||
installed `mcp` PyPI package that FastMCP imports internally.
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Model Context Protocol server — exposes Voicebox tools to local AI agents.
|
||||
|
||||
Mounts a FastMCP instance at /mcp on the main FastAPI app (Streamable HTTP).
|
||||
A bundled stdio shim (backend/mcp_shim) forwards JSON-RPC into the same
|
||||
endpoint for MCP clients that only speak stdio.
|
||||
"""
|
||||
|
||||
from .server import mount_into
|
||||
|
||||
__all__ = ["mount_into"]
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Per-request client identity for MCP calls.
|
||||
|
||||
MCP clients identify themselves via an ``X-Voicebox-Client-Id`` HTTP header
|
||||
(direct-HTTP clients set it in their MCP config; the stdio shim forwards it
|
||||
from the ``VOICEBOX_CLIENT_ID`` env var). Middleware copies the value into a
|
||||
ContextVar so tool implementations can read it without plumbing the request
|
||||
object through every service call.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from contextvars import ContextVar
|
||||
from datetime import datetime
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||
|
||||
# Tool handlers read this to apply per-client voice bindings.
|
||||
current_client_id: ContextVar[str | None] = ContextVar(
|
||||
"current_client_id", default=None
|
||||
)
|
||||
|
||||
|
||||
class ClientIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Copy X-Voicebox-Client-Id into a ContextVar and stamp last_seen_at.
|
||||
|
||||
Only stamps on MCP-endpoint requests (anything under ``/mcp``) so
|
||||
unrelated REST traffic with the header set won't advance the
|
||||
last-seen timestamp — the Settings UI uses that to show when each
|
||||
client was last heard from.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
super().__init__(app)
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
client_id = request.headers.get(CLIENT_ID_HEADER)
|
||||
token = current_client_id.set(client_id)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
finally:
|
||||
current_client_id.reset(token)
|
||||
|
||||
if client_id and request.url.path.startswith("/mcp"):
|
||||
_stamp_last_seen(client_id)
|
||||
return response
|
||||
|
||||
|
||||
def _stamp_last_seen(client_id: str) -> None:
|
||||
"""Update or create the MCPClientBinding row for this client_id."""
|
||||
try:
|
||||
from ..database import get_db
|
||||
from ..database.models import MCPClientBinding
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
db = next(get_db())
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
row = MCPClientBinding(client_id=client_id)
|
||||
db.add(row)
|
||||
row.last_seen_at = datetime.utcnow()
|
||||
db.commit()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not stamp last_seen_at for %s", client_id, exc_info=True
|
||||
)
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""In-memory pub/sub for speaking-pill SSE broadcasts.
|
||||
|
||||
MCP ``voicebox.speak`` calls and the REST ``POST /speak`` route publish
|
||||
start/end events that DictateWindow subscribes to via /events/speak, so the
|
||||
floating pill surfaces whenever an agent is speaking.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
|
||||
# Each subscriber gets its own queue. Bounded to drop oldest if a client lags.
|
||||
_subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
|
||||
|
||||
def subscribe() -> asyncio.Queue[dict[str, Any]]:
|
||||
"""Register a new subscriber; caller must call unsubscribe() when done."""
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=64)
|
||||
_subscribers.add(queue)
|
||||
return queue
|
||||
|
||||
|
||||
def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
_subscribers.discard(queue)
|
||||
|
||||
|
||||
def publish(kind: str, payload: dict[str, Any]) -> None:
|
||||
"""Fan out to all current subscribers. Non-blocking; drops on full queue."""
|
||||
event = {"kind": kind, **payload}
|
||||
for queue in list(_subscribers):
|
||||
try:
|
||||
queue.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
# Slow subscriber — skip rather than block publishers.
|
||||
pass
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Voice profile resolution for MCP tool calls.
|
||||
|
||||
Precedence:
|
||||
1. Explicit tool arg (profile name or id)
|
||||
2. Per-client MCPClientBinding.profile_id
|
||||
3. CaptureSettings.default_playback_voice_id (global default)
|
||||
4. None — caller raises a helpful error
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import VoiceProfile as DBVoiceProfile, get_db
|
||||
from ..database.models import CaptureSettings
|
||||
from ..services.profiles import get_profile_orm_by_name_or_id as _lookup_profile
|
||||
|
||||
|
||||
def resolve_profile(
|
||||
explicit: str | None,
|
||||
client_id: str | None,
|
||||
db: Session,
|
||||
) -> DBVoiceProfile | None:
|
||||
"""Apply the full precedence chain and return the profile ORM row (or None)."""
|
||||
if explicit:
|
||||
profile = _lookup_profile(explicit, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
# Explicit but not found — return None so the caller can report it.
|
||||
return None
|
||||
|
||||
if client_id:
|
||||
# Per-client binding. Imported lazily so this module stays importable
|
||||
# even before the migration adds the table on first boot.
|
||||
from ..database.models import MCPClientBinding # noqa: WPS433
|
||||
|
||||
binding = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if binding and binding.profile_id:
|
||||
profile = _lookup_profile(binding.profile_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
# Global default from capture settings.
|
||||
settings = db.query(CaptureSettings).filter(CaptureSettings.id == 1).first()
|
||||
if settings and settings.default_playback_voice_id:
|
||||
profile = _lookup_profile(settings.default_playback_voice_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def with_db() -> Session:
|
||||
"""Utility for tool handlers that aren't managed by FastAPI's Depends."""
|
||||
return next(get_db())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Construct the FastMCP server and mount it on the FastAPI app.
|
||||
|
||||
The MCP endpoint lives at ``/mcp`` (Streamable HTTP transport). Modern MCP
|
||||
clients (Claude Code, Cursor, Windsurf, VS Code MCP extensions) connect
|
||||
directly via URL; older stdio-only clients use the ``voicebox-mcp`` shim
|
||||
binary bundled with the desktop app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from typing import Callable
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .context import ClientIdMiddleware
|
||||
from .tools import register_tools
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_mcp_server() -> FastMCP:
|
||||
"""Create the FastMCP instance with Voicebox tools registered."""
|
||||
mcp = FastMCP(
|
||||
name="voicebox",
|
||||
instructions=(
|
||||
"Voicebox is a local voice I/O layer. Use `voicebox.speak` to "
|
||||
"play text in a voice profile, `voicebox.transcribe` for "
|
||||
"audio→text, and the `list_*` tools to discover profiles and "
|
||||
"captures."
|
||||
),
|
||||
)
|
||||
register_tools(mcp)
|
||||
return mcp
|
||||
|
||||
|
||||
def mount_into(
|
||||
app: FastAPI,
|
||||
*,
|
||||
extra_startup: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
"""Attach the MCP app to ``app`` at ``/mcp`` and install the client-id middleware.
|
||||
|
||||
``extra_startup`` — if provided, runs during the FastAPI lifespan. This
|
||||
is the hook that lets ``app.py`` keep its existing startup/shutdown
|
||||
bodies while also driving FastMCP's session manager.
|
||||
"""
|
||||
mcp = build_mcp_server()
|
||||
mcp_app = mcp.http_app(path="/", transport="http")
|
||||
|
||||
# ClientIdMiddleware must run before FastMCP so the ContextVar is set
|
||||
# by the time tool handlers execute. Starlette composes middlewares
|
||||
# outermost-first, so adding here on the parent app is correct.
|
||||
app.add_middleware(ClientIdMiddleware)
|
||||
app.mount("/mcp", mcp_app)
|
||||
app.state.mcp_lifespan = mcp_app.router.lifespan_context
|
||||
logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", ""))
|
||||
|
||||
|
||||
def compose_lifespan(*lifespans):
|
||||
"""Combine multiple async context managers into a single FastAPI lifespan.
|
||||
|
||||
Used by ``create_app`` to run the existing Voicebox startup/shutdown
|
||||
together with FastMCP's session manager (which MUST run in the
|
||||
ASGI lifespan for Streamable HTTP to work).
|
||||
"""
|
||||
|
||||
@asynccontextmanager
|
||||
async def _combined(app):
|
||||
async with AsyncExitStack() as stack:
|
||||
for cm_factory in lifespans:
|
||||
cm = cm_factory(app) if callable(cm_factory) else cm_factory
|
||||
await stack.enter_async_context(cm)
|
||||
yield
|
||||
|
||||
return _combined
|
||||
@@ -0,0 +1,321 @@
|
||||
"""Voicebox MCP tool implementations.
|
||||
|
||||
Thin wrappers over existing services/routes. Tools are registered with dotted
|
||||
names (``voicebox.speak`` etc.) so they look natural in agent logs —
|
||||
the Python function name stays snake_case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64 as b64
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..services import captures as captures_service
|
||||
from ..services import profiles as profiles_service
|
||||
from . import events as mcp_events
|
||||
from .context import current_client_id
|
||||
from .resolve import resolve_profile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Absolute-path transcribes are bounded to keep a bad client from
|
||||
# asking us to ingest a 20 GB file.
|
||||
MAX_TRANSCRIBE_BYTES = 200 * 1024 * 1024 # 200 MB
|
||||
|
||||
|
||||
def register_tools(mcp: FastMCP) -> None:
|
||||
"""Attach all Voicebox tools to the given FastMCP instance."""
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.speak",
|
||||
description=(
|
||||
"Speak text in a Voicebox voice profile. Returns a generation id "
|
||||
"the caller can poll at /generate/{id}/status. Audio plays on the "
|
||||
"user's speakers and is saved to the Captures / History tab."
|
||||
),
|
||||
)
|
||||
async def voicebox_speak(
|
||||
text: str,
|
||||
profile: str | None = None,
|
||||
engine: str | None = None,
|
||||
intent: Literal["respond", "rewrite", "compose"] | None = None,
|
||||
language: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Speak ``text`` in a voice profile.
|
||||
|
||||
``profile`` accepts a voice profile name (e.g. "Morgan") or id. If
|
||||
omitted, the server looks up the per-client binding for the calling
|
||||
MCP client, then falls back to the global default voice.
|
||||
|
||||
``intent`` only matters for profiles that have a personality prompt —
|
||||
when set, the text is first transformed by the LLM (respond to it,
|
||||
rewrite it in character, or compose a fresh utterance). Leave unset
|
||||
for plain TTS.
|
||||
"""
|
||||
db = next(get_db())
|
||||
try:
|
||||
vp = resolve_profile(profile, current_client_id.get(), db)
|
||||
if vp is None:
|
||||
raise ValueError(
|
||||
"No voice profile resolved. Pass `profile=` with a "
|
||||
"voice profile name or id, or set a default voice in "
|
||||
"Voicebox → Settings → MCP."
|
||||
)
|
||||
|
||||
# Persona path if intent requested and personality present.
|
||||
if intent is not None and vp.personality:
|
||||
return await _speak_with_persona(
|
||||
profile_id=vp.id,
|
||||
profile_name=vp.name,
|
||||
text=text,
|
||||
engine=engine,
|
||||
intent=intent,
|
||||
language=language,
|
||||
db=db,
|
||||
)
|
||||
|
||||
return await _speak_plain(
|
||||
profile_id=vp.id,
|
||||
profile_name=vp.name,
|
||||
text=text,
|
||||
engine=engine,
|
||||
language=language,
|
||||
db=db,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.transcribe",
|
||||
description=(
|
||||
"Transcribe an audio clip to text using Voicebox's local Whisper. "
|
||||
"Pass exactly one of `audio_base64` (bytes as base64) or "
|
||||
"`audio_path` (absolute local file path)."
|
||||
),
|
||||
)
|
||||
async def voicebox_transcribe(
|
||||
audio_base64: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if bool(audio_base64) == bool(audio_path):
|
||||
raise ValueError(
|
||||
"Pass exactly one of `audio_base64` or `audio_path`."
|
||||
)
|
||||
|
||||
# Absolute-path mode: validate and transcribe in place.
|
||||
if audio_path is not None:
|
||||
path = Path(audio_path)
|
||||
if not path.is_absolute():
|
||||
raise ValueError("`audio_path` must be absolute.")
|
||||
if not path.is_file():
|
||||
raise ValueError(f"File not found: {audio_path}")
|
||||
if path.stat().st_size > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"File exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
return await _transcribe_file(path, language, model)
|
||||
|
||||
# Base64 mode: decode into a temp file, transcribe, clean up.
|
||||
try:
|
||||
raw = b64.b64decode(audio_base64, validate=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Invalid audio_base64: {exc}") from exc
|
||||
if len(raw) > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"Audio exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".wav", delete=False
|
||||
) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
return await _transcribe_file(tmp_path, language, model)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_captures",
|
||||
description=(
|
||||
"List recent voice captures (dictations, recordings, uploads) "
|
||||
"with their transcripts. Most-recent first."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_captures(
|
||||
limit: int = 20, offset: int = 0
|
||||
) -> dict[str, Any]:
|
||||
if not (1 <= limit <= 200):
|
||||
raise ValueError("`limit` must be between 1 and 200.")
|
||||
if offset < 0:
|
||||
raise ValueError("`offset` must be >= 0.")
|
||||
db = next(get_db())
|
||||
try:
|
||||
items, total = captures_service.list_captures(
|
||||
db, limit=limit, offset=offset
|
||||
)
|
||||
return {
|
||||
"captures": [
|
||||
item.model_dump(mode="json") for item in items
|
||||
],
|
||||
"total": total,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_profiles",
|
||||
description=(
|
||||
"List available voice profiles (both cloned voices and presets). "
|
||||
"Use the returned `name` with voicebox.speak(profile=...)."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_profiles() -> dict[str, Any]:
|
||||
db = next(get_db())
|
||||
try:
|
||||
profiles = await profiles_service.list_profiles(db)
|
||||
return {
|
||||
"profiles": [
|
||||
{
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"voice_type": p.voice_type,
|
||||
"language": p.language,
|
||||
"has_personality": bool(getattr(p, "personality", None)),
|
||||
}
|
||||
for p in profiles
|
||||
]
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ─── Speak helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _speak_plain(
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_name: str,
|
||||
text: str,
|
||||
engine: str | None,
|
||||
language: str | None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""Plain TTS path — mirrors POST /generate. No LLM transform."""
|
||||
from ..routes.generations import generate_speech
|
||||
|
||||
req = models.GenerationRequest(
|
||||
profile_id=profile_id,
|
||||
text=text,
|
||||
language=language or "en",
|
||||
engine=engine or "qwen",
|
||||
)
|
||||
generation = await generate_speech(req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
|
||||
async def _speak_with_persona(
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_name: str,
|
||||
text: str,
|
||||
engine: str | None,
|
||||
intent: str,
|
||||
language: str | None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""LLM-transformed path — reuses POST /profiles/{id}/speak."""
|
||||
from ..routes.profiles import speak_in_character
|
||||
|
||||
req = models.PersonalitySpeakRequest(
|
||||
text=text,
|
||||
persist=True,
|
||||
language=language,
|
||||
engine=engine,
|
||||
intent=intent,
|
||||
)
|
||||
generation = await speak_in_character(profile_id, req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
|
||||
def _speak_response(
|
||||
generation, profile_name: str, *, source: str
|
||||
) -> dict[str, Any]:
|
||||
"""Normalize a GenerationResponse into the MCP tool's return shape.
|
||||
|
||||
Also fires a speak-start event so the DictateWindow pill surfaces
|
||||
the agent's speech. Speak-end is fired from run_generation's
|
||||
completion hook.
|
||||
"""
|
||||
payload = generation.model_dump(mode="json") if hasattr(
|
||||
generation, "model_dump"
|
||||
) else dict(generation)
|
||||
generation_id = payload.get("id")
|
||||
mcp_events.publish(
|
||||
"speak-start",
|
||||
{
|
||||
"generation_id": generation_id,
|
||||
"profile_name": profile_name,
|
||||
"source": source,
|
||||
"client_id": current_client_id.get(),
|
||||
},
|
||||
)
|
||||
return {
|
||||
"generation_id": generation_id,
|
||||
"status": payload.get("status"),
|
||||
"profile": profile_name,
|
||||
"source": source,
|
||||
"poll_url": f"/generate/{generation_id}/status"
|
||||
if generation_id
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
# ─── Transcribe helper ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _transcribe_file(
|
||||
path: Path, language: str | None, model: str | None
|
||||
) -> dict[str, Any]:
|
||||
from ..backends import WHISPER_HF_REPOS
|
||||
from ..services import transcribe as transcribe_service
|
||||
from ..utils.audio import load_audio
|
||||
|
||||
whisper = transcribe_service.get_whisper_model()
|
||||
model_size = model or whisper.model_size
|
||||
valid = list(WHISPER_HF_REPOS.keys())
|
||||
if model_size not in valid:
|
||||
raise ValueError(
|
||||
f"Invalid STT model '{model_size}'. Must be one of: {', '.join(valid)}"
|
||||
)
|
||||
|
||||
# load_audio is sync; keep the event loop responsive.
|
||||
audio, sr = await asyncio.to_thread(load_audio, str(path))
|
||||
duration = len(audio) / sr
|
||||
|
||||
if (
|
||||
not whisper.is_loaded() or whisper.model_size != model_size
|
||||
) and not whisper._is_model_cached(model_size):
|
||||
raise ValueError(
|
||||
f"Whisper model '{model_size}' is not yet downloaded. Open "
|
||||
"Voicebox → Settings → Models to download it first."
|
||||
)
|
||||
|
||||
text = await whisper.transcribe(str(path), language, model_size)
|
||||
return {
|
||||
"text": text,
|
||||
"duration": duration,
|
||||
"language": language,
|
||||
"model": model_size,
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Stdio → Streamable HTTP bridge for the Voicebox MCP server.
|
||||
|
||||
Some MCP clients only know how to spawn a subprocess and talk to it over
|
||||
stdin/stdout (the "stdio" transport). This package is a ~150-line adapter:
|
||||
the client spawns us as ``voicebox-mcp``; we proxy every JSON-RPC frame
|
||||
to http://127.0.0.1:17493/mcp/ and stream responses back out.
|
||||
|
||||
All the real work (tools, models, inference) lives in the Voicebox server
|
||||
process — this package contains no business logic.
|
||||
"""
|
||||
@@ -0,0 +1,197 @@
|
||||
"""voicebox-mcp — stdio ↔ Streamable-HTTP MCP proxy.
|
||||
|
||||
Some MCP clients only speak stdio. They spawn this binary, we pipe each
|
||||
JSON-RPC message to ``http://127.0.0.1:<port>/mcp/``, and stream the
|
||||
server's response back. The Voicebox server does all the real work.
|
||||
|
||||
Environment variables:
|
||||
VOICEBOX_PORT Voicebox server port (default 17493).
|
||||
VOICEBOX_HOST Host (default 127.0.0.1).
|
||||
VOICEBOX_CLIENT_ID Forwarded as X-Voicebox-Client-Id on every request.
|
||||
|
||||
Stdout is JSON-RPC only. Diagnostics go to stderr.
|
||||
Exit 0 on clean EOF, 1 on transport error, 2 if backend never answers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||
SESSION_HEADER = "mcp-session-id"
|
||||
HEALTH_TIMEOUT_S = 30.0
|
||||
DEFAULT_PORT = 17493
|
||||
|
||||
|
||||
def _err(msg: str) -> None:
|
||||
print(f"voicebox-mcp: {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _base_url() -> tuple[str, str]:
|
||||
host = os.environ.get("VOICEBOX_HOST", "127.0.0.1")
|
||||
port = int(os.environ.get("VOICEBOX_PORT", str(DEFAULT_PORT)))
|
||||
return f"http://{host}:{port}/mcp/", f"http://{host}:{port}/health"
|
||||
|
||||
|
||||
async def _wait_for_backend(client: httpx.AsyncClient, health_url: str) -> bool:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + HEALTH_TIMEOUT_S
|
||||
while loop.time() < deadline:
|
||||
try:
|
||||
r = await client.get(health_url, timeout=2.0)
|
||||
if r.status_code == 200:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
async def _read_stdin_line() -> str | None:
|
||||
"""Async-read a single line from stdin. Returns None on EOF."""
|
||||
loop = asyncio.get_running_loop()
|
||||
line = await loop.run_in_executor(None, sys.stdin.readline)
|
||||
if not line:
|
||||
return None
|
||||
return line
|
||||
|
||||
|
||||
def _write_stdout(obj: Any) -> None:
|
||||
"""Write a JSON object to stdout as one line, flushed."""
|
||||
sys.stdout.write(json.dumps(obj, separators=(",", ":")))
|
||||
sys.stdout.write("\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
async def _handle_request(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
raw: str,
|
||||
headers: dict[str, str],
|
||||
session_id: list[str | None],
|
||||
) -> None:
|
||||
"""Forward one JSON-RPC payload to the server and relay the response."""
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
_err(f"invalid JSON on stdin: {exc}")
|
||||
return
|
||||
|
||||
req_headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
**headers,
|
||||
}
|
||||
if session_id[0]:
|
||||
req_headers[SESSION_HEADER] = session_id[0]
|
||||
|
||||
# Notifications (no "id") don't expect a response body. Server returns
|
||||
# 202 Accepted and we stay quiet.
|
||||
is_notification = isinstance(message, dict) and "id" not in message
|
||||
|
||||
async with client.stream(
|
||||
"POST", url, headers=req_headers, content=raw.encode("utf-8")
|
||||
) as response:
|
||||
# Capture session id on initialize.
|
||||
if session_id[0] is None:
|
||||
sid = response.headers.get(SESSION_HEADER)
|
||||
if sid:
|
||||
session_id[0] = sid
|
||||
|
||||
if response.status_code == 202:
|
||||
return # notification acknowledged
|
||||
if response.status_code >= 400:
|
||||
body = await response.aread()
|
||||
_err(
|
||||
f"server {response.status_code}: "
|
||||
f"{body.decode('utf-8', errors='replace')[:400]}"
|
||||
)
|
||||
if is_notification:
|
||||
return
|
||||
_write_stdout(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": message.get("id"),
|
||||
"error": {
|
||||
"code": -32000,
|
||||
"message": (
|
||||
f"Voicebox MCP proxy got HTTP {response.status_code}"
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
ctype = response.headers.get("content-type", "")
|
||||
if "text/event-stream" in ctype:
|
||||
# SSE frames: lines prefixed "data: ..." contain the JSON-RPC msg.
|
||||
async for line in response.aiter_lines():
|
||||
if line.startswith("data:"):
|
||||
payload = line[5:].strip()
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
_write_stdout(json.loads(payload))
|
||||
except json.JSONDecodeError:
|
||||
_err(f"malformed SSE payload: {payload[:200]}")
|
||||
else:
|
||||
body = await response.aread()
|
||||
try:
|
||||
_write_stdout(json.loads(body))
|
||||
except json.JSONDecodeError:
|
||||
_err(
|
||||
f"non-JSON response ({ctype}): "
|
||||
f"{body.decode('utf-8', errors='replace')[:200]}"
|
||||
)
|
||||
|
||||
|
||||
async def _run() -> int:
|
||||
url, health_url = _base_url()
|
||||
forward_headers: dict[str, str] = {}
|
||||
client_id = os.environ.get("VOICEBOX_CLIENT_ID")
|
||||
if client_id:
|
||||
forward_headers[CLIENT_ID_HEADER] = client_id
|
||||
|
||||
session_id: list[str | None] = [None]
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(300.0)) as client:
|
||||
if not await _wait_for_backend(client, health_url):
|
||||
_err(
|
||||
f"timed out waiting for Voicebox at {health_url} — is the app open?"
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
while True:
|
||||
line = await _read_stdin_line()
|
||||
if line is None:
|
||||
return 0
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
await _handle_request(
|
||||
client, url, line, forward_headers, session_id
|
||||
)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
return 0
|
||||
except Exception as exc:
|
||||
_err(f"proxy failed: {exc!r}")
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return asyncio.run(_run())
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -248,6 +248,7 @@ class CaptureSettingsResponse(BaseModel):
|
||||
preserve_technical: bool = True
|
||||
allow_auto_paste: bool = True
|
||||
default_playback_voice_id: Optional[str] = None
|
||||
hotkey_enabled: bool = False
|
||||
chord_push_to_talk_keys: List[str] = Field(default_factory=lambda: ["MetaRight", "AltGr"])
|
||||
chord_toggle_to_talk_keys: List[str] = Field(
|
||||
default_factory=lambda: ["MetaRight", "AltGr", "Space"]
|
||||
@@ -269,6 +270,7 @@ class CaptureSettingsUpdate(BaseModel):
|
||||
preserve_technical: Optional[bool] = None
|
||||
allow_auto_paste: Optional[bool] = None
|
||||
default_playback_voice_id: Optional[str] = None
|
||||
hotkey_enabled: Optional[bool] = None
|
||||
chord_push_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||
chord_toggle_to_talk_keys: Optional[List[str]] = Field(default=None, min_length=1, max_length=6)
|
||||
|
||||
@@ -294,6 +296,68 @@ class GenerationSettingsUpdate(BaseModel):
|
||||
autoplay_on_generate: Optional[bool] = None
|
||||
|
||||
|
||||
class MCPClientBindingResponse(BaseModel):
|
||||
"""Per-MCP-client voice binding — what voice / engine / intent the server
|
||||
should use when a given client_id calls voicebox.speak without args."""
|
||||
|
||||
client_id: str
|
||||
label: Optional[str] = None
|
||||
profile_id: Optional[str] = None
|
||||
default_engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
default_intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
last_seen_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class MCPClientBindingUpsert(BaseModel):
|
||||
"""Create or update a binding. Matched by ``client_id``."""
|
||||
|
||||
client_id: str = Field(..., min_length=1, max_length=64)
|
||||
label: Optional[str] = Field(None, max_length=128)
|
||||
profile_id: Optional[str] = None
|
||||
default_engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
default_intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
|
||||
|
||||
class MCPClientBindingListResponse(BaseModel):
|
||||
items: List[MCPClientBindingResponse]
|
||||
|
||||
|
||||
class SpeakRequest(BaseModel):
|
||||
"""Body for POST /speak — non-MCP REST surface that mirrors voicebox.speak."""
|
||||
|
||||
text: str = Field(..., min_length=1, max_length=10000)
|
||||
profile: Optional[str] = Field(
|
||||
None,
|
||||
description="Voice profile name or id. Falls back to per-client binding, then default.",
|
||||
)
|
||||
engine: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$",
|
||||
)
|
||||
intent: Optional[str] = Field(
|
||||
None, pattern="^(respond|rewrite|compose)$"
|
||||
)
|
||||
language: Optional[str] = Field(
|
||||
None,
|
||||
pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he|ar|da|el|fi|hi|ms|nl|no|pl|sv|sw|tr)$",
|
||||
)
|
||||
|
||||
|
||||
class LLMGenerateRequest(BaseModel):
|
||||
"""Request model for LLM text generation."""
|
||||
|
||||
|
||||
@@ -62,6 +62,11 @@ pedalboard>=0.9.0
|
||||
# HTTP client (for CUDA backend download)
|
||||
httpx>=0.27.0
|
||||
|
||||
# MCP server (Model Context Protocol) — lets local AI agents call
|
||||
# voicebox.speak / .transcribe / .list_captures / .list_profiles
|
||||
fastmcp>=3.0,<4.0
|
||||
sse-starlette>=2.0
|
||||
|
||||
# Utilities
|
||||
python-multipart>=0.0.6
|
||||
Pillow>=10.0.0
|
||||
|
||||
@@ -20,6 +20,9 @@ def register_routers(app: FastAPI) -> None:
|
||||
from .settings import router as settings_router
|
||||
from .tasks import router as tasks_router
|
||||
from .cuda import router as cuda_router
|
||||
from .speak import router as speak_router
|
||||
from .mcp_bindings import router as mcp_bindings_router
|
||||
from .events import router as events_router
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(profiles_router)
|
||||
@@ -36,3 +39,6 @@ def register_routers(app: FastAPI) -> None:
|
||||
app.include_router(settings_router)
|
||||
app.include_router(tasks_router)
|
||||
app.include_router(cuda_router)
|
||||
app.include_router(speak_router)
|
||||
app.include_router(mcp_bindings_router)
|
||||
app.include_router(events_router)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Server-Sent-Event streams the frontend subscribes to.
|
||||
|
||||
``GET /events/speak`` — broadcasts ``speak-start`` / ``speak-end`` events
|
||||
whenever an agent-initiated speak (MCP tool or POST /speak) runs. The
|
||||
DictateWindow uses them to show the floating pill in a `speaking` state.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from ..mcp_server import events as mcp_events
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/events/speak")
|
||||
async def speak_events(request: Request):
|
||||
"""SSE stream of speak-start / speak-end events."""
|
||||
|
||||
async def event_stream():
|
||||
queue = mcp_events.subscribe()
|
||||
try:
|
||||
# Immediate hello so EventSource knows the connection is live.
|
||||
yield {"event": "ready", "data": "{}"}
|
||||
while True:
|
||||
if await request.is_disconnected():
|
||||
return
|
||||
try:
|
||||
event = await asyncio.wait_for(queue.get(), timeout=15.0)
|
||||
except asyncio.TimeoutError:
|
||||
# Heartbeat so proxies don't reap idle streams.
|
||||
yield {"event": "ping", "data": "{}"}
|
||||
continue
|
||||
kind = event.pop("kind", "message")
|
||||
yield {"event": kind, "data": json.dumps(event)}
|
||||
finally:
|
||||
mcp_events.unsubscribe(queue)
|
||||
|
||||
return EventSourceResponse(event_stream())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""REST endpoints for per-MCP-client voice binding settings.
|
||||
|
||||
The Settings UI uses these to let users configure distinct voices per
|
||||
agent (Claude Code in Morgan, Cursor in Scarlett, ...). The ``client_id``
|
||||
column is the same value the MCP client sends in ``X-Voicebox-Client-Id``
|
||||
(or the stdio shim pulls from ``VOICEBOX_CLIENT_ID``).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..database.models import MCPClientBinding
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/mcp/bindings",
|
||||
response_model=models.MCPClientBindingListResponse,
|
||||
)
|
||||
async def list_mcp_bindings(db: Session = Depends(get_db)):
|
||||
rows = (
|
||||
db.query(MCPClientBinding)
|
||||
.order_by(MCPClientBinding.client_id)
|
||||
.all()
|
||||
)
|
||||
return models.MCPClientBindingListResponse(
|
||||
items=[models.MCPClientBindingResponse.model_validate(r) for r in rows]
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/mcp/bindings",
|
||||
response_model=models.MCPClientBindingResponse,
|
||||
)
|
||||
async def upsert_mcp_binding(
|
||||
data: models.MCPClientBindingUpsert,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Create-or-update a binding. Matches by client_id."""
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == data.client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
row = MCPClientBinding(client_id=data.client_id)
|
||||
db.add(row)
|
||||
|
||||
row.label = data.label
|
||||
row.profile_id = data.profile_id
|
||||
row.default_engine = data.default_engine
|
||||
row.default_intent = data.default_intent
|
||||
row.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return models.MCPClientBindingResponse.model_validate(row)
|
||||
|
||||
|
||||
@router.delete("/mcp/bindings/{client_id}")
|
||||
async def delete_mcp_binding(
|
||||
client_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Binding not found")
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"deleted": client_id}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""POST /speak — REST wrapper around voicebox.speak for non-MCP callers.
|
||||
|
||||
Shell scripts, ACP, A2A, or any agent that doesn't speak MCP can hit this
|
||||
endpoint to play text through a cloned voice. Uses the same profile
|
||||
resolution and generation pipeline as the MCP tool, so per-client
|
||||
bindings (via X-Voicebox-Client-Id) work identically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..mcp_server import events as mcp_events
|
||||
from ..mcp_server.resolve import resolve_profile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/speak", response_model=models.GenerationResponse)
|
||||
async def speak(
|
||||
data: models.SpeakRequest,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Speak text in a voice profile. Mirrors voicebox.speak (MCP).
|
||||
|
||||
Response shape matches POST /generate — a ``GenerationResponse`` with
|
||||
``status="generating"`` and an ``id`` the caller polls at
|
||||
``GET /generate/{id}/status``.
|
||||
"""
|
||||
client_id = request.headers.get("X-Voicebox-Client-Id")
|
||||
profile = resolve_profile(data.profile, client_id, db)
|
||||
if profile is None:
|
||||
if data.profile:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Voice profile '{data.profile}' not found.",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"No voice profile resolved. Pass `profile` (name or id), "
|
||||
"or configure a default in Voicebox → Settings → MCP."
|
||||
),
|
||||
)
|
||||
|
||||
# Persona path if intent requested AND profile has a personality prompt.
|
||||
if data.intent is not None and profile.personality:
|
||||
from .profiles import speak_in_character
|
||||
|
||||
generation = await speak_in_character(
|
||||
profile.id,
|
||||
models.PersonalitySpeakRequest(
|
||||
text=data.text,
|
||||
persist=True,
|
||||
language=data.language,
|
||||
engine=data.engine,
|
||||
intent=data.intent,
|
||||
),
|
||||
db,
|
||||
)
|
||||
else:
|
||||
# Plain TTS path — matches POST /generate.
|
||||
from .generations import generate_speech
|
||||
|
||||
generation = await generate_speech(
|
||||
models.GenerationRequest(
|
||||
profile_id=profile.id,
|
||||
text=data.text,
|
||||
language=data.language or "en",
|
||||
engine=data.engine or "qwen",
|
||||
),
|
||||
db,
|
||||
)
|
||||
|
||||
mcp_events.publish(
|
||||
"speak-start",
|
||||
{
|
||||
"generation_id": getattr(generation, "id", None),
|
||||
"profile_name": profile.name,
|
||||
"source": "rest",
|
||||
"client_id": client_id,
|
||||
},
|
||||
)
|
||||
return generation
|
||||
@@ -134,6 +134,7 @@ async def run_generation(
|
||||
db=bg_db,
|
||||
error="Generation cancelled",
|
||||
)
|
||||
_notify_speak_end(generation_id, status="cancelled")
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
await history.update_generation_status(
|
||||
@@ -142,11 +143,28 @@ async def run_generation(
|
||||
db=bg_db,
|
||||
error=str(e),
|
||||
)
|
||||
_notify_speak_end(generation_id, status="failed")
|
||||
else:
|
||||
_notify_speak_end(generation_id, status="completed")
|
||||
finally:
|
||||
task_manager.complete_generation(generation_id)
|
||||
bg_db.close()
|
||||
|
||||
|
||||
def _notify_speak_end(generation_id: str, *, status: str) -> None:
|
||||
"""Publish a speak-end event; the frontend ignores unknown ids."""
|
||||
try:
|
||||
from ..mcp_server import events as mcp_events
|
||||
|
||||
mcp_events.publish(
|
||||
"speak-end",
|
||||
{"generation_id": generation_id, "status": status},
|
||||
)
|
||||
except Exception:
|
||||
# Never let event pub/sub break generation completion.
|
||||
pass
|
||||
|
||||
|
||||
def _save_generate(
|
||||
*,
|
||||
generation_id: str,
|
||||
|
||||
@@ -275,6 +275,27 @@ async def get_profile(
|
||||
return _profile_to_response(profile)
|
||||
|
||||
|
||||
def get_profile_orm_by_name_or_id(
|
||||
name_or_id: str,
|
||||
db: Session,
|
||||
) -> DBVoiceProfile | None:
|
||||
"""Resolve a profile from a user-supplied string that may be either id or name.
|
||||
|
||||
Id is tried first (fast path, matches UUIDs). Name fallback is
|
||||
case-insensitive so agents can say "Morgan" regardless of casing.
|
||||
"""
|
||||
if not name_or_id:
|
||||
return None
|
||||
row = db.query(DBVoiceProfile).filter(DBVoiceProfile.id == name_or_id).first()
|
||||
if row is not None:
|
||||
return row
|
||||
return (
|
||||
db.query(DBVoiceProfile)
|
||||
.filter(func.lower(DBVoiceProfile.name) == name_or_id.lower())
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
async def get_profile_samples(
|
||||
profile_id: str,
|
||||
db: Session,
|
||||
|
||||
@@ -46,6 +46,8 @@ tmp_ret = collect_all('espeakng_loader')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('en_core_web_sm')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('unidic_lite')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('mlx')
|
||||
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
|
||||
tmp_ret = collect_all('mlx_audio')
|
||||
|
||||
Reference in New Issue
Block a user