generated from Labyricorn/labyricorn-project-template
Initial commit (forked from jamiepine/voicebox)
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
# Voicebox MCP server
|
||||
|
||||
Local **Model Context Protocol** server — lets any MCP-aware agent
|
||||
(Claude Code, Cursor, Windsurf, VS Code MCP extensions, etc.) speak text
|
||||
in your cloned voices, transcribe audio, and browse captures.
|
||||
|
||||
The server runs inside the same `uvicorn` process as the rest of Voicebox
|
||||
and is mounted at `/mcp` (Streamable HTTP transport).
|
||||
|
||||
## Install into your agent
|
||||
|
||||
Preferred — direct HTTP:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": { "X-Voicebox-Client-Id": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Fallback — stdio shim (when the client doesn't speak HTTP MCP). The
|
||||
`voicebox-mcp` binary ships inside the Voicebox.app bundle:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"command": "/Applications/Voicebox.app/Contents/MacOS/voicebox-mcp",
|
||||
"env": { "VOICEBOX_CLIENT_ID": "claude-code" }
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Claude Code one-liner:
|
||||
|
||||
```
|
||||
claude mcp add voicebox \
|
||||
--transport http \
|
||||
--url http://127.0.0.1:17493/mcp \
|
||||
--header "X-Voicebox-Client-Id: claude-code"
|
||||
```
|
||||
|
||||
## Tools
|
||||
|
||||
| Name | Purpose |
|
||||
|---|---|
|
||||
| `voicebox.speak` | Speak text in a voice profile. Returns a generation id you can poll. |
|
||||
| `voicebox.transcribe` | Whisper transcription of a base64 blob or an absolute local path. |
|
||||
| `voicebox.list_captures` | Recent captures (dictation / recording / file) with transcripts. |
|
||||
| `voicebox.list_profiles` | Available voice profiles (cloned + preset). |
|
||||
|
||||
All tools resolve voice profiles in this precedence:
|
||||
|
||||
1. Explicit `profile` arg (name or id — case-insensitive)
|
||||
2. Per-client binding keyed by `X-Voicebox-Client-Id`
|
||||
3. `capture_settings.default_playback_voice_id` (global default)
|
||||
|
||||
Bindings are managed via `GET|PUT /mcp/bindings` or in the app under
|
||||
Settings → MCP.
|
||||
|
||||
## Debug with MCP Inspector
|
||||
|
||||
```
|
||||
npx @modelcontextprotocol/inspector http://127.0.0.1:17493/mcp
|
||||
```
|
||||
|
||||
Point it at the URL, hit "List tools," call `voicebox.list_profiles`
|
||||
first to confirm wiring, then `voicebox.speak` for end-to-end.
|
||||
|
||||
## Non-MCP REST surface
|
||||
|
||||
`POST /speak` is a thin wrapper on the same code path for callers that
|
||||
don't speak MCP (shell scripts, ACP, A2A):
|
||||
|
||||
```
|
||||
curl -X POST http://127.0.0.1:17493/speak \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'X-Voicebox-Client-Id: claude-code' \
|
||||
-d '{"text":"Build complete.","profile":"Morgan"}'
|
||||
```
|
||||
|
||||
## Code layout
|
||||
|
||||
```
|
||||
backend/mcp_server/
|
||||
├── __init__.py # re-export mount_into
|
||||
├── server.py # build_mcp_server() + mount_into(app)
|
||||
├── tools.py # @mcp.tool() implementations
|
||||
├── context.py # ClientIdMiddleware + current_client_id ContextVar
|
||||
├── resolve.py # profile resolution precedence
|
||||
├── events.py # pub/sub queue for /events/speak pill SSE
|
||||
└── README.md # you are here
|
||||
|
||||
backend/mcp_shim/ # stdio ↔ Streamable-HTTP proxy (see its README)
|
||||
```
|
||||
|
||||
The package is **`mcp_server`**, not `mcp`, to avoid shadowing the
|
||||
installed `mcp` PyPI package that FastMCP imports internally.
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Model Context Protocol server — exposes Voicebox tools to local AI agents.
|
||||
|
||||
Mounts a FastMCP instance at /mcp on the main FastAPI app (Streamable HTTP).
|
||||
A bundled stdio shim (backend/mcp_shim) forwards JSON-RPC into the same
|
||||
endpoint for MCP clients that only speak stdio.
|
||||
"""
|
||||
|
||||
from .server import mount_into
|
||||
|
||||
__all__ = ["mount_into"]
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Per-request client identity for MCP calls.
|
||||
|
||||
MCP clients identify themselves via an ``X-Voicebox-Client-Id`` HTTP header
|
||||
(direct-HTTP clients set it in their MCP config; the stdio shim forwards it
|
||||
from the ``VOICEBOX_CLIENT_ID`` env var). Middleware copies the value into a
|
||||
ContextVar so tool implementations can read it without plumbing the request
|
||||
object through every service call.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import logging
|
||||
from contextvars import ContextVar
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Strong refs to in-flight stamp tasks so asyncio.create_task results
|
||||
# don't get garbage-collected mid-flight (cf. asyncio.create_task docs).
|
||||
_pending_stamps: set[asyncio.Task] = set()
|
||||
|
||||
CLIENT_ID_HEADER = "X-Voicebox-Client-Id"
|
||||
|
||||
# Tool handlers read this to apply per-client voice bindings.
|
||||
current_client_id: ContextVar[str | None] = ContextVar(
|
||||
"current_client_id", default=None
|
||||
)
|
||||
|
||||
# Remote address of the in-flight request. Used by tools that gate
|
||||
# host-filesystem access to loopback callers (see voicebox.transcribe).
|
||||
current_remote_addr: ContextVar[str | None] = ContextVar(
|
||||
"current_remote_addr", default=None
|
||||
)
|
||||
|
||||
|
||||
def request_is_loopback() -> bool:
|
||||
"""True when the in-flight request originated on the loopback interface.
|
||||
|
||||
Returns False if no request is in flight or the remote address can't be
|
||||
parsed — callers gating filesystem reads on this should treat that as
|
||||
"deny".
|
||||
"""
|
||||
addr = current_remote_addr.get()
|
||||
if not addr:
|
||||
return False
|
||||
try:
|
||||
return ipaddress.ip_address(addr).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
# Endpoints that consume X-Voicebox-Client-Id for its MCP-semantic
|
||||
# meaning (per-client profile resolution + per-client default_personality).
|
||||
# These are the paths where a stamp into last_seen_at is accurate.
|
||||
# Unrelated REST traffic that happens to set the header is intentionally
|
||||
# ignored so the Settings UI's "last heard from" column only reflects
|
||||
# calls that actually acted on the client's bindings.
|
||||
#
|
||||
# - /mcp — FastMCP tool calls (voicebox.speak, voicebox.transcribe, …)
|
||||
# and the /mcp/bindings admin surface. The admin surface is never
|
||||
# called with the header in practice (the frontend manages bindings
|
||||
# over plain REST), so the `startswith("/mcp")` match doesn't cause
|
||||
# false stamps.
|
||||
# - /speak — REST mirror of voicebox.speak for non-MCP agents (shell
|
||||
# scripts, ACP, A2A). Uses the same per-client binding lookup, so its
|
||||
# callers belong in the last-seen list too.
|
||||
_STAMPED_PATH_PREFIXES: tuple[str, ...] = ("/mcp", "/speak")
|
||||
|
||||
|
||||
class ClientIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Copy X-Voicebox-Client-Id into a ContextVar and stamp last_seen_at
|
||||
for requests that act on the caller's MCP bindings."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
super().__init__(app)
|
||||
|
||||
async def dispatch(self, request: Request, call_next) -> Response:
|
||||
client_id = request.headers.get(CLIENT_ID_HEADER)
|
||||
remote_addr = request.client.host if request.client else None
|
||||
client_token = current_client_id.set(client_id)
|
||||
addr_token = current_remote_addr.set(remote_addr)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
finally:
|
||||
current_client_id.reset(client_token)
|
||||
current_remote_addr.reset(addr_token)
|
||||
|
||||
if client_id and _is_stamped_path(request.url.path):
|
||||
_enqueue_stamp(client_id)
|
||||
return response
|
||||
|
||||
|
||||
def _enqueue_stamp(client_id: str) -> None:
|
||||
"""Fire-and-forget the SQLite write so it doesn't block the response.
|
||||
|
||||
The stamp does sync SQLAlchemy I/O; running it inline on the event loop
|
||||
serialises every MCP request behind the SQLite write and starves SSE
|
||||
streams. ``asyncio.to_thread`` parks it on the default executor while
|
||||
the response goes back to the caller.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# Middleware shouldn't run outside a loop, but if it ever does
|
||||
# (tests, weird wsgi shim), do the write inline rather than drop it.
|
||||
_stamp_last_seen(client_id)
|
||||
return
|
||||
task = loop.create_task(asyncio.to_thread(_stamp_last_seen, client_id))
|
||||
_pending_stamps.add(task)
|
||||
task.add_done_callback(_pending_stamps.discard)
|
||||
|
||||
|
||||
def _is_stamped_path(path: str) -> bool:
|
||||
# Require a path boundary so a future ``/speakers`` or ``/mcpfoo``
|
||||
# route doesn't silently inherit the stamp from ``/speak`` / ``/mcp``.
|
||||
return any(path == p or path.startswith(p + "/") for p in _STAMPED_PATH_PREFIXES)
|
||||
|
||||
|
||||
def _stamp_last_seen(client_id: str) -> None:
|
||||
"""Update or create the MCPClientBinding row for this client_id."""
|
||||
try:
|
||||
from ..database import get_db
|
||||
from ..database.models import MCPClientBinding
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
db = next(get_db())
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
row = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if row is None:
|
||||
row = MCPClientBinding(client_id=client_id)
|
||||
db.add(row)
|
||||
row.last_seen_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Could not stamp last_seen_at for %s", client_id, exc_info=True
|
||||
)
|
||||
db.rollback()
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""In-memory pub/sub for speaking-pill SSE broadcasts.
|
||||
|
||||
MCP ``voicebox.speak`` calls and the REST ``POST /speak`` route publish
|
||||
start/end events that DictateWindow subscribes to via /events/speak, so the
|
||||
floating pill surfaces whenever an agent is speaking.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
|
||||
# Each subscriber gets its own queue. Bounded to drop oldest if a client lags.
|
||||
_subscribers: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
|
||||
|
||||
def subscribe() -> asyncio.Queue[dict[str, Any]]:
|
||||
"""Register a new subscriber; caller must call unsubscribe() when done."""
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=64)
|
||||
_subscribers.add(queue)
|
||||
return queue
|
||||
|
||||
|
||||
def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
_subscribers.discard(queue)
|
||||
|
||||
|
||||
def publish(kind: str, payload: dict[str, Any]) -> None:
|
||||
"""Fan out to all current subscribers. Non-blocking; drops on full queue.
|
||||
|
||||
Each subscriber gets its own dict copy — the SSE consumer calls
|
||||
``event.pop("kind", ...)``, so sharing a single dict between queues
|
||||
would mean the first consumer to drain its queue strips ``kind`` from
|
||||
the object the next consumer later reads.
|
||||
"""
|
||||
for queue in list(_subscribers):
|
||||
event = {"kind": kind, **payload}
|
||||
try:
|
||||
queue.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
# Slow subscriber — skip rather than block publishers.
|
||||
pass
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Voice profile resolution for MCP tool calls.
|
||||
|
||||
Precedence:
|
||||
1. Explicit tool arg (profile name or id)
|
||||
2. Per-client MCPClientBinding.profile_id
|
||||
3. CaptureSettings.default_playback_voice_id (global default)
|
||||
4. None — caller raises a helpful error
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import VoiceProfile as DBVoiceProfile, get_db
|
||||
from ..database.models import CaptureSettings
|
||||
from ..services.profiles import get_profile_orm_by_name_or_id as _lookup_profile
|
||||
|
||||
|
||||
def resolve_profile(
|
||||
explicit: str | None,
|
||||
client_id: str | None,
|
||||
db: Session,
|
||||
) -> DBVoiceProfile | None:
|
||||
"""Apply the full precedence chain and return the profile ORM row (or None)."""
|
||||
if explicit:
|
||||
profile = _lookup_profile(explicit, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
# Explicit but not found — return None so the caller can report it.
|
||||
return None
|
||||
|
||||
if client_id:
|
||||
# Per-client binding. Imported lazily so this module stays importable
|
||||
# even before the migration adds the table on first boot.
|
||||
from ..database.models import MCPClientBinding # noqa: WPS433
|
||||
|
||||
binding = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
if binding and binding.profile_id:
|
||||
profile = _lookup_profile(binding.profile_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
# Global default from capture settings.
|
||||
settings = db.query(CaptureSettings).filter(CaptureSettings.id == 1).first()
|
||||
if settings and settings.default_playback_voice_id:
|
||||
profile = _lookup_profile(settings.default_playback_voice_id, db)
|
||||
if profile is not None:
|
||||
return profile
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def with_db() -> Session:
|
||||
"""Utility for tool handlers that aren't managed by FastAPI's Depends."""
|
||||
return next(get_db())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Construct the FastMCP server and mount it on the FastAPI app.
|
||||
|
||||
The MCP endpoint lives at ``/mcp`` (Streamable HTTP transport). Modern MCP
|
||||
clients (Claude Code, Cursor, Windsurf, VS Code MCP extensions) connect
|
||||
directly via URL; older stdio-only clients use the ``voicebox-mcp`` shim
|
||||
binary bundled with the desktop app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .context import ClientIdMiddleware
|
||||
from .tools import register_tools
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_mcp_server() -> FastMCP:
|
||||
"""Create the FastMCP instance with Voicebox tools registered."""
|
||||
mcp = FastMCP(
|
||||
name="voicebox",
|
||||
instructions=(
|
||||
"Voicebox is a local voice I/O layer. Use `voicebox.speak` to "
|
||||
"play text in a voice profile, `voicebox.transcribe` for "
|
||||
"audio→text, and the `list_*` tools to discover profiles and "
|
||||
"captures."
|
||||
),
|
||||
)
|
||||
register_tools(mcp)
|
||||
return mcp
|
||||
|
||||
|
||||
def mount_into(
|
||||
app: FastAPI,
|
||||
*,
|
||||
extra_startup: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
"""Attach the MCP app to ``app`` at ``/mcp`` and install the client-id middleware.
|
||||
|
||||
``extra_startup`` — if provided, runs during the FastAPI lifespan. This
|
||||
is the hook that lets ``app.py`` keep its existing startup/shutdown
|
||||
bodies while also driving FastMCP's session manager.
|
||||
"""
|
||||
mcp = build_mcp_server()
|
||||
mcp_app = mcp.http_app(path="/", transport="http")
|
||||
|
||||
# ClientIdMiddleware must run before FastMCP so the ContextVar is set
|
||||
# by the time tool handlers execute. Starlette composes middlewares
|
||||
# outermost-first, so adding here on the parent app is correct.
|
||||
app.add_middleware(ClientIdMiddleware)
|
||||
app.mount("/mcp", mcp_app)
|
||||
app.state.mcp_lifespan = mcp_app.router.lifespan_context
|
||||
logger.info("MCP: mounted at /mcp (FastMCP %s)", getattr(mcp, "version", ""))
|
||||
|
||||
|
||||
def compose_lifespan(*lifespans):
|
||||
"""Combine multiple async context managers into a single FastAPI lifespan.
|
||||
|
||||
Used by ``create_app`` to run the existing Voicebox startup/shutdown
|
||||
together with FastMCP's session manager (which MUST run in the
|
||||
ASGI lifespan for Streamable HTTP to work).
|
||||
"""
|
||||
|
||||
@asynccontextmanager
|
||||
async def _combined(app):
|
||||
async with AsyncExitStack() as stack:
|
||||
for cm_factory in lifespans:
|
||||
cm = cm_factory(app) if callable(cm_factory) else cm_factory
|
||||
await stack.enter_async_context(cm)
|
||||
yield
|
||||
|
||||
return _combined
|
||||
@@ -0,0 +1,330 @@
|
||||
"""Voicebox MCP tool implementations.
|
||||
|
||||
Thin wrappers over existing services/routes. Tools are registered with dotted
|
||||
names (``voicebox.speak`` etc.) so they look natural in agent logs —
|
||||
the Python function name stays snake_case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64 as b64
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastmcp import FastMCP
|
||||
|
||||
from .. import models
|
||||
from ..database import get_db
|
||||
from ..services import captures as captures_service
|
||||
from ..services import profiles as profiles_service
|
||||
from . import events as mcp_events
|
||||
from .context import current_client_id, request_is_loopback
|
||||
from .resolve import resolve_profile
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Absolute-path transcribes are bounded to keep a bad client from
|
||||
# asking us to ingest a 20 GB file.
|
||||
MAX_TRANSCRIBE_BYTES = 200 * 1024 * 1024 # 200 MB
|
||||
|
||||
|
||||
def register_tools(mcp: FastMCP) -> None:
|
||||
"""Attach all Voicebox tools to the given FastMCP instance."""
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.speak",
|
||||
description=(
|
||||
"Speak text in a Voicebox voice profile. Returns a generation id "
|
||||
"the caller can poll at /generate/{id}/status. Audio plays on the "
|
||||
"user's speakers and is saved to the Captures / History tab."
|
||||
),
|
||||
)
|
||||
async def voicebox_speak(
|
||||
text: str,
|
||||
profile: str | None = None,
|
||||
engine: str | None = None,
|
||||
personality: bool | None = None,
|
||||
language: str | None = None,
|
||||
model_size: Literal["1.7B", "0.6B", "1B", "3B"] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Speak ``text`` in a voice profile.
|
||||
|
||||
``profile`` accepts a voice profile name (e.g. "Morgan") or id. If
|
||||
omitted, the server looks up the per-client binding for the calling
|
||||
MCP client, then falls back to the global default voice.
|
||||
|
||||
``personality`` only matters for profiles that have a personality
|
||||
prompt — when true, the text is first rewritten in character by the
|
||||
LLM before TTS. When omitted, the per-client binding's
|
||||
``default_personality`` flag decides; when that is unset, the
|
||||
default is plain TTS.
|
||||
|
||||
``model_size`` selects a model variant for engines that ship more
|
||||
than one — ``qwen`` and ``qwen_custom_voice`` accept "1.7B" (default)
|
||||
or "0.6B"; ``tada`` accepts "1B" or "3B". Other engines ignore it.
|
||||
Omit to use the engine default. Requesting a smaller variant (e.g.
|
||||
"0.6B") is faster and avoids reloading a heavier model between calls.
|
||||
"""
|
||||
from ..database.models import MCPClientBinding
|
||||
|
||||
db = next(get_db())
|
||||
try:
|
||||
client_id = current_client_id.get()
|
||||
vp = resolve_profile(profile, client_id, db)
|
||||
if vp is None:
|
||||
raise ValueError(
|
||||
"No voice profile resolved. Pass `profile=` with a "
|
||||
"voice profile name or id, or set a default voice in "
|
||||
"Voicebox → Settings → MCP."
|
||||
)
|
||||
|
||||
binding = None
|
||||
if client_id:
|
||||
binding = (
|
||||
db.query(MCPClientBinding)
|
||||
.filter(MCPClientBinding.client_id == client_id)
|
||||
.first()
|
||||
)
|
||||
|
||||
resolved_personality = personality
|
||||
if resolved_personality is None and binding is not None:
|
||||
resolved_personality = bool(binding.default_personality)
|
||||
|
||||
resolved_engine = engine
|
||||
if resolved_engine is None and binding is not None:
|
||||
resolved_engine = binding.default_engine
|
||||
|
||||
use_persona = bool(resolved_personality) and bool(vp.personality)
|
||||
return await _speak(
|
||||
profile_id=vp.id,
|
||||
profile_name=vp.name,
|
||||
text=text,
|
||||
engine=resolved_engine,
|
||||
language=language,
|
||||
personality=use_persona,
|
||||
model_size=model_size,
|
||||
db=db,
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.transcribe",
|
||||
description=(
|
||||
"Transcribe an audio clip to text using Voicebox's local Whisper. "
|
||||
"Pass exactly one of `audio_base64` (bytes as base64) or "
|
||||
"`audio_path` (absolute local file path — loopback callers only)."
|
||||
),
|
||||
)
|
||||
async def voicebox_transcribe(
|
||||
audio_base64: str | None = None,
|
||||
audio_path: str | None = None,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if bool(audio_base64) == bool(audio_path):
|
||||
raise ValueError(
|
||||
"Pass exactly one of `audio_base64` or `audio_path`."
|
||||
)
|
||||
|
||||
# Absolute-path mode: validate and transcribe in place. Restricted
|
||||
# to loopback callers so a Voicebox bound on 0.0.0.0 doesn't double
|
||||
# as an unauthenticated arbitrary-local-file read primitive.
|
||||
if audio_path is not None:
|
||||
if not request_is_loopback():
|
||||
raise ValueError(
|
||||
"`audio_path` is only available to loopback callers — "
|
||||
"remote callers must use `audio_base64`."
|
||||
)
|
||||
path = Path(audio_path)
|
||||
if not path.is_absolute():
|
||||
raise ValueError("`audio_path` must be absolute.")
|
||||
if not path.is_file():
|
||||
raise ValueError(f"File not found: {audio_path}")
|
||||
if path.stat().st_size > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"File exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
return await _transcribe_file(path, language, model)
|
||||
|
||||
# Base64 mode: decode into a temp file, transcribe, clean up.
|
||||
try:
|
||||
raw = b64.b64decode(audio_base64, validate=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Invalid audio_base64: {exc}") from exc
|
||||
if len(raw) > MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError(
|
||||
f"Audio exceeds {MAX_TRANSCRIBE_BYTES // (1024 * 1024)} MB limit."
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".wav", delete=False
|
||||
) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
return await _transcribe_file(tmp_path, language, model)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_captures",
|
||||
description=(
|
||||
"List recent voice captures (dictations, recordings, uploads) "
|
||||
"with their transcripts. Most-recent first."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_captures(
|
||||
limit: int = 20, offset: int = 0
|
||||
) -> dict[str, Any]:
|
||||
if not (1 <= limit <= 200):
|
||||
raise ValueError("`limit` must be between 1 and 200.")
|
||||
if offset < 0:
|
||||
raise ValueError("`offset` must be >= 0.")
|
||||
db = next(get_db())
|
||||
try:
|
||||
items, total = captures_service.list_captures(
|
||||
db, limit=limit, offset=offset
|
||||
)
|
||||
return {
|
||||
"captures": [
|
||||
item.model_dump(mode="json") for item in items
|
||||
],
|
||||
"total": total,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@mcp.tool(
|
||||
name="voicebox.list_profiles",
|
||||
description=(
|
||||
"List available voice profiles (both cloned voices and presets). "
|
||||
"Use the returned `name` with voicebox.speak(profile=...)."
|
||||
),
|
||||
)
|
||||
async def voicebox_list_profiles() -> dict[str, Any]:
|
||||
db = next(get_db())
|
||||
try:
|
||||
profiles = await profiles_service.list_profiles(db)
|
||||
return {
|
||||
"profiles": [
|
||||
{
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"voice_type": p.voice_type,
|
||||
"language": p.language,
|
||||
"has_personality": bool(getattr(p, "personality", None)),
|
||||
}
|
||||
for p in profiles
|
||||
]
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ─── Speak helper ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _speak(
|
||||
*,
|
||||
profile_id: str,
|
||||
profile_name: str,
|
||||
text: str,
|
||||
engine: str | None,
|
||||
language: str | None,
|
||||
personality: bool,
|
||||
model_size: str | None = None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""Delegate to POST /generate — the route handles personality-rewrite
|
||||
internally when ``personality=true`` and the profile has a prompt."""
|
||||
from ..routes.generations import generate_speech
|
||||
|
||||
# model_size=None is intentional: generate_speech normalizes it to the
|
||||
# engine default (see routes/generations.py), so an omitted size behaves
|
||||
# exactly like the REST /generate endpoint with no model_size in the body.
|
||||
req = models.GenerationRequest(
|
||||
profile_id=profile_id,
|
||||
text=text,
|
||||
language=language or "en",
|
||||
engine=engine,
|
||||
personality=personality,
|
||||
model_size=model_size,
|
||||
)
|
||||
generation = await generate_speech(req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
|
||||
def _speak_response(
|
||||
generation, profile_name: str, *, source: str
|
||||
) -> dict[str, Any]:
|
||||
"""Normalize a GenerationResponse into the MCP tool's return shape.
|
||||
|
||||
Also fires a speak-start event so the DictateWindow pill surfaces
|
||||
the agent's speech. Speak-end is fired from run_generation's
|
||||
completion hook.
|
||||
"""
|
||||
payload = generation.model_dump(mode="json") if hasattr(
|
||||
generation, "model_dump"
|
||||
) else dict(generation)
|
||||
generation_id = payload.get("id")
|
||||
mcp_events.publish(
|
||||
"speak-start",
|
||||
{
|
||||
"generation_id": generation_id,
|
||||
"profile_name": profile_name,
|
||||
"source": source,
|
||||
"client_id": current_client_id.get(),
|
||||
},
|
||||
)
|
||||
return {
|
||||
"generation_id": generation_id,
|
||||
"status": payload.get("status"),
|
||||
"profile": profile_name,
|
||||
"source": source,
|
||||
"poll_url": f"/generate/{generation_id}/status"
|
||||
if generation_id
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
# ─── Transcribe helper ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _transcribe_file(
|
||||
path: Path, language: str | None, model: str | None
|
||||
) -> dict[str, Any]:
|
||||
from ..backends import WHISPER_HF_REPOS
|
||||
from ..services import transcribe as transcribe_service
|
||||
from ..utils.audio import load_audio
|
||||
|
||||
whisper = transcribe_service.get_whisper_model()
|
||||
model_size = model or whisper.model_size
|
||||
valid = list(WHISPER_HF_REPOS.keys())
|
||||
if model_size not in valid:
|
||||
raise ValueError(
|
||||
f"Invalid STT model '{model_size}'. Must be one of: {', '.join(valid)}"
|
||||
)
|
||||
|
||||
# load_audio is sync; keep the event loop responsive.
|
||||
audio, sr = await asyncio.to_thread(load_audio, str(path))
|
||||
duration = len(audio) / sr
|
||||
|
||||
if (
|
||||
not whisper.is_loaded() or whisper.model_size != model_size
|
||||
) and not whisper._is_model_cached(model_size):
|
||||
raise ValueError(
|
||||
f"Whisper model '{model_size}' is not yet downloaded. Open "
|
||||
"Voicebox → Settings → Models to download it first."
|
||||
)
|
||||
|
||||
text = await whisper.transcribe(str(path), language, model_size)
|
||||
return {
|
||||
"text": text,
|
||||
"duration": duration,
|
||||
"language": language,
|
||||
"model": model_size,
|
||||
}
|
||||
Reference in New Issue
Block a user