Files
voicebox/backend/database/models.py
T
James PineandClaude Opus 4.7 0cef2c9fe1 feat(mcp): local MCP server exposes voicebox.* tools to AI agents
Mounts FastMCP at /mcp (Streamable HTTP) so Claude Code, Cursor,
Windsurf, and the VS Code MCP extensions can call voicebox.speak,
voicebox.transcribe, voicebox.list_captures, and voicebox.list_profiles
against the running Voicebox server.

Backend
- new backend/mcp_server package (tools, middleware, profile resolve,
  pub/sub events); named mcp_server to avoid shadowing the installed mcp
  PyPI package FastMCP imports internally
- app.py migrated from @app.on_event to lifespan= so FastMCP's session
  manager cohabits with Voicebox's startup/shutdown
- new MCPClientBinding table + /mcp/bindings CRUD; ClientIdMiddleware
  reads X-Voicebox-Client-Id into a ContextVar and stamps last_seen_at
- profile resolution precedence: explicit -> per-client binding ->
  capture_settings.default_playback_voice_id
- POST /speak REST wrapper for non-MCP callers (shell, ACP, A2A)
- GET /events/speak SSE broadcasts speak-start / speak-end so the pill
  surfaces agent-initiated speech
- backend/mcp_shim proxy (plain httpx) for stdio-only MCP clients
- PyInstaller spec updates + new --shim build target (~18 MB)

Frontend
- Settings -> MCP page with HTTP / stdio / claude-mcp-add copy snippets,
  default voice picker, per-client bindings table, connection status
- useMCPBindings, useSpeakEvents hooks
- CapturePill gains 'speaking' state; DictateWindow subscribes to SSE
  and emits dictate:show so the Rust side surfaces the pill window

Native
- tauri.conf.json externalBin now includes voicebox-mcp
- show_dictate_window helper + dictate:show listener in main.rs
- (also in this commit: InputMonitoringGate UX, hotkey_monitor tweaks,
  landing footer/navbar updates, new overview docs for captures /
  dictation / mcp-server / voice-personalities)

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
2026-04-22 22:05:30 -07:00

274 lines
11 KiB
Python

"""ORM model definitions for the voicebox SQLite database."""
from datetime import datetime
import uuid
from sqlalchemy import Column, String, Integer, Float, DateTime, Text, ForeignKey, Boolean, JSON
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class VoiceProfile(Base):
"""Voice profile.
voice_type discriminates three flavours:
- "cloned" — traditional reference-audio profiles (all cloning engines)
- "preset" — engine-specific pre-built voice (e.g. Kokoro voices)
- "designed" — text-described voice (e.g. Qwen CustomVoice, future)
"""
__tablename__ = "profiles"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, unique=True, nullable=False)
description = Column(Text)
language = Column(String, default="en")
avatar_path = Column(String, nullable=True)
effects_chain = Column(Text, nullable=True)
# Voice type system — added v0.3.x
voice_type = Column(String, default="cloned") # "cloned" | "preset" | "designed"
preset_engine = Column(String, nullable=True) # e.g. "kokoro" — only for preset
preset_voice_id = Column(String, nullable=True) # e.g. "am_adam" — only for preset
design_prompt = Column(Text, nullable=True) # text description — only for designed
default_engine = Column(String, nullable=True) # auto-selected engine, locked for preset
# Free-form character prompt used by the compose / rewrite / respond / speak
# endpoints. Describes *what* this voice says and how, orthogonal to how
# it sounds (which is handled by the preset / cloning metadata above).
personality = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class ProfileSample(Base):
"""Audio sample attached to a voice profile."""
__tablename__ = "profile_samples"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
profile_id = Column(String, ForeignKey("profiles.id"), nullable=False)
audio_path = Column(String, nullable=False)
reference_text = Column(Text, nullable=False)
class Generation(Base):
"""A single TTS generation."""
__tablename__ = "generations"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
profile_id = Column(String, ForeignKey("profiles.id"), nullable=False)
text = Column(Text, nullable=False)
language = Column(String, default="en")
audio_path = Column(String, nullable=True)
duration = Column(Float, nullable=True)
seed = Column(Integer)
instruct = Column(Text)
engine = Column(String, default="qwen")
model_size = Column(String, nullable=True)
status = Column(String, default="completed")
error = Column(Text, nullable=True)
is_favorited = Column(Boolean, default=False)
# Origin of this generation — "manual" for regular /generate calls,
# "personality_speak" for rows created by POST /profiles/{id}/speak.
# Future sources (bulk import, agent replies, etc.) can extend this.
source = Column(String, nullable=False, default="manual")
created_at = Column(DateTime, default=datetime.utcnow)
class Story(Base):
"""A story that sequences multiple generations."""
__tablename__ = "stories"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
description = Column(Text)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class StoryItem(Base):
"""Links a generation to a story at a specific timecode."""
__tablename__ = "story_items"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
story_id = Column(String, ForeignKey("stories.id"), nullable=False)
generation_id = Column(String, ForeignKey("generations.id"), nullable=False)
version_id = Column(String, ForeignKey("generation_versions.id"), nullable=True)
start_time_ms = Column(Integer, nullable=False, default=0)
track = Column(Integer, nullable=False, default=0)
trim_start_ms = Column(Integer, nullable=False, default=0)
trim_end_ms = Column(Integer, nullable=False, default=0)
created_at = Column(DateTime, default=datetime.utcnow)
class Project(Base):
"""Audio studio project (JSON blob)."""
__tablename__ = "projects"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
data = Column(Text)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class GenerationVersion(Base):
"""A version of a generation's audio (original, processed, alternate takes)."""
__tablename__ = "generation_versions"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
generation_id = Column(String, ForeignKey("generations.id"), nullable=False)
label = Column(String, nullable=False)
audio_path = Column(String, nullable=False)
effects_chain = Column(Text, nullable=True)
source_version_id = Column(String, ForeignKey("generation_versions.id"), nullable=True)
is_default = Column(Boolean, default=False)
created_at = Column(DateTime, default=datetime.utcnow)
class EffectPreset(Base):
"""Saved effect chain preset."""
__tablename__ = "effect_presets"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, unique=True, nullable=False)
description = Column(Text, nullable=True)
effects_chain = Column(Text, nullable=False)
is_builtin = Column(Boolean, default=False)
sort_order = Column(Integer, default=100)
created_at = Column(DateTime, default=datetime.utcnow)
class AudioChannel(Base):
"""Audio output channel (bus)."""
__tablename__ = "audio_channels"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
is_default = Column(Boolean, default=False)
created_at = Column(DateTime, default=datetime.utcnow)
class ChannelDeviceMapping(Base):
"""Mapping between a channel and an OS audio device."""
__tablename__ = "channel_device_mappings"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
channel_id = Column(String, ForeignKey("audio_channels.id"), nullable=False)
device_id = Column(String, nullable=False)
class ProfileChannelMapping(Base):
"""Many-to-many mapping between voice profiles and audio channels."""
__tablename__ = "profile_channel_mappings"
profile_id = Column(String, ForeignKey("profiles.id"), primary_key=True)
channel_id = Column(String, ForeignKey("audio_channels.id"), primary_key=True)
class CaptureSettings(Base):
"""Singleton row holding user defaults for the capture/refine flow.
Kept server-side so every window, CLI client, and API consumer reads the
same preferences. The ``id`` column is always 1.
"""
__tablename__ = "capture_settings"
id = Column(Integer, primary_key=True, default=1)
stt_model = Column(String, nullable=False, default="turbo")
language = Column(String, nullable=False, default="auto")
auto_refine = Column(Boolean, nullable=False, default=True)
llm_model = Column(String, nullable=False, default="0.6B")
smart_cleanup = Column(Boolean, nullable=False, default=True)
self_correction = Column(Boolean, nullable=False, default=True)
preserve_technical = Column(Boolean, nullable=False, default=True)
allow_auto_paste = Column(Boolean, nullable=False, default=True)
default_playback_voice_id = Column(String, nullable=True)
# Default OFF — opting in is what triggers the macOS Input Monitoring TCC
# prompt. We deliberately don't spawn the global keyboard tap until the
# user flips this on so a fresh-install user doesn't see a scary
# "Voicebox would like to receive keystrokes from any application" dialog
# before they've even opened the Captures tab.
hotkey_enabled = Column(Boolean, nullable=False, default=False)
# Lists of rdev::Key variant names (e.g. "MetaRight", "AltGr"). Right-hand
# modifiers by default so they don't collide with left-hand system
# shortcuts (Cmd+Opt+I devtools, Cmd+Opt+Esc force-quit).
chord_push_to_talk_keys = Column(
JSON, nullable=False, default=lambda: ["MetaRight", "AltGr"]
)
chord_toggle_to_talk_keys = Column(
JSON, nullable=False, default=lambda: ["MetaRight", "AltGr", "Space"]
)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class GenerationSettings(Base):
"""Singleton row for long-form TTS generation preferences."""
__tablename__ = "generation_settings"
id = Column(Integer, primary_key=True, default=1)
max_chunk_chars = Column(Integer, nullable=False, default=800)
crossfade_ms = Column(Integer, nullable=False, default=50)
normalize_audio = Column(Boolean, nullable=False, default=True)
autoplay_on_generate = Column(Boolean, nullable=False, default=True)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class MCPClientBinding(Base):
"""Per-MCP-client settings (voice profile, engine, intent).
Lets users bind distinct voices to distinct agents — e.g. Claude Code
speaks in "Morgan," Cursor in "Scarlett." The MCP client identifies
itself via the ``X-Voicebox-Client-Id`` HTTP header; direct-HTTP
clients set it in their MCP config's ``headers`` block, the stdio
shim forwards it from the ``VOICEBOX_CLIENT_ID`` env var.
"""
__tablename__ = "mcp_client_bindings"
client_id = Column(String, primary_key=True)
label = Column(String, nullable=True) # display name
profile_id = Column(String, ForeignKey("profiles.id"), nullable=True)
default_engine = Column(String, nullable=True)
# "respond" | "rewrite" | "compose" — null means plain TTS (no LLM transform).
default_intent = Column(String, nullable=True)
last_seen_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class Capture(Base):
"""A single voice input capture (dictation, recording, or uploaded file).
Stores the original audio alongside the raw transcript and, optionally, a
refined version produced by the LLM. Refinement flags are serialized as
JSON so we can reproduce the prompt that generated the refined text.
"""
__tablename__ = "captures"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
audio_path = Column(String, nullable=False)
source = Column(String, nullable=False, default="file") # dictation | recording | file
language = Column(String, nullable=True)
duration_ms = Column(Integer, nullable=True)
transcript_raw = Column(Text, nullable=False, default="")
transcript_refined = Column(Text, nullable=True)
stt_model = Column(String, nullable=True)
llm_model = Column(String, nullable=True)
refinement_flags = Column(Text, nullable=True) # JSON blob
created_at = Column(DateTime, default=datetime.utcnow)