mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 14:20:42 -07:00
James PineandClaude Opus 4.7
87c582ad54
Ships the Capture release end to end. Global-hotkey dictation with synthetic paste into the focused app on macOS and Windows, an on-screen pill across recording / transcribing / refining, customizable push-to- talk and toggle chords, and an accessibility-permission prompt scoped to Settings → Captures with inline re-check feedback. Voice profiles gain optional personalities that power compose / rewrite / respond actions via a local Qwen3 LLM — shared with refinement, so there is one local LLM in the app, not two. Refinement hardened with deterministic Whisper-loop collapse before the LLM sees the transcript, per-capture flag snapshots for re-runs, and a ten-transcript evaluation harness across every bundled refinement size. Version bump 0.4.5 → 0.5.0. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
569 lines
19 KiB
Python
569 lines
19 KiB
Python
"""Voice profile endpoints."""
|
|
|
|
import io
|
|
import json as _json
|
|
import logging
|
|
import tempfile
|
|
import uuid
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
|
from fastapi.responses import FileResponse, StreamingResponse
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .. import config, models
|
|
from ..app import safe_content_disposition
|
|
from ..database import VoiceProfile as DBVoiceProfile, get_db
|
|
from ..services import channels, export_import, history, personality, profiles
|
|
from ..services.profiles import _profile_to_response
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
@router.post("/profiles", response_model=models.VoiceProfileResponse)
|
|
async def create_profile(
|
|
data: models.VoiceProfileCreate,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Create a new voice profile."""
|
|
try:
|
|
return await profiles.create_profile(data, db)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
|
|
@router.get("/profiles", response_model=list[models.VoiceProfileResponse])
|
|
async def list_profiles(db: Session = Depends(get_db)):
|
|
"""List all voice profiles."""
|
|
return await profiles.list_profiles(db)
|
|
|
|
|
|
@router.post("/profiles/import", response_model=models.VoiceProfileResponse)
|
|
async def import_profile(
|
|
file: UploadFile = File(...),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Import a voice profile from a ZIP archive."""
|
|
MAX_FILE_SIZE = 100 * 1024 * 1024
|
|
|
|
content = await file.read()
|
|
|
|
if len(content) > MAX_FILE_SIZE:
|
|
raise HTTPException(
|
|
status_code=400, detail=f"File too large. Maximum size is {MAX_FILE_SIZE / (1024 * 1024)}MB"
|
|
)
|
|
|
|
try:
|
|
profile = await export_import.import_profile_from_zip(content, db)
|
|
return profile
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
# ── Preset Voice Endpoints ───────────────────────────────────────────
|
|
# These MUST be declared before /profiles/{profile_id} to avoid the
|
|
# wildcard swallowing "presets" as a profile_id.
|
|
|
|
|
|
@router.get("/profiles/presets/{engine}")
|
|
async def list_preset_voices(engine: str):
|
|
"""List available preset voices for an engine."""
|
|
if engine == "kokoro":
|
|
from ..backends.kokoro_backend import KOKORO_VOICES
|
|
|
|
return {
|
|
"engine": engine,
|
|
"voices": [
|
|
{
|
|
"voice_id": vid,
|
|
"name": name,
|
|
"gender": gender,
|
|
"language": lang,
|
|
}
|
|
for vid, name, gender, lang in KOKORO_VOICES
|
|
],
|
|
}
|
|
if engine == "qwen_custom_voice":
|
|
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
|
|
|
|
return {
|
|
"engine": engine,
|
|
"voices": [
|
|
{
|
|
"voice_id": speaker_id,
|
|
"name": display_name,
|
|
"gender": gender,
|
|
"language": lang,
|
|
}
|
|
for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES
|
|
],
|
|
}
|
|
return {"engine": engine, "voices": []}
|
|
|
|
@router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
|
|
async def get_profile(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get a voice profile by ID."""
|
|
profile = await profiles.get_profile(profile_id, db)
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
return profile
|
|
|
|
|
|
@router.put("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
|
|
async def update_profile(
|
|
profile_id: str,
|
|
data: models.VoiceProfileCreate,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Update a voice profile."""
|
|
try:
|
|
profile = await profiles.update_profile(profile_id, data, db)
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
return profile
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
|
|
@router.delete("/profiles/{profile_id}")
|
|
async def delete_profile(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Delete a voice profile."""
|
|
success = await profiles.delete_profile(profile_id, db)
|
|
if not success:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
return {"message": "Profile deleted successfully"}
|
|
|
|
|
|
SAMPLE_MAX_FILE_SIZE = 50 * 1024 * 1024 # 50 MB
|
|
SAMPLE_UPLOAD_CHUNK_SIZE = 1024 * 1024 # 1 MB
|
|
|
|
|
|
@router.post("/profiles/{profile_id}/samples", response_model=models.ProfileSampleResponse)
|
|
async def add_profile_sample(
|
|
profile_id: str,
|
|
file: UploadFile = File(...),
|
|
reference_text: str = Form(...),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Add a sample to a voice profile."""
|
|
_allowed_audio_exts = {".wav", ".mp3", ".m4a", ".ogg", ".flac", ".aac", ".webm", ".opus"}
|
|
_uploaded_ext = Path(file.filename or "").suffix.lower()
|
|
file_suffix = _uploaded_ext if _uploaded_ext in _allowed_audio_exts else ".wav"
|
|
|
|
with tempfile.NamedTemporaryFile(suffix=file_suffix, delete=False) as tmp:
|
|
total_size = 0
|
|
while chunk := await file.read(SAMPLE_UPLOAD_CHUNK_SIZE):
|
|
total_size += len(chunk)
|
|
if total_size > SAMPLE_MAX_FILE_SIZE:
|
|
Path(tmp.name).unlink(missing_ok=True)
|
|
raise HTTPException(
|
|
status_code=413,
|
|
detail=f"File too large (max {SAMPLE_MAX_FILE_SIZE // (1024 * 1024)} MB)",
|
|
)
|
|
tmp.write(chunk)
|
|
tmp_path = tmp.name
|
|
|
|
try:
|
|
sample = await profiles.add_profile_sample(
|
|
profile_id,
|
|
tmp_path,
|
|
reference_text,
|
|
db,
|
|
)
|
|
return sample
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"Failed to process audio file: {str(e)}")
|
|
finally:
|
|
Path(tmp_path).unlink(missing_ok=True)
|
|
|
|
|
|
@router.get("/profiles/{profile_id}/samples", response_model=list[models.ProfileSampleResponse])
|
|
async def get_profile_samples(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get all samples for a profile."""
|
|
return await profiles.get_profile_samples(profile_id, db)
|
|
|
|
|
|
@router.delete("/profiles/samples/{sample_id}")
|
|
async def delete_profile_sample(
|
|
sample_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Delete a profile sample."""
|
|
success = await profiles.delete_profile_sample(sample_id, db)
|
|
if not success:
|
|
raise HTTPException(status_code=404, detail="Sample not found")
|
|
return {"message": "Sample deleted successfully"}
|
|
|
|
|
|
@router.put("/profiles/samples/{sample_id}", response_model=models.ProfileSampleResponse)
|
|
async def update_profile_sample(
|
|
sample_id: str,
|
|
data: models.ProfileSampleUpdate,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Update a profile sample's reference text."""
|
|
sample = await profiles.update_profile_sample(sample_id, data.reference_text, db)
|
|
if not sample:
|
|
raise HTTPException(status_code=404, detail="Sample not found")
|
|
return sample
|
|
|
|
|
|
@router.post("/profiles/{profile_id}/avatar", response_model=models.VoiceProfileResponse)
|
|
async def upload_profile_avatar(
|
|
profile_id: str,
|
|
file: UploadFile = File(...),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Upload or update avatar image for a profile."""
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename).suffix) as tmp:
|
|
content = await file.read()
|
|
tmp.write(content)
|
|
tmp_path = tmp.name
|
|
|
|
try:
|
|
profile = await profiles.upload_avatar(profile_id, tmp_path, db)
|
|
return profile
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
finally:
|
|
Path(tmp_path).unlink(missing_ok=True)
|
|
|
|
|
|
@router.get("/profiles/{profile_id}/avatar")
|
|
async def get_profile_avatar(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get avatar image for a profile."""
|
|
profile = await profiles.get_profile(profile_id, db)
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
|
|
if not profile.avatar_path:
|
|
raise HTTPException(status_code=404, detail="No avatar found for this profile")
|
|
|
|
avatar_path = config.resolve_storage_path(profile.avatar_path)
|
|
if avatar_path is None or not avatar_path.exists():
|
|
raise HTTPException(status_code=404, detail="Avatar file not found")
|
|
|
|
return FileResponse(avatar_path)
|
|
|
|
|
|
@router.delete("/profiles/{profile_id}/avatar")
|
|
async def delete_profile_avatar(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Delete avatar image for a profile."""
|
|
success = await profiles.delete_avatar(profile_id, db)
|
|
if not success:
|
|
raise HTTPException(status_code=404, detail="Profile not found or no avatar to delete")
|
|
return {"message": "Avatar deleted successfully"}
|
|
|
|
|
|
@router.get("/profiles/{profile_id}/export")
|
|
async def export_profile(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Export a voice profile as a ZIP archive."""
|
|
try:
|
|
profile = await profiles.get_profile(profile_id, db)
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
|
|
zip_bytes = export_import.export_profile_to_zip(profile_id, db)
|
|
|
|
safe_name = "".join(c for c in profile.name if c.isalnum() or c in (" ", "-", "_")).strip()
|
|
if not safe_name:
|
|
safe_name = "profile"
|
|
filename = f"profile-{safe_name}.voicebox.zip"
|
|
|
|
return StreamingResponse(
|
|
io.BytesIO(zip_bytes),
|
|
media_type="application/zip",
|
|
headers={"Content-Disposition": safe_content_disposition("attachment", filename)},
|
|
)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.get("/profiles/{profile_id}/channels")
|
|
async def get_profile_channels(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get list of channel IDs assigned to a profile."""
|
|
try:
|
|
channel_ids = await channels.get_profile_channels(profile_id, db)
|
|
return {"channel_ids": channel_ids}
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
|
|
@router.put("/profiles/{profile_id}/channels")
|
|
async def set_profile_channels(
|
|
profile_id: str,
|
|
data: models.ProfileChannelAssignment,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Set which channels a profile is assigned to."""
|
|
try:
|
|
await channels.set_profile_channels(profile_id, data, db)
|
|
return {"message": "Profile channels updated successfully"}
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
|
|
@router.put("/profiles/{profile_id}/effects", response_model=models.VoiceProfileResponse)
|
|
async def update_profile_effects(
|
|
profile_id: str,
|
|
data: models.ProfileEffectsUpdate,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Set or clear the default effects chain for a voice profile."""
|
|
profile = db.query(DBVoiceProfile).filter_by(id=profile_id).first()
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
|
|
if data.effects_chain is not None:
|
|
from ..utils.effects import validate_effects_chain
|
|
|
|
chain_dicts = [e.model_dump() for e in data.effects_chain]
|
|
error = validate_effects_chain(chain_dicts)
|
|
if error:
|
|
raise HTTPException(status_code=400, detail=error)
|
|
profile.effects_chain = _json.dumps(chain_dicts)
|
|
else:
|
|
profile.effects_chain = None
|
|
|
|
profile.updated_at = datetime.utcnow()
|
|
db.commit()
|
|
db.refresh(profile)
|
|
|
|
return _profile_to_response(profile)
|
|
|
|
|
|
# ── Personality endpoints ─────────────────────────────────────────────
|
|
# compose / rewrite / respond / speak. All four require a non-empty
|
|
# personality on the profile; the service layer raises ValueError which
|
|
# we translate to HTTP 400. compose and rewrite power the generate-box
|
|
# UI; respond is API-only for conversational / agent-style callers;
|
|
# speak chains LLM → TTS in one call.
|
|
|
|
|
|
def _load_profile_for_personality(profile_id: str, db: Session) -> DBVoiceProfile:
|
|
profile = db.query(DBVoiceProfile).filter_by(id=profile_id).first()
|
|
if not profile:
|
|
raise HTTPException(status_code=404, detail="Profile not found")
|
|
return profile
|
|
|
|
|
|
def _resolve_speak_engine(
|
|
data: models.PersonalitySpeakRequest,
|
|
profile: DBVoiceProfile,
|
|
) -> str:
|
|
return (
|
|
data.engine
|
|
or getattr(profile, "default_engine", None)
|
|
or getattr(profile, "preset_engine", None)
|
|
or "qwen"
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/profiles/{profile_id}/compose",
|
|
response_model=models.PersonalityTextResponse,
|
|
)
|
|
async def compose_in_character(
|
|
profile_id: str,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Produce a fresh utterance in the profile's character voice."""
|
|
profile = _load_profile_for_personality(profile_id, db)
|
|
try:
|
|
result = await personality.compose_as_profile(profile.personality)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
return models.PersonalityTextResponse(
|
|
text=result.text, model_size=result.model_size
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/profiles/{profile_id}/rewrite",
|
|
response_model=models.PersonalityTextResponse,
|
|
)
|
|
async def rewrite_in_character(
|
|
profile_id: str,
|
|
data: models.PersonalityTextRequest,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Restate the user's text in the profile's character voice."""
|
|
profile = _load_profile_for_personality(profile_id, db)
|
|
try:
|
|
result = await personality.rewrite_as_profile(profile.personality, data.text)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
return models.PersonalityTextResponse(
|
|
text=result.text, model_size=result.model_size
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/profiles/{profile_id}/respond",
|
|
response_model=models.PersonalityTextResponse,
|
|
)
|
|
async def respond_in_character(
|
|
profile_id: str,
|
|
data: models.PersonalityTextRequest,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Produce an in-character reply to the user's text. API-only surface."""
|
|
profile = _load_profile_for_personality(profile_id, db)
|
|
try:
|
|
result = await personality.respond_as_profile(profile.personality, data.text)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
return models.PersonalityTextResponse(
|
|
text=result.text, model_size=result.model_size
|
|
)
|
|
|
|
|
|
@router.post("/profiles/{profile_id}/speak")
|
|
async def speak_in_character(
|
|
profile_id: str,
|
|
data: models.PersonalitySpeakRequest,
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""LLM (by intent) → TTS, returned either as a generation row the client
|
|
polls (``persist=true``) or a direct wav stream (``persist=false``).
|
|
|
|
Response shape depends on ``persist``:
|
|
- ``true``: 200 JSON ``GenerationResponse`` with ``status="generating"``.
|
|
Row is tagged ``source="personality_speak"``.
|
|
- ``false``: 200 ``audio/wav`` streaming response, nothing persisted.
|
|
"""
|
|
from ..backends import engine_has_model_sizes, load_engine_model
|
|
from ..services.generation import generate_audio_sync, run_generation
|
|
from ..services.task_queue import enqueue_generation
|
|
from ..utils.tasks import get_task_manager
|
|
|
|
profile = _load_profile_for_personality(profile_id, db)
|
|
|
|
engine = _resolve_speak_engine(data, profile)
|
|
try:
|
|
profiles.validate_profile_engine(profile, engine)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
# Run the LLM transform per requested intent. personality.* enforce
|
|
# the empty-personality guard — catch and translate here.
|
|
try:
|
|
if data.intent == "compose":
|
|
llm_result = await personality.compose_as_profile(profile.personality)
|
|
elif data.intent == "rewrite":
|
|
llm_result = await personality.rewrite_as_profile(
|
|
profile.personality, data.text
|
|
)
|
|
else: # "respond"
|
|
llm_result = await personality.respond_as_profile(
|
|
profile.personality, data.text
|
|
)
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
|
spoken_text = llm_result.text.strip()
|
|
if not spoken_text:
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail="LLM produced empty output; nothing to speak.",
|
|
)
|
|
|
|
resolved_language = data.language or getattr(profile, "language", None) or "en"
|
|
model_size = "1.7B" if engine_has_model_sizes(engine) else None
|
|
|
|
if not data.persist:
|
|
# Transient path — generate synchronously, stream wav back.
|
|
# ``load_engine_model`` is defensive against engines that don't
|
|
# take a size (kokoro, etc.); pass "default" to match the
|
|
# in-tree signature default.
|
|
await load_engine_model(engine, model_size or "default")
|
|
wav_bytes = await generate_audio_sync(
|
|
profile_id=profile_id,
|
|
text=spoken_text,
|
|
language=resolved_language,
|
|
engine=engine,
|
|
model_size=model_size or "default",
|
|
)
|
|
return StreamingResponse(
|
|
iter([wav_bytes]),
|
|
media_type="audio/wav",
|
|
headers={"Content-Disposition": 'inline; filename="speech.wav"'},
|
|
)
|
|
|
|
# Persistent path — mirrors /generate exactly, plus source marker.
|
|
generation_id = str(uuid.uuid4())
|
|
task_manager = get_task_manager()
|
|
|
|
generation = await history.create_generation(
|
|
profile_id=profile_id,
|
|
text=spoken_text,
|
|
language=resolved_language,
|
|
audio_path="",
|
|
duration=0,
|
|
seed=None,
|
|
db=db,
|
|
instruct=None,
|
|
generation_id=generation_id,
|
|
status="generating",
|
|
engine=engine,
|
|
model_size=model_size if engine_has_model_sizes(engine) else None,
|
|
source="personality_speak",
|
|
)
|
|
|
|
task_manager.start_generation(
|
|
task_id=generation_id,
|
|
profile_id=profile_id,
|
|
text=spoken_text,
|
|
)
|
|
|
|
enqueue_generation(
|
|
generation_id,
|
|
run_generation(
|
|
generation_id=generation_id,
|
|
profile_id=profile_id,
|
|
text=spoken_text,
|
|
language=resolved_language,
|
|
engine=engine,
|
|
model_size=model_size,
|
|
seed=None,
|
|
normalize=True,
|
|
effects_chain=None,
|
|
instruct=None,
|
|
mode="generate",
|
|
),
|
|
)
|
|
|
|
return generation
|