mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-17 05:40:42 -07:00
Jamie Pine
fe14df5fda
Mechanical sweep of items called out in the PR review: - qwen_llm_backend: AutoModelForCausalLM.from_pretrained(torch_dtype=…) is deprecated in transformers ≥4.41 in favor of dtype=. Renamed. - routes/llm: try/except around backend.generate() raised HTTPException(500, detail=str(e)) which leaks stack traces / paths to clients and trips Ruff B904. Now logs the original exception server-side and hands the client a generic message; chained via `from e` to preserve traceback context. - mcp_bindings + mcp_server/context: datetime.utcnow() is deprecated since 3.12. Switched the two assignment sites to datetime.now(timezone.utc). The schema-level `default=datetime.utcnow` defaults in database/models.py are left for a later schema-aware pass. - routes/generations: `logger = …` sat between two import blocks (Ruff E402). Moved below imports. - mcp_server/server + tests/test_refinement_samples: typing.Callable / typing.Iterable have been preferred-via collections.abc since 3.9 (Ruff UP035). - routes/events: `except asyncio.TimeoutError` aliases plain `TimeoutError` since 3.11 (UP041). - services/captures: hoisted WHISPER_NATIVE_FORMATS to module scope (was a function-local UPPER_SNAKE that tripped N806) and replaced the raw_path.unlink try/except OSError-pass with contextlib.suppress (SIM105). Semantic equivalence preserved — written_files.remove(raw_path) still only runs when unlink succeeds because it sits inside the suppressed block after the unlink call. - database/migrations: hoisted the duplicate `import sqlite3` from inside two helper bodies to a single module-level import.
81 lines
2.9 KiB
Python
81 lines
2.9 KiB
Python
"""LLM inference endpoints."""
|
|
|
|
import logging
|
|
|
|
from fastapi import APIRouter, HTTPException
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from .. import models
|
|
from ..backends import get_llm_model_configs
|
|
from ..services import llm
|
|
from ..services.task_queue import create_background_task
|
|
from ..utils.tasks import get_task_manager
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
@router.post("/llm/generate", response_model=models.LLMGenerateResponse)
|
|
async def llm_generate(request: models.LLMGenerateRequest):
|
|
"""Run a single-turn Qwen3 completion."""
|
|
backend = llm.get_llm_model()
|
|
model_size = request.model_size or backend.model_size
|
|
|
|
valid_sizes = {cfg.model_size for cfg in get_llm_model_configs()}
|
|
if model_size not in valid_sizes:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Invalid LLM size '{model_size}'. Must be one of: {sorted(valid_sizes)}",
|
|
)
|
|
|
|
already_loaded = backend.is_loaded() and backend.model_size == model_size
|
|
if not already_loaded and not backend._is_model_cached(model_size):
|
|
progress_model_name = f"qwen3-{model_size.lower()}"
|
|
task_manager = get_task_manager()
|
|
|
|
async def download_llm_background():
|
|
try:
|
|
await backend.load_model(model_size)
|
|
task_manager.complete_download(progress_model_name)
|
|
except Exception as e:
|
|
task_manager.error_download(progress_model_name, str(e))
|
|
|
|
task_manager.start_download(progress_model_name)
|
|
create_background_task(download_llm_background())
|
|
|
|
return JSONResponse(
|
|
status_code=202,
|
|
content={
|
|
"message": f"Qwen3 {model_size} is being downloaded. Please wait and try again.",
|
|
"model_name": progress_model_name,
|
|
"downloading": True,
|
|
},
|
|
)
|
|
|
|
examples: list[tuple[str, str]] | None = None
|
|
if request.examples:
|
|
for pair in request.examples:
|
|
if len(pair) != 2:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Each example must be a [user, assistant] pair",
|
|
)
|
|
examples = [(pair[0], pair[1]) for pair in request.examples]
|
|
|
|
try:
|
|
text = await backend.generate(
|
|
prompt=request.prompt,
|
|
system=request.system,
|
|
max_tokens=request.max_tokens,
|
|
temperature=request.temperature,
|
|
model_size=model_size,
|
|
examples=examples,
|
|
)
|
|
return models.LLMGenerateResponse(text=text, model_size=model_size)
|
|
except Exception as e:
|
|
# The backend exception text can include filesystem paths and stack
|
|
# frames — log it server-side and hand the client a generic message.
|
|
logger.exception("LLM generate failed")
|
|
raise HTTPException(status_code=500, detail="LLM generation failed") from e
|