Labyricorn/voicebox
1
0
Fork 0
mirror of https://github.com/jamiepine/voicebox.git synced 2026-09-19 23:00:45 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(mlx): serialize accelerator lifecycle and inference

Browse Source 
Route MLX load, inference, unload, reset, cache cleanup, and shutdown through a single worker. Add affinity and concurrent-unload regression coverage.\n\nVerified: 17 related backend tests; frontend CI; cargo check.
This commit is contained in:
Jamie Pine
2026-07-19 17:45:52 -07:00
parent f2cf2a729d
commit 0070c04bcf
10 changed files with 286 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/llm.py
+4 -4
View File
@@ -2,7 +2,7 @@
LLM inference module - delegates to backend abstraction layer.
"""
from ..backends import get_llm_backend, LLMBackend
from ..backends import LLMBackend, get_llm_backend, unload_backend
def get_llm_model() -> LLMBackend:
@@ -10,6 +10,6 @@ def get_llm_model() -> LLMBackend:
return get_llm_backend()
def unload_llm_model() -> None:
"""Unload LLM model to free memory."""
get_llm_backend().unload_model()
async def unload_llm_model() -> None:
"""Unload LLM model to free memory, serialized onto the MLX worker."""
await unload_backend(get_llm_backend())