feat(capture): dictation, personalities, 0.5.0

Ships the Capture release end to end. Global-hotkey dictation with
synthetic paste into the focused app on macOS and Windows, an on-screen
pill across recording / transcribing / refining, customizable push-to-
talk and toggle chords, and an accessibility-permission prompt scoped to
Settings → Captures with inline re-check feedback.

Voice profiles gain optional personalities that power compose / rewrite /
respond actions via a local Qwen3 LLM — shared with refinement, so there
is one local LLM in the app, not two.

Refinement hardened with deterministic Whisper-loop collapse before the
LLM sees the transcript, per-capture flag snapshots for re-runs, and a
ten-transcript evaluation harness across every bundled refinement size.

Version bump 0.4.5 → 0.5.0.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
2026-04-22 18:49:16 -07:00
co-authored by Claude Opus 4.7
parent ed2eec591a
commit 87c582ad54
84 changed files with 11043 additions and 512 deletions
+161 -6
View File
@@ -18,6 +18,9 @@ from typing import Protocol, Optional, Tuple, List
from typing_extensions import runtime_checkable
import numpy as np
DEFAULT_LLM_MAX_TOKENS = 512
DEFAULT_LLM_TEMPERATURE = 0.7
from ..utils.platform_detect import get_backend_type
LANGUAGE_CODE_TO_NAME = {
@@ -160,11 +163,47 @@ class STTBackend(Protocol):
...
@runtime_checkable
class LLMBackend(Protocol):
"""Protocol for local LLM (chat/completion) backend implementations."""
async def load_model(self, model_size: str) -> None:
"""Load LLM weights and tokenizer."""
...
async def generate(
self,
prompt: str,
system: Optional[str] = None,
max_tokens: int = DEFAULT_LLM_MAX_TOKENS,
temperature: float = DEFAULT_LLM_TEMPERATURE,
model_size: Optional[str] = None,
examples: Optional[list[tuple[str, str]]] = None,
) -> str:
"""Run a single-turn chat completion and return the assistant reply.
``examples`` is an optional list of ``(user, assistant)`` pairs
prepended to the conversation as proper chat turns — small models
pattern-match on inline system-prompt examples (echoing them
verbatim for unrelated inputs), but treat structured turns as
data and generalize instead. Used by the refinement service.
"""
...
def unload_model(self) -> None:
...
def is_loaded(self) -> bool:
...
# Global backend instances
_tts_backend: Optional[TTSBackend] = None
_tts_backends: dict[str, TTSBackend] = {}
_tts_backends_lock = threading.Lock()
_stt_backend: Optional[STTBackend] = None
_llm_backends: dict[str, LLMBackend] = {}
_llm_backends_lock = threading.Lock()
# Supported TTS engines — keyed by engine name, value is the backend class import path.
# The factory function uses this for the if/elif chain; the model configs live on the backend classes.
@@ -178,6 +217,10 @@ TTS_ENGINES = {
"kokoro": "Kokoro",
}
LLM_ENGINES = {
"qwen_llm": "Qwen3 LLM",
}
def _get_qwen_model_configs() -> list[ModelConfig]:
"""Return Qwen model configs with backend-aware HF repo IDs."""
@@ -365,9 +408,66 @@ def _get_whisper_configs() -> list[ModelConfig]:
]
def _get_qwen_llm_configs() -> list[ModelConfig]:
"""Return Qwen3 LLM configs with backend-aware HF repo IDs.
MLX path uses 4-bit community quantizations for Apple Silicon; PyTorch path
uses the upstream instruct weights.
"""
backend_type = get_backend_type()
if backend_type == "mlx":
repo_0_6 = "mlx-community/Qwen3-0.6B-4bit"
repo_1_7 = "mlx-community/Qwen3-1.7B-4bit"
repo_4 = "mlx-community/Qwen3-4B-4bit"
else:
repo_0_6 = "Qwen/Qwen3-0.6B"
repo_1_7 = "Qwen/Qwen3-1.7B"
repo_4 = "Qwen/Qwen3-4B"
common_languages = [
"en", "zh", "ja", "ko", "de", "fr", "ru", "pt", "es", "it",
]
return [
ModelConfig(
model_name="qwen3-0.6b",
display_name="Qwen3 0.6B",
engine="qwen_llm",
hf_repo_id=repo_0_6,
model_size="0.6B",
size_mb=400 if backend_type == "mlx" else 1400,
languages=common_languages,
),
ModelConfig(
model_name="qwen3-1.7b",
display_name="Qwen3 1.7B",
engine="qwen_llm",
hf_repo_id=repo_1_7,
model_size="1.7B",
size_mb=1100 if backend_type == "mlx" else 3500,
languages=common_languages,
),
ModelConfig(
model_name="qwen3-4b",
display_name="Qwen3 4B",
engine="qwen_llm",
hf_repo_id=repo_4,
model_size="4B",
size_mb=2500 if backend_type == "mlx" else 8000,
languages=common_languages,
),
]
def get_all_model_configs() -> list[ModelConfig]:
"""Return the full list of model configs (TTS + STT)."""
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
"""Return the full list of model configs (TTS + STT + LLM)."""
return (
_get_qwen_model_configs()
+ _get_qwen_custom_voice_configs()
+ _get_non_qwen_tts_configs()
+ _get_whisper_configs()
+ _get_qwen_llm_configs()
)
def get_tts_model_configs() -> list[ModelConfig]:
@@ -375,6 +475,11 @@ def get_tts_model_configs() -> list[ModelConfig]:
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs()
def get_llm_model_configs() -> list[ModelConfig]:
"""Return only LLM model configs."""
return _get_qwen_llm_configs()
# Lookup helpers — these replace the if/elif chains in main.py
@@ -440,7 +545,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default")
def unload_model_by_config(config: ModelConfig) -> bool:
"""Unload a model given its config. Returns True if it was loaded, False otherwise."""
from . import get_tts_backend_for_engine
from ..services import tts, transcribe
from ..services import tts, transcribe, llm as llm_service
if config.engine == "whisper":
whisper_model = transcribe.get_whisper_model()
@@ -449,6 +554,14 @@ def unload_model_by_config(config: ModelConfig) -> bool:
return True
return False
if config.engine == "qwen_llm":
backend = llm_service.get_llm_model()
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
if backend.is_loaded() and loaded_size == config.model_size:
backend.unload_model()
return True
return False
if config.engine == "qwen":
tts_model = tts.get_tts_model()
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
@@ -476,13 +589,18 @@ def unload_model_by_config(config: ModelConfig) -> bool:
def check_model_loaded(config: ModelConfig) -> bool:
"""Check if a model is currently loaded."""
from . import get_tts_backend_for_engine
from ..services import tts, transcribe
from ..services import tts, transcribe, llm as llm_service
try:
if config.engine == "whisper":
whisper_model = transcribe.get_whisper_model()
return whisper_model.is_loaded() and getattr(whisper_model, "model_size", None) == config.model_size
if config.engine == "qwen_llm":
backend = llm_service.get_llm_model()
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
return backend.is_loaded() and loaded_size == config.model_size
if config.engine == "qwen":
tts_model = tts.get_tts_model()
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
@@ -502,7 +620,7 @@ def check_model_loaded(config: ModelConfig) -> bool:
def get_model_load_func(config: ModelConfig):
"""Return a callable that loads/downloads the model."""
from . import get_tts_backend_for_engine
from ..services import tts, transcribe
from ..services import tts, transcribe, llm as llm_service
if config.engine == "whisper":
return lambda: transcribe.get_whisper_model().load_model(config.model_size)
@@ -513,6 +631,9 @@ def get_model_load_func(config: ModelConfig):
if config.engine == "qwen_custom_voice":
return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size)
if config.engine == "qwen_llm":
return lambda: llm_service.get_llm_model().load_model(config.model_size)
return lambda: get_tts_backend_for_engine(config.engine).load_model()
@@ -613,9 +734,43 @@ def get_stt_backend() -> STTBackend:
return _stt_backend
def get_llm_backend() -> LLMBackend:
"""Get or create the default Qwen3 LLM backend based on platform."""
return get_llm_backend_for_engine("qwen_llm")
def get_llm_backend_for_engine(engine: str) -> LLMBackend:
"""Get or create an LLM backend for the given engine."""
global _llm_backends
if engine in _llm_backends:
return _llm_backends[engine]
with _llm_backends_lock:
if engine in _llm_backends:
return _llm_backends[engine]
if engine == "qwen_llm":
backend_type = get_backend_type()
if backend_type == "mlx":
from .qwen_llm_backend import MLXQwenLLMBackend
backend = MLXQwenLLMBackend()
else:
from .qwen_llm_backend import PyTorchQwenLLMBackend
backend = PyTorchQwenLLMBackend()
else:
raise ValueError(f"Unknown LLM engine: {engine}. Supported: {list(LLM_ENGINES.keys())}")
_llm_backends[engine] = backend
return backend
def reset_backends():
"""Reset backend instances (useful for testing)."""
global _tts_backend, _tts_backends, _stt_backend
global _tts_backend, _tts_backends, _stt_backend, _llm_backends
_tts_backend = None
_tts_backends.clear()
_stt_backend = None
_llm_backends.clear()
+290
View File
@@ -0,0 +1,290 @@
"""
Qwen3 LLM backend implementations.
Provides MLX (Apple Silicon, 4-bit community quants) and PyTorch
(transformers AutoModelForCausalLM) paths that share the same
`LLMBackend` protocol and model-load progress plumbing as the TTS
and STT engines.
"""
import asyncio
import logging
from typing import Optional
from . import LLMBackend, DEFAULT_LLM_MAX_TOKENS, DEFAULT_LLM_TEMPERATURE
from .base import (
is_model_cached,
get_torch_device,
empty_device_cache,
manual_seed,
model_load_progress,
)
from ..utils.hf_offline_patch import force_offline_if_cached
logger = logging.getLogger(__name__)
PYTORCH_HF_REPOS = {
"0.6B": "Qwen/Qwen3-0.6B",
"1.7B": "Qwen/Qwen3-1.7B",
"4B": "Qwen/Qwen3-4B",
}
MLX_HF_REPOS = {
"0.6B": "mlx-community/Qwen3-0.6B-4bit",
"1.7B": "mlx-community/Qwen3-1.7B-4bit",
"4B": "mlx-community/Qwen3-4B-4bit",
}
def _progress_name(model_size: str) -> str:
return f"qwen3-{model_size.lower()}"
def _build_messages(
prompt: str,
system: Optional[str],
examples: Optional[list[tuple[str, str]]] = None,
) -> list[dict]:
messages: list[dict] = []
if system:
messages.append({"role": "system", "content": system})
if examples:
for user_text, assistant_text in examples:
messages.append({"role": "user", "content": user_text})
messages.append({"role": "assistant", "content": assistant_text})
messages.append({"role": "user", "content": prompt})
return messages
class PyTorchQwenLLMBackend:
"""Qwen3 LLM backend using HuggingFace transformers."""
def __init__(self, model_size: str = "0.6B"):
self.model = None
self.tokenizer = None
self.model_size = model_size
self._current_model_size: Optional[str] = None
self.device = self._get_device()
def _get_device(self) -> str:
return get_torch_device(allow_xpu=True, allow_directml=True, allow_mps=True)
def is_loaded(self) -> bool:
return self.model is not None
def _get_model_path(self, model_size: str) -> str:
if model_size not in PYTORCH_HF_REPOS:
raise ValueError(f"Unknown Qwen3 size: {model_size}")
return PYTORCH_HF_REPOS[model_size]
def _is_model_cached(self, model_size: str) -> bool:
return is_model_cached(self._get_model_path(model_size))
async def load_model(self, model_size: Optional[str] = None) -> None:
if model_size is None:
model_size = self.model_size
if self.model is not None and self._current_model_size == model_size:
return
if self.model is not None and self._current_model_size != model_size:
self.unload_model()
await asyncio.to_thread(self._load_model_sync, model_size)
def _load_model_sync(self, model_size: str) -> None:
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
progress_model_name = _progress_name(model_size)
is_cached = self._is_model_cached(model_size)
repo = self._get_model_path(model_size)
with model_load_progress(progress_model_name, is_cached):
logger.info("Loading Qwen3 %s on %s...", model_size, self.device)
with force_offline_if_cached(is_cached, progress_model_name):
self.tokenizer = AutoTokenizer.from_pretrained(repo)
dtype = torch.float16 if self.device in ("cuda", "mps") else torch.float32
self.model = AutoModelForCausalLM.from_pretrained(
repo,
torch_dtype=dtype,
)
self.model.to(self.device)
self.model.eval()
self._current_model_size = model_size
self.model_size = model_size
logger.info("Qwen3 %s loaded successfully", model_size)
def unload_model(self) -> None:
if self.model is None:
return
del self.model
del self.tokenizer
self.model = None
self.tokenizer = None
self._current_model_size = None
empty_device_cache(self.device)
logger.info("Qwen3 unloaded")
async def generate(
self,
prompt: str,
system: Optional[str] = None,
max_tokens: int = DEFAULT_LLM_MAX_TOKENS,
temperature: float = DEFAULT_LLM_TEMPERATURE,
model_size: Optional[str] = None,
examples: Optional[list[tuple[str, str]]] = None,
) -> str:
await self.load_model(model_size)
return await asyncio.to_thread(
self._generate_sync, prompt, system, max_tokens, temperature, examples
)
def _generate_sync(
self,
prompt: str,
system: Optional[str],
max_tokens: int,
temperature: float,
examples: Optional[list[tuple[str, str]]] = None,
) -> str:
import torch
messages = _build_messages(prompt, system, examples)
text = self.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
do_sample = temperature > 0
generate_kwargs = {
"max_new_tokens": max_tokens,
"do_sample": do_sample,
"pad_token_id": self.tokenizer.eos_token_id,
}
if do_sample:
generate_kwargs["temperature"] = temperature
generate_kwargs["top_p"] = 0.9
with torch.no_grad():
output_ids = self.model.generate(**inputs, **generate_kwargs)
input_len = inputs["input_ids"].shape[1]
new_tokens = output_ids[0, input_len:]
return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
class MLXQwenLLMBackend:
"""Qwen3 LLM backend using mlx-lm (Apple Silicon)."""
def __init__(self, model_size: str = "0.6B"):
self.model = None
self.tokenizer = None
self.model_size = model_size
self._current_model_size: Optional[str] = None
def is_loaded(self) -> bool:
return self.model is not None
def _get_model_path(self, model_size: str) -> str:
if model_size not in MLX_HF_REPOS:
raise ValueError(f"Unknown Qwen3 size: {model_size}")
return MLX_HF_REPOS[model_size]
def _is_model_cached(self, model_size: str) -> bool:
return is_model_cached(
self._get_model_path(model_size),
weight_extensions=(".safetensors", ".bin", ".npz"),
)
async def load_model(self, model_size: Optional[str] = None) -> None:
if model_size is None:
model_size = self.model_size
if self.model is not None and self._current_model_size == model_size:
return
if self.model is not None and self._current_model_size != model_size:
self.unload_model()
await asyncio.to_thread(self._load_model_sync, model_size)
def _load_model_sync(self, model_size: str) -> None:
from mlx_lm import load as mlx_load
progress_model_name = _progress_name(model_size)
is_cached = self._is_model_cached(model_size)
repo = self._get_model_path(model_size)
with model_load_progress(progress_model_name, is_cached):
logger.info("Loading Qwen3 %s via MLX...", model_size)
with force_offline_if_cached(is_cached, progress_model_name):
loaded = mlx_load(repo)
# mlx_lm.load returns (model, tokenizer) by default and
# (model, tokenizer, config) when return_config=True.
self.model = loaded[0]
self.tokenizer = loaded[1]
self._current_model_size = model_size
self.model_size = model_size
logger.info("Qwen3 %s (MLX) loaded successfully", model_size)
def unload_model(self) -> None:
if self.model is None:
return
del self.model
del self.tokenizer
self.model = None
self.tokenizer = None
self._current_model_size = None
logger.info("Qwen3 (MLX) unloaded")
async def generate(
self,
prompt: str,
system: Optional[str] = None,
max_tokens: int = DEFAULT_LLM_MAX_TOKENS,
temperature: float = DEFAULT_LLM_TEMPERATURE,
model_size: Optional[str] = None,
examples: Optional[list[tuple[str, str]]] = None,
) -> str:
await self.load_model(model_size)
return await asyncio.to_thread(
self._generate_sync, prompt, system, max_tokens, temperature, examples
)
def _generate_sync(
self,
prompt: str,
system: Optional[str],
max_tokens: int,
temperature: float,
examples: Optional[list[tuple[str, str]]] = None,
) -> str:
from mlx_lm import generate as mlx_generate
from mlx_lm.sample_utils import make_sampler
messages = _build_messages(prompt, system, examples)
chat_prompt = self.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
sampler = make_sampler(temp=temperature, top_p=0.9) if temperature > 0 else None
text = mlx_generate(
self.model,
self.tokenizer,
prompt=chat_prompt,
max_tokens=max_tokens,
sampler=sampler,
verbose=False,
)
return text.strip()