Compare commits

..
Author SHA1 Message Date
James Pine fc553e94e5 docs: link hermes plugin guide instead of unshipped provider-guide pages 2026-07-12 18:35:16 -07:00
James Pine 68feec8305 docs: hermes-voicebox is published on PyPI 2026-07-12 16:11:46 -07:00
James Pine 126daf53a8 docs: add Hermes Agent integration guide
New overview page covering both integration surfaces: the MCP hookup
(hermes mcp install voicebox — catalog entry submitted upstream) for
agent-invoked speak/transcribe/captures tools, and the hermes-voicebox
provider plugin (github.com/jamiepine/hermes-voicebox) that routes
Hermes's entire voice pipeline — spoken replies, Telegram voice
bubbles, and incoming voice-message transcription — through local
Voicebox.
2026-07-12 15:23:59 -07:00
69 changed files with 318 additions and 5518 deletions
+1 -2
View File
@@ -8,8 +8,7 @@ tauri/
landing/
docs/
mlx-test/
scripts/*
!scripts/rocm-entrypoint.sh
scripts/
# Dependencies & build artifacts (rebuilt in Docker)
node_modules/
-2
View File
@@ -1,2 +0,0 @@
package.json text eol=lf
scripts/*.sh text eol=lf
+1 -1
View File
@@ -91,7 +91,7 @@ On Windows, to build with CUDA support for local testing:
just build-local # Build CPU + CUDA server binaries + Tauri installer
```
This builds the CPU sidecar (bundled with the app), the CUDA binary (placed in `%APPDATA%/sh.voicebox.app/backends/` for runtime GPU switching), and the installable Tauri app.
This builds the CPU sidecar (bundled with the app), the CUDA binary (placed in `%APPDATA%/com.voicebox.app/backends/` for runtime GPU switching), and the installable Tauri app.
Creates platform-specific installers (`.dmg`, `.msi`, `.AppImage`) in `tauri/src-tauri/target/release/bundle/`.
+3 -10
View File
@@ -20,11 +20,8 @@ COPY package.json bun.lock CHANGELOG.md ./
COPY app/ ./app/
COPY web/ ./web/
# Normalize line endings first (a Windows CRLF checkout would otherwise
# defeat the `-z 's/,\n ]/…/'` match below, since it's LF-anchored), then
# strip workspaces not needed for web build, and fix trailing comma
RUN sed -i 's/\r$//' package.json && \
sed -i '/"tauri"/d; /"landing"/d' package.json && \
# Strip workspaces not needed for web build, and fix trailing comma
RUN sed -i '/"tauri"/d; /"landing"/d' package.json && \
sed -i -z 's/,\n ]/\n ]/' package.json
RUN bun install --no-save
# Build frontend (skip tsc — upstream has pre-existing type errors)
@@ -103,11 +100,7 @@ EXPOSE 17493
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=60s \
CMD curl -f http://localhost:17493/health || exit 1
# Entrypoint joins GPU groups then drops to the voicebox user.
# Normalize CRLF (a Windows checkout otherwise leaves the shebang as
# `#!/bin/sh\r`, which Linux can't resolve — reported as a misleading
# "no such file or directory" even though the file exists).
# Entrypoint joins GPU groups then drops to the voicebox user
COPY --chmod=755 scripts/rocm-entrypoint.sh /usr/local/bin/entrypoint.sh
RUN sed -i 's/\r$//' /usr/local/bin/entrypoint.sh
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "17493"]
+1 -2
View File
@@ -270,8 +270,7 @@ Use cases: agent dev loops (dictate a question, hear the answer in a cloned voic
| Platform | Backend | Notes |
| ------------------------ | -------------- | ---------------------------------------------- |
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
| Windows (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
| Linux (NVIDIA) | PyTorch (CUDA) | Use a local/remote Python backend with CUDA PyTorch |
| Windows / Linux (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
| Windows (any GPU) | DirectML | Universal Windows GPU support |
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
@@ -139,7 +139,7 @@ export function EngineModelSelector({ form, compact, selectedProfile }: EngineMo
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent side={compact ? 'top' : undefined}>
<SelectContent>
{availableOptions.map((opt) => (
<SelectItem key={opt.value} value={opt.value} className={itemClass}>
{opt.label}
@@ -555,7 +555,7 @@ export function FloatingGenerateBox({
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all w-full">
<SelectValue placeholder={t('generation.voiceSelector.placeholder')} />
</SelectTrigger>
<SelectContent side="top">
<SelectContent>
{profiles?.map((profile) => (
<SelectItem key={profile.id} value={profile.id} className="text-xs">
{profile.name}
@@ -582,7 +582,7 @@ export function FloatingGenerateBox({
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent side="top">
<SelectContent>
{engineLangs.map((lang) => (
<SelectItem key={lang.value} value={lang.value} className="text-xs">
{lang.label}
@@ -610,7 +610,7 @@ export function FloatingGenerateBox({
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
<SelectValue placeholder={t('generation.effects.none')} />
</SelectTrigger>
<SelectContent side="top">
<SelectContent>
<SelectItem value="none" className="text-xs">
{t('generation.effects.none')}
</SelectItem>
@@ -1,151 +0,0 @@
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { Cloud, Loader2 } from 'lucide-react';
import { useEffect, useState } from 'react';
import { Button } from '@/components/ui/button';
import { useToast } from '@/components/ui/use-toast';
import { apiClient } from '@/lib/api/client';
import { SettingRow, SettingSection } from './SettingRow';
// "Log in with browser" device pairing. The backend opens the system browser
// and completes the code exchange; here we just kick it off and poll status
// until the link goes live. The API key never touches the frontend.
export function CloudSection() {
const { toast } = useToast();
const queryClient = useQueryClient();
const [polling, setPolling] = useState(false);
const { data: status } = useQuery({
queryKey: ['cloud-status'],
queryFn: () => apiClient.getCloudStatus(),
refetchInterval: polling ? 2000 : false,
});
const connected = status?.connected ?? false;
// Once the browser flow completes, stop polling and celebrate.
useEffect(() => {
if (connected && polling) {
setPolling(false);
toast({
title: 'Connected to Voicebox Cloud',
description: `Linked as ${status?.device_name ?? 'this device'}.`,
});
}
}, [connected, polling, status?.device_name, toast]);
// Give up after two minutes so an abandoned browser flow doesn't leave the
// button stuck on "Waiting for browser…". The backend state stays valid for
// ten, so the user can simply start again.
useEffect(() => {
if (!polling) return;
const timeoutId = window.setTimeout(() => {
setPolling(false);
toast({
title: 'Sign-in timed out',
description: 'The browser sign-in was not completed. Try again.',
variant: 'destructive',
});
}, 120_000);
return () => window.clearTimeout(timeoutId);
}, [polling, toast]);
const startLogin = useMutation({
mutationFn: () => apiClient.startCloudLogin(),
onSuccess: () => {
setPolling(true);
toast({
title: 'Continue in your browser',
description: 'Authorize this device, then return here.',
});
},
onError: (error: Error) =>
toast({
title: 'Could not start sign-in',
description: error.message,
variant: 'destructive',
}),
});
const disconnect = useMutation({
mutationFn: () => apiClient.disconnectCloud(),
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['cloud-status'] });
toast({
title: 'Disconnected',
description:
'This device is no longer linked. The key stays valid until revoked in your account.',
});
},
onError: (error: Error) =>
toast({ title: 'Could not disconnect', description: error.message, variant: 'destructive' }),
});
const busy = startLogin.isPending || polling;
return (
<SettingSection
title="Voicebox Cloud"
description="End-to-end encrypted backup & sync across your devices."
>
<SettingRow
title={connected ? 'Connected' : 'Account'}
description={
connected
? `Linked as ${status?.device_name ?? 'this device'}${
status?.key_prefix ? ` · ${status.key_prefix}…` : ''
}`
: 'Log in to back up and sync your captures and generations.'
}
action={
connected ? (
<Button
disabled={disconnect.isPending}
onClick={() => disconnect.mutate()}
size="sm"
variant="outline"
>
{disconnect.isPending ? (
<>
<Loader2 className="h-3.5 w-3.5 mr-1.5 animate-spin" />
Disconnecting…
</>
) : (
'Disconnect'
)}
</Button>
) : (
<Button disabled={busy} onClick={() => startLogin.mutate()} size="sm">
{busy ? (
<>
<Loader2 className="h-3.5 w-3.5 mr-1.5 animate-spin" />
{polling ? 'Waiting for browser…' : 'Opening…'}
</>
) : (
<>
<Cloud className="h-3.5 w-3.5 mr-1.5" />
Log in with browser
</>
)}
</Button>
)
}
/>
{connected && (
<SettingRow
title="Manage"
description="Revoke this device, add API keys, or manage billing from your account."
>
<a
className="text-sm text-accent hover:underline"
href={status?.dashboard_url ?? 'https://voicebox.sh/account'}
rel="noopener noreferrer"
target="_blank"
>
Open account dashboard ↗
</a>
</SettingRow>
)}
</SettingSection>
);
}
@@ -14,7 +14,6 @@ import { useAutoUpdater } from '@/hooks/useAutoUpdater';
import { useServerHealth } from '@/lib/hooks/useServer';
import { usePlatform } from '@/platform/PlatformContext';
import { useServerStore } from '@/stores/serverStore';
import { CloudSection } from './CloudSection';
import { LanguageSelect } from './LanguageSelect';
import { SettingRow, SettingSection } from './SettingRow';
import { ThemeSelect } from './ThemeSelect';
@@ -208,8 +207,6 @@ export function GeneralPage() {
/>
</SettingSection>
<CloudSection />
<ApiReferenceCard serverUrl={serverUrl} />
{platform.metadata.isTauri && <UpdatesSection />}
+1 -10
View File
@@ -2,25 +2,19 @@ import i18n from 'i18next';
import LanguageDetector from 'i18next-browser-languagedetector';
import { initReactI18next } from 'react-i18next';
import en from './locales/en/translation.json';
import es from './locales/es/translation.json';
import fr from './locales/fr/translation.json';
import it from './locales/it/translation.json';
import ja from './locales/ja/translation.json';
import ko from './locales/ko/translation.json';
import ptBR from './locales/pt-BR/translation.json';
import zhCN from './locales/zh-CN/translation.json';
import zhTW from './locales/zh-TW/translation.json';
import fr from './locales/fr/translation.json';
export const SUPPORTED_LANGUAGES = [
{ code: 'en', label: 'English' },
{ code: 'es', label: 'Español' },
{ code: 'pt-BR', label: 'Português (Brasil)' },
{ code: 'ja', label: '日本語' },
{ code: 'ko', label: '한국어' },
{ code: 'zh-CN', label: '简体中文' },
{ code: 'zh-TW', label: '繁體中文' },
{ code: 'fr', label: 'Français' },
{ code: 'it', label: 'Italiano' },
] as const;
export type LanguageCode = (typeof SUPPORTED_LANGUAGES)[number]['code'];
@@ -31,14 +25,11 @@ i18n
.init({
resources: {
en: { translation: en },
es: { translation: es },
'pt-BR': { translation: ptBR },
ja: { translation: ja },
ko: { translation: ko },
'zh-CN': { translation: zhCN },
'zh-TW': { translation: zhTW },
fr: { translation: fr },
it: { translation: it },
},
fallbackLng: 'en',
supportedLngs: SUPPORTED_LANGUAGES.map((l) => l.code),
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-17
View File
@@ -51,8 +51,6 @@ import type {
MCPClientBinding,
MCPClientBindingListResponse,
MCPClientBindingUpsert,
CloudLoginStartResponse,
CloudStatus,
} from './types';
function formatErrorDetail(detail: unknown, fallback: string): string {
@@ -940,21 +938,6 @@ class ApiClient {
return response.blob();
}
// Cloud (backup & sync) — browser-based device login. startCloudLogin opens
// the system browser server-side; the UI then polls getCloudStatus until the
// backend completes the exchange and the link goes live.
async getCloudStatus(): Promise<CloudStatus> {
return this.request<CloudStatus>('/cloud/status');
}
async startCloudLogin(): Promise<CloudLoginStartResponse> {
return this.request<CloudLoginStartResponse>('/cloud/login/start', { method: 'POST' });
}
async disconnectCloud(): Promise<CloudStatus> {
return this.request<CloudStatus>('/cloud/disconnect', { method: 'POST' });
}
}
export const apiClient = new ApiClient();
+1 -19
View File
@@ -287,10 +287,7 @@ export interface CudaDownloadProgress {
export interface CudaStatus {
available: boolean; // CUDA binary exists on disk
active: boolean; // Currently running the CUDA binary
binary_path: string | null;
cuda_libs_version: string | null;
download_supported: boolean; // Platform has a matching release asset
unsupported_reason: string | null;
binary_path?: string;
downloading: boolean; // Download in progress
download_progress?: CudaDownloadProgress;
}
@@ -545,18 +542,3 @@ export interface MCPClientBindingUpsert {
export interface MCPClientBindingListResponse {
items: MCPClientBinding[];
}
/* ─── Cloud (backup & sync) ───────────────────────────────────────────── */
export interface CloudLoginStartResponse {
authorize_url: string;
}
export interface CloudStatus {
connected: boolean;
device_name: string | null;
account_user_id: string | null;
key_prefix: string | null;
connected_at: string | null;
dashboard_url: string;
}
+4 -8
View File
@@ -47,14 +47,12 @@ export function useExportGeneration() {
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
const blob = await apiClient.exportGeneration(generationId);
// Create safe filename from text. Append a short id so exports of
// similarly-worded generations don't collide on the same filename
// (the first 30 chars are frequently identical).
// Create safe filename from text
const safeText = text
.substring(0, 30)
.replace(/[^a-z0-9]/gi, '-')
.toLowerCase();
const filename = `generation-${safeText}-${generationId.substring(0, 8)}.voicebox.zip`;
const filename = `generation-${safeText}.voicebox.zip`;
await platform.filesystem.saveFile(filename, blob, [
{
@@ -75,14 +73,12 @@ export function useExportGenerationAudio() {
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
const blob = await apiClient.exportGenerationAudio(generationId);
// Create safe filename from text. Append a short id so exports of
// similarly-worded generations don't collide on the same filename
// (the first 30 chars are frequently identical).
// Create safe filename from text
const safeText = text
.substring(0, 30)
.replace(/[^a-z0-9]/gi, '-')
.toLowerCase();
const filename = `${safeText}-${generationId.substring(0, 8)}.wav`;
const filename = `${safeText}.wav`;
await platform.filesystem.saveFile(filename, blob, [
{
+15 -18
View File
@@ -1,5 +1,5 @@
import { formatDistance } from 'date-fns';
import { es, fr, ja, zhCN, zhTW } from 'date-fns/locale';
import { ja, zhCN, zhTW, fr } from 'date-fns/locale';
import i18n from '@/i18n';
export function formatDuration(seconds: number): string {
@@ -10,8 +10,6 @@ export function formatDuration(seconds: number): string {
function getDateLocale() {
switch (i18n.language) {
case 'es':
return es;
case 'ja':
return ja;
case 'zh-CN':
@@ -25,28 +23,27 @@ function getDateLocale() {
}
}
// Backend timestamps are naive UTC — append `Z` so JS doesn't parse a
// timezone-less date-time string as local time.
function parseServerDate(date: string | Date): Date {
if (typeof date !== 'string') {
return date;
}
const dateStr = date.trim();
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
return new Date(`${dateStr}Z`);
}
return new Date(dateStr);
}
export function formatDate(date: string | Date): string {
return formatDistance(parseServerDate(date), new Date(), {
let dateObj: Date;
if (typeof date === 'string') {
const dateStr = date.trim();
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
dateObj = new Date(`${dateStr}Z`);
} else {
dateObj = new Date(dateStr);
}
} else {
dateObj = date;
}
return formatDistance(dateObj, new Date(), {
addSuffix: true,
locale: getDateLocale(),
}).replace(/^about /i, '');
}
export function formatAbsoluteDate(date: string | Date): string {
const dateObj = parseServerDate(date);
const dateObj = typeof date === 'string' ? new Date(date) : date;
return dateObj.toLocaleString(i18n.language, {
month: 'short',
day: 'numeric',
-7
View File
@@ -38,13 +38,6 @@ logging.basicConfig(
logger = logging.getLogger(__name__)
# An empty HSA_OVERRIDE_GFX_VERSION poisons the ROCm HSA runtime. It is
# treated as "force-empty" and no GPU is detected, even natively supported
# ones (e.g. gfx1201 / RX 9070 on ROCm 7.2). docker-compose can't
# conditionally omit an env var, so we clean it up here before torch loads.
if not os.environ.get("HSA_OVERRIDE_GFX_VERSION"):
os.environ.pop("HSA_OVERRIDE_GFX_VERSION", None)
# AMD GPU environment variables must be set before torch import
# Only set HSA_OVERRIDE_GFX_VERSION for older GPUs that need it.
# RDNA 3+ (gfx1100+) and RDNA 4 (gfx1200+) are natively supported by ROCm
-15
View File
@@ -56,7 +56,6 @@ class ModelConfig:
model_size: str = "default"
size_mb: int = 0
needs_trim: bool = False
retries_runaway: bool = False
supports_instruct: bool = False
languages: list[str] = field(default_factory=lambda: ["en"])
@@ -233,10 +232,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
# mlx-audio can continue after an EOS miss with silence followed by
# codec noise. Retry only the affected text as smaller chunks.
retries_runaway = backend_type == "mlx"
return [
ModelConfig(
model_name="qwen-tts-1.7B",
@@ -245,7 +240,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
hf_repo_id=repo_1_7b,
model_size="1.7B",
size_mb=3500,
retries_runaway=retries_runaway,
supports_instruct=False, # Base model drops instruct silently
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
),
@@ -256,7 +250,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
hf_repo_id=repo_0_6b,
model_size="0.6B",
size_mb=1200,
retries_runaway=retries_runaway,
supports_instruct=False,
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
),
@@ -511,14 +504,6 @@ def engine_needs_trim(engine: str) -> bool:
return False
def engine_retries_runaway(engine: str) -> bool:
"""Whether unstable output should be retried in smaller chunks."""
for cfg in get_tts_model_configs():
if cfg.engine == engine:
return cfg.retries_runaway
return False
def engine_has_model_sizes(engine: str) -> bool:
"""Whether this engine supports multiple model sizes (only Qwen currently)."""
configs = [c for c in get_tts_model_configs() if c.engine == engine]
+2 -6
View File
@@ -248,13 +248,9 @@ class HumeTadaBackend:
audio = audio.T # (samples, channels) -> (channels, samples)
audio = audio.to(device)
# Encode with forced alignment.
# Must run under inference_mode: encoder params still require
# grad by default, and an autograd graph across the DAC/Snake
# stack can balloon VRAM far past the model footprint (#890).
# Encode with forced alignment
text_arg = [reference_text] if reference_text else None
with torch.inference_mode():
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
# Serialize EncoderOutput to a dict of CPU tensors for caching
prompt_dict = {}
+1 -6
View File
@@ -96,16 +96,11 @@ KOKORO_VOICES = [
("pf_dora", "Dora", "female", "pt"),
("pm_alex", "Alex", "male", "pt"),
("pm_santa", "Santa", "male", "pt"),
# Chinese female
# Chinese
("zf_xiaobei", "Xiaobei", "female", "zh"),
("zf_xiaoni", "Xiaoni", "female", "zh"),
("zf_xiaoxiao", "Xiaoxiao", "female", "zh"),
("zf_xiaoyi", "Xiaoyi", "female", "zh"),
# Chinese male
("zm_yunjian", "Yunjian", "male", "zh"),
("zm_yunxi", "Yunxi", "male", "zh"),
("zm_yunxia", "Yunxia", "male", "zh"),
("zm_yunyang", "Yunyang", "male", "zh"),
]
# Map our ISO language codes to Kokoro lang_code characters
+12 -15
View File
@@ -19,6 +19,7 @@ from .base import (
manual_seed,
model_load_progress,
)
from ..utils.hf_offline_patch import force_offline_if_cached
logger = logging.getLogger(__name__)
@@ -102,19 +103,15 @@ class PyTorchQwenLLMBackend:
with model_load_progress(progress_model_name, is_cached):
logger.info("Loading Qwen3 %s on %s...", model_size, self.device)
# Loads run with the process's default HF_HUB_OFFLINE state.
# Forcing offline for cached models flips process-global state
# and silently switches every concurrent download/load on other
# threads to offline mode (issue #841) — the same regression
# removed app-wide in #524/#530.
self.tokenizer = AutoTokenizer.from_pretrained(repo)
dtype = torch.float16 if self.device in ("cuda", "mps") else torch.float32
self.model = AutoModelForCausalLM.from_pretrained(
repo,
dtype=dtype,
)
self.model.to(self.device)
self.model.eval()
with force_offline_if_cached(is_cached, progress_model_name):
self.tokenizer = AutoTokenizer.from_pretrained(repo)
dtype = torch.float16 if self.device in ("cuda", "mps") else torch.float32
self.model = AutoModelForCausalLM.from_pretrained(
repo,
dtype=dtype,
)
self.model.to(self.device)
self.model.eval()
self._current_model_size = model_size
self.model_size = model_size
@@ -226,8 +223,8 @@ class MLXQwenLLMBackend:
with model_load_progress(progress_model_name, is_cached):
logger.info("Loading Qwen3 %s via MLX...", model_size)
# See the PyTorch loader comment — no offline forcing (issue #841).
loaded = mlx_load(repo)
with force_offline_if_cached(is_cached, progress_model_name):
loaded = mlx_load(repo)
# mlx_lm.load returns (model, tokenizer) by default and
# (model, tokenizer, config) when return_config=True.
-3
View File
@@ -330,9 +330,6 @@ def build_server(cuda=False, rocm=False):
]
)
if sys.version_info >= (3, 13):
args.extend(["--hidden-import", "audioop"])
# Add CUDA/ROCm-specific hidden imports
if cuda or rocm:
variant = "ROCm" if rocm else "CUDA"
-19
View File
@@ -80,11 +80,6 @@ def resolve_storage_path(path: str | Path | None) -> Path | None:
return None
stored_path = Path(path)
# Empty paths (e.g. failed generations) must not resolve to the data
# dir itself, which exists and would defeat the callers' 404 guards.
# Path("") is truthy, so check parts rather than the raw value.
if not stored_path.parts:
return None
if stored_path.is_absolute():
rebased_path = _path_relative_to_any_data_dir(stored_path)
if rebased_path is not None:
@@ -143,17 +138,3 @@ def get_models_dir() -> Path:
path = _data_dir / "models"
path.mkdir(parents=True, exist_ok=True)
return path
# Voicebox Cloud (backup & sync). Two hosts: the web app owns auth + device
# pairing (voicebox.sh), the API owns sync + account endpoints
# (api.voicebox.sh). Override both for local development, e.g.
# VOICEBOX_CLOUD_URL=http://localhost:17592 VOICEBOX_CLOUD_API_URL=http://localhost:17593
def get_cloud_web_url() -> str:
"""Base URL of the Voicebox Cloud web app (auth + /connect + exchange)."""
return os.environ.get("VOICEBOX_CLOUD_URL", "https://voicebox.sh").rstrip("/")
def get_cloud_api_url() -> str:
"""Base URL of the Voicebox Cloud API (bearer-authenticated sync/account)."""
return os.environ.get("VOICEBOX_CLOUD_API_URL", "https://api.voicebox.sh").rstrip("/")
-2
View File
@@ -11,7 +11,6 @@ from .models import (
Capture,
CaptureSettings,
ChannelDeviceMapping,
CloudSettings,
EffectPreset,
Generation,
GenerationSettings,
@@ -33,7 +32,6 @@ __all__ = [
"Capture",
"CaptureSettings",
"ChannelDeviceMapping",
"CloudSettings",
"EffectPreset",
"Generation",
"GenerationSettings",
-22
View File
@@ -234,28 +234,6 @@ class GenerationSettings(Base):
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class CloudSettings(Base):
"""Singleton row holding the link to a Voicebox Cloud account.
Populated by the "Log in with browser" pairing flow (see services/cloud.py):
the browser hands back a one-time code, which the backend exchanges for an
``api_key`` it stores here. The key is a bearer credential for
api.voicebox.sh — auth only, never an encryption key (E2E key material lives
elsewhere). Stored in the local app database alongside the user's other data;
moving it to the OS keychain is a future hardening step. The ``id`` is
always 1; a null ``api_key`` means "not connected".
"""
__tablename__ = "cloud_settings"
id = Column(Integer, primary_key=True, default=1)
api_key = Column(String, nullable=True)
device_name = Column(String, nullable=True)
account_user_id = Column(String, nullable=True)
connected_at = Column(DateTime, nullable=True)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class MCPClientBinding(Base):
"""Per-MCP-client settings (voice profile, engine, personality default).
+1 -14
View File
@@ -12,7 +12,7 @@ import base64 as b64
import logging
import tempfile
from pathlib import Path
from typing import Any, Literal
from typing import Any
from fastmcp import FastMCP
@@ -49,7 +49,6 @@ def register_tools(mcp: FastMCP) -> None:
engine: str | None = None,
personality: bool | None = None,
language: str | None = None,
model_size: Literal["1.7B", "0.6B", "1B", "3B"] | None = None,
) -> dict[str, Any]:
"""Speak ``text`` in a voice profile.
@@ -62,12 +61,6 @@ def register_tools(mcp: FastMCP) -> None:
LLM before TTS. When omitted, the per-client binding's
``default_personality`` flag decides; when that is unset, the
default is plain TTS.
``model_size`` selects a model variant for engines that ship more
than one — ``qwen`` and ``qwen_custom_voice`` accept "1.7B" (default)
or "0.6B"; ``tada`` accepts "1B" or "3B". Other engines ignore it.
Omit to use the engine default. Requesting a smaller variant (e.g.
"0.6B") is faster and avoids reloading a heavier model between calls.
"""
from ..database.models import MCPClientBinding
@@ -106,7 +99,6 @@ def register_tools(mcp: FastMCP) -> None:
engine=resolved_engine,
language=language,
personality=use_persona,
model_size=model_size,
db=db,
)
finally:
@@ -236,23 +228,18 @@ async def _speak(
engine: str | None,
language: str | None,
personality: bool,
model_size: str | None = None,
db,
) -> dict[str, Any]:
"""Delegate to POST /generate — the route handles personality-rewrite
internally when ``personality=true`` and the profile has a prompt."""
from ..routes.generations import generate_speech
# model_size=None is intentional: generate_speech normalizes it to the
# engine default (see routes/generations.py), so an omitted size behaves
# exactly like the REST /generate endpoint with no model_size in the body.
req = models.GenerationRequest(
profile_id=profile_id,
text=text,
language=language or "en",
engine=engine,
personality=personality,
model_size=model_size,
)
generation = await generate_speech(req, db)
return _speak_response(generation, profile_name, source="mcp")
-21
View File
@@ -794,24 +794,3 @@ class AvailableEffectsResponse(BaseModel):
"""Response listing all available effect types."""
effects: List[AvailableEffect]
# ─── Cloud (backup & sync) ──────────────────────────────────────────────
class CloudLoginStartResponse(BaseModel):
"""Returned when the desktop kicks off browser login. The backend has
already opened the browser; the URL is included for fallback/debugging."""
authorize_url: str
class CloudStatusResponse(BaseModel):
"""Current link between this device and a Voicebox Cloud account."""
connected: bool
device_name: Optional[str] = None
account_user_id: Optional[str] = None
key_prefix: Optional[str] = None
connected_at: Optional[datetime] = None
dashboard_url: str
+1 -2
View File
@@ -16,8 +16,7 @@ miniaudio>=1.59
# mlx_audio.stt.load) works fine on transformers 4.57.x in practice.
#
# Install it via `pip install --no-deps mlx-audio==0.4.1` after this file
# (see .github/workflows/release.yml and the setup-python recipe in the
# justfile). Most other mlx-audio runtime deps
# (see .github/workflows/release.yml). Most other mlx-audio runtime deps
# (huggingface_hub, librosa, mlx-lm, numba, numpy, protobuf, pyloudnorm,
# sounddevice, tqdm) are already in requirements.txt or pulled in by
# other engines.
-1
View File
@@ -53,7 +53,6 @@ en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_
unidic-lite>=1.0.8
# Audio processing
audioop-lts>=0.2.1; python_version >= "3.13"
librosa>=0.10.0
soundfile>=0.12.0
numpy>=1.24.0,<2.0
-2
View File
@@ -24,7 +24,6 @@ def register_routers(app: FastAPI) -> None:
from .speak import router as speak_router
from .mcp_bindings import router as mcp_bindings_router
from .events import router as events_router
from .cloud import router as cloud_router
app.include_router(health_router)
app.include_router(profiles_router)
@@ -45,4 +44,3 @@ def register_routers(app: FastAPI) -> None:
app.include_router(speak_router)
app.include_router(mcp_bindings_router)
app.include_router(events_router)
app.include_router(cloud_router)
+4 -9
View File
@@ -34,7 +34,7 @@ async def get_version_audio(version_id: str, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="Version not found")
audio_path = config.resolve_storage_path(version.audio_path)
if audio_path is None or not audio_path.is_file():
if audio_path is None or not audio_path.exists():
raise HTTPException(status_code=404, detail="Audio file not found")
return FileResponse(
@@ -52,13 +52,8 @@ async def get_audio(generation_id: str, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="Generation not found")
audio_path = config.resolve_storage_path(generation.audio_path)
if audio_path is None or not audio_path.is_file():
detail = (
"Generation failed; no audio available"
if generation.status == "failed"
else "Audio file not found"
)
raise HTTPException(status_code=404, detail=detail)
if audio_path is None or not audio_path.exists():
raise HTTPException(status_code=404, detail="Audio file not found")
return FileResponse(
audio_path,
@@ -77,7 +72,7 @@ async def get_sample_audio(sample_id: str, db: Session = Depends(get_db)):
raise HTTPException(status_code=404, detail="Sample not found")
audio_path = config.resolve_storage_path(sample.audio_path)
if audio_path is None or not audio_path.is_file():
if audio_path is None or not audio_path.exists():
raise HTTPException(status_code=404, detail="Audio file not found")
return FileResponse(
-75
View File
@@ -1,75 +0,0 @@
"""Voicebox Cloud device login routes.
The browser-based pairing flow:
1. POST /cloud/login/start — opens the browser to the cloud authorize page.
2. GET /cloud/callback — the browser lands here with a one-time code;
the backend exchanges it for an API key.
3. GET /cloud/status — the UI polls this to learn when it connected.
4. POST /cloud/disconnect — forget the local credential.
"""
import socket
from fastapi import APIRouter, Depends, Request
from fastapi.responses import HTMLResponse
from sqlalchemy.orm import Session
from .. import models
from ..database import get_db
from ..services import cloud as cloud_service
router = APIRouter(prefix="/cloud", tags=["cloud"])
def _callback_url(request: Request) -> str:
# Always loopback — the cloud only redirects codes to 127.0.0.1/localhost.
port = request.url.port or 17493
return f"http://127.0.0.1:{port}/cloud/callback"
@router.post("/login/start", response_model=models.CloudLoginStartResponse)
async def start_cloud_login(request: Request):
device_name = socket.gethostname() or "Desktop"
authorize_url = cloud_service.start_login(_callback_url(request), device_name)
return models.CloudLoginStartResponse(authorize_url=authorize_url)
@router.get("/callback", response_class=HTMLResponse)
async def cloud_callback(
request: Request,
code: str = "",
state: str = "",
db: Session = Depends(get_db),
):
ok, message = await cloud_service.handle_callback(db, code=code, state=state)
heading = "You're connected" if ok else "Couldn't connect"
accent = "#16a34a" if ok else "#dc2626"
sub = (
"Voicebox is now linked to your account. You can close this tab and return to the app."
if ok
else message
)
html = f"""<!doctype html>
<html lang="en"><head><meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Voicebox Cloud</title>
<style>
body {{ margin:0; min-height:100vh; display:flex; align-items:center; justify-content:center;
font-family: ui-sans-serif, system-ui, -apple-system, sans-serif; background:#0b0b0d; color:#e7e7ea; }}
.card {{ max-width:28rem; padding:2.5rem; text-align:center; }}
h1 {{ font-size:1.5rem; margin:0 0 .5rem; color:{accent}; }}
p {{ color:#a1a1aa; line-height:1.5; }}
</style></head>
<body><div class="card"><h1>{heading}</h1><p>{sub}</p></div></body></html>"""
return HTMLResponse(content=html, status_code=200 if ok else 400)
@router.get("/status", response_model=models.CloudStatusResponse)
async def cloud_status(db: Session = Depends(get_db)):
return models.CloudStatusResponse(**cloud_service.get_status(db))
@router.post("/disconnect", response_model=models.CloudStatusResponse)
async def cloud_disconnect(db: Session = Depends(get_db)):
cloud_service.disconnect(db)
return models.CloudStatusResponse(**cloud_service.get_status(db))
-4
View File
@@ -26,10 +26,6 @@ async def download_cuda_backend():
"""Download the CUDA backend binary."""
from ..services import cuda
unsupported_reason = cuda.get_cuda_download_unsupported_reason()
if unsupported_reason:
raise HTTPException(status_code=409, detail=unsupported_reason)
if cuda.get_cuda_binary_path() is not None:
raise HTTPException(status_code=409, detail="CUDA backend already downloaded")
+1 -13
View File
@@ -321,13 +321,7 @@ async def stream_speech(
db: Session = Depends(get_db),
):
"""Generate speech and stream the WAV audio directly without saving to disk."""
from ..backends import (
engine_needs_trim,
engine_retries_runaway,
ensure_model_cached_or_raise,
get_tts_backend_for_engine,
load_engine_model,
)
from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim
profile = await profiles.get_profile(data.profile_id, db)
if not profile:
@@ -353,15 +347,10 @@ async def stream_speech(
from ..utils.chunked_tts import generate_chunked
trim_fn = None
runaway_detector = None
if engine_needs_trim(engine):
from ..utils.audio import trim_tts_output
trim_fn = trim_tts_output
if engine_retries_runaway(engine):
from ..utils.audio import has_tts_runaway
runaway_detector = has_tts_runaway
audio, sample_rate = await generate_chunked(
tts_model,
@@ -373,7 +362,6 @@ async def stream_speech(
max_chunk_chars=data.max_chunk_chars,
crossfade_ms=data.crossfade_ms,
trim_fn=trim_fn,
runaway_detector=runaway_detector,
)
effects_chain_config = None
+2 -6
View File
@@ -151,9 +151,7 @@ async def export_generation(
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
if not safe_text:
safe_text = "generation"
# Append a short id so exports of similarly-worded generations don't collide
# on the same filename (the first 30 chars are frequently identical).
filename = f"generation-{safe_text}-{generation_id[:8]}.voicebox.zip"
filename = f"generation-{safe_text}.voicebox.zip"
return StreamingResponse(
io.BytesIO(zip_bytes),
@@ -182,9 +180,7 @@ async def export_generation_audio(
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
if not safe_text:
safe_text = "generation"
# Append a short id so exports of similarly-worded generations don't collide
# on the same filename (the first 30 chars are frequently identical).
filename = f"{safe_text}-{generation_id[:8]}.wav"
filename = f"{safe_text}.wav"
return FileResponse(
audio_path,
+1 -4
View File
@@ -231,10 +231,7 @@ async def get_model_status():
backend_type = get_backend_type()
task_manager = get_task_manager()
# Pending only — an errored task stays in the active list for the
# error/retry UI, but reporting it as "downloading" here would mask
# the model's real cache state until the app restarts (issue #925).
active_download_names = {task.model_name for task in task_manager.get_pending_downloads()}
active_download_names = {task.model_name for task in task_manager.get_active_downloads()}
try:
from huggingface_hub import scan_cache_dir
+1 -1
View File
@@ -232,7 +232,7 @@ async def upload_profile_avatar(
db: Session = Depends(get_db),
):
"""Upload or update avatar image for a profile."""
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename or "").suffix) as tmp:
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename).suffix) as tmp:
content = await file.read()
tmp.write(content)
tmp_path = tmp.name
+3 -24
View File
@@ -15,10 +15,6 @@ router = APIRouter()
UPLOAD_CHUNK_SIZE = 1024 * 1024 # 1MB
# Same set profiles.py accepts for voice samples. librosa picks its decoder from the
# file extension, so the temp file has to keep the uploaded one.
ALLOWED_AUDIO_EXTS = {".wav", ".mp3", ".m4a", ".ogg", ".flac", ".aac", ".webm", ".opus"}
@router.post("/transcribe", response_model=models.TranscriptionResponse)
async def transcribe_audio(
@@ -27,33 +23,18 @@ async def transcribe_audio(
model: str | None = Form(None),
):
"""Transcribe audio file to text."""
uploaded_ext = Path(file.filename or "").suffix.lower()
file_suffix = uploaded_ext if uploaded_ext in ALLOWED_AUDIO_EXTS else ".wav"
with tempfile.NamedTemporaryFile(suffix=file_suffix, delete=False) as tmp:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
tmp.write(chunk)
tmp_path = tmp.name
stt_path = tmp_path
try:
from ..utils.audio import load_audio, save_audio
from ..utils.audio import load_audio
from ..backends import WHISPER_HF_REPOS
audio, sr = await asyncio.to_thread(load_audio, tmp_path)
duration = len(audio) / sr
# The STT backend (mlx_audio.stt -> miniaudio) only decodes
# WAV/FLAC/MP3/Vorbis, so browser recordings uploaded as WebM/Opus
# fail with "unsupported file format" (issue: web-mode dictation).
# librosa already decoded the file above (it falls back to
# audioread/ffmpeg for exotic containers), so re-encode that PCM to a
# temp WAV and hand *that* to Whisper. WAV inputs pass through
# unchanged.
if file_suffix != ".wav":
stt_path = f"{tmp_path}.stt.wav"
await asyncio.to_thread(save_audio, audio, stt_path, sr)
whisper_model = transcribe.get_whisper_model()
model_size = model if model else whisper_model.model_size
@@ -88,7 +69,7 @@ async def transcribe_audio(
},
)
text = await whisper_model.transcribe(stt_path, language, model_size)
text = await whisper_model.transcribe(tmp_path, language, model_size)
return models.TranscriptionResponse(
text=text,
@@ -101,5 +82,3 @@ async def transcribe_audio(
raise HTTPException(status_code=500, detail=str(e))
finally:
Path(tmp_path).unlink(missing_ok=True)
if stt_path != tmp_path:
Path(stt_path).unlink(missing_ok=True)
-183
View File
@@ -1,183 +0,0 @@
"""
Voicebox Cloud device login — the "Log in with browser" flow.
The desktop opens the browser to ``{web}/connect``; the user authorizes while
signed in; the cloud redirects a single-use code back to this backend's loopback
callback. We exchange that code (server-to-server, over TLS) for a ``voicebox_…``
API key, verify the key against the API, and store it locally. The key never
travels through a browser URL, and an unfinished flow leaves nothing behind.
The ``state`` we mint and round-trip prevents login-CSRF: a callback whose state
we didn't issue (e.g. an attacker tricking the user into hitting the loopback
callback with their own code) is rejected.
"""
import logging
import secrets
import time
import webbrowser
from urllib.parse import urlencode
import httpx
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from .. import config
from ..database import CloudSettings as DBCloudSettings
logger = logging.getLogger(__name__)
SINGLETON_ID = 1
PENDING_TTL_SECONDS = 600 # the whole browser flow must finish within 10 min
# state -> expiry epoch. In-memory: a single backend process owns the flow, and a
# dropped pairing should simply be restarted.
_pending: dict[str, float] = {}
def _prune() -> None:
now = time.time()
for state, expiry in list(_pending.items()):
if expiry < now:
_pending.pop(state, None)
def _json_dict(response: httpx.Response) -> dict | None:
"""Parsed JSON body, or None when it isn't a JSON object."""
try:
payload = response.json()
except ValueError:
return None
return payload if isinstance(payload, dict) else None
def _consume_state(state: str) -> bool:
"""Validate and single-use-consume a pending state."""
_prune()
expiry = _pending.pop(state, None)
return expiry is not None and expiry >= time.time()
def start_login(callback_url: str, device_name: str) -> str:
"""Mint a state, build the authorize URL, and open the browser.
Returns the authorize URL (also opened here) so the caller can surface it as
a fallback if the browser didn't open.
"""
state = secrets.token_urlsafe(24)
_prune()
_pending[state] = time.time() + PENDING_TTL_SECONDS
params = urlencode({"redirect_uri": callback_url, "state": state, "name": device_name})
authorize_url = f"{config.get_cloud_web_url()}/connect?{params}"
try:
webbrowser.open(authorize_url)
except Exception: # pragma: no cover - platform dependent
logger.exception("failed to open browser for cloud login")
return authorize_url
async def handle_callback(db: Session, code: str, state: str) -> tuple[bool, str]:
"""Exchange the code for an API key and store it. Returns (ok, message)."""
if not _consume_state(state):
return False, "This sign-in link is invalid or has expired. Start again from the app."
if not code:
return False, "Missing authorization code."
web = config.get_cloud_web_url()
api = config.get_cloud_api_url()
try:
async with httpx.AsyncClient(timeout=15.0) as client:
exchanged = await client.post(f"{web}/api/connect/exchange", json={"code": code})
if exchanged.status_code != 200:
logger.warning("cloud exchange rejected code: %s", exchanged.status_code)
return False, "Could not complete sign-in — the code was rejected."
payload = _json_dict(exchanged)
if payload is None:
logger.warning("cloud exchange returned a non-JSON payload")
return False, "Voicebox Cloud returned an unexpected response."
api_key = payload.get("key")
device_name = payload.get("label")
if not api_key:
return False, "Voicebox Cloud did not return a key."
# Confirm the freshly minted key actually authenticates the API.
me = await client.get(
f"{api}/v1/account/me",
headers={"Authorization": f"Bearer {api_key}"},
)
if me.status_code != 200:
logger.warning("minted key failed verification: %s", me.status_code)
return False, "Sign-in succeeded but the key could not be verified."
# The 200 above proves the key works; the user id is best-effort.
data = (_json_dict(me) or {}).get("data")
account_user_id = data.get("userId") if isinstance(data, dict) else None
except httpx.HTTPError:
logger.exception("network error during cloud exchange")
return False, "Could not reach Voicebox Cloud. Check your connection and try again."
_store_key(db, api_key=api_key, device_name=device_name, account_user_id=account_user_id)
logger.info("connected to Voicebox Cloud as device %r", device_name)
return True, "Connected"
def _get_or_create_row(db: Session) -> DBCloudSettings:
row = db.query(DBCloudSettings).filter(DBCloudSettings.id == SINGLETON_ID).first()
if row is None:
row = DBCloudSettings(id=SINGLETON_ID)
db.add(row)
try:
db.commit()
except IntegrityError:
# Another request created the singleton concurrently.
db.rollback()
row = db.query(DBCloudSettings).filter(DBCloudSettings.id == SINGLETON_ID).one()
else:
db.refresh(row)
return row
def _store_key(db: Session, *, api_key: str, device_name: str | None, account_user_id: str | None):
from datetime import datetime
row = _get_or_create_row(db)
row.api_key = api_key
row.device_name = device_name
row.account_user_id = account_user_id
row.connected_at = datetime.utcnow()
db.commit()
def get_status(db: Session) -> dict:
"""Local view of the cloud link — never returns the full key."""
row = _get_or_create_row(db)
connected = bool(row.api_key)
# Prefix only: "voicebox_" (9) + 8 chars, matching the cloud's key_prefix.
key_prefix = row.api_key[:17] if row.api_key else None
return {
"connected": connected,
"device_name": row.device_name if connected else None,
"account_user_id": row.account_user_id if connected else None,
"key_prefix": key_prefix,
"connected_at": row.connected_at if connected else None,
"dashboard_url": f"{config.get_cloud_web_url()}/account",
}
def disconnect(db: Session) -> None:
"""Forget the local credential. The key remains valid on the server until
revoked from the account dashboard — surface that in the UI."""
row = _get_or_create_row(db)
row.api_key = None
row.device_name = None
row.account_user_id = None
row.connected_at = None
db.commit()
def get_api_key(db: Session) -> str | None:
"""The stored bearer key, for the (future) sync client. None if not linked."""
row = _get_or_create_row(db)
return row.api_key
+1 -32
View File
@@ -21,9 +21,9 @@ import tarfile
from pathlib import Path
from typing import Optional
from .. import __version__
from ..config import get_data_dir
from ..utils.progress import get_progress_manager
from .. import __version__
logger = logging.getLogger(__name__)
@@ -31,8 +31,6 @@ GITHUB_RELEASES_URL = "https://github.com/jamiepine/voicebox/releases/download"
PROGRESS_KEY = "cuda-backend"
CUDA_DOWNLOAD_UNSUPPORTED_REASON = "Downloadable CUDA backend releases are currently only published for Windows."
# The current expected CUDA libs version. Bump this when we change the
# CUDA toolkit version or torch's CUDA dependency changes (e.g. cu126 -> cu128).
CUDA_LIBS_VERSION = "cu128-v1"
@@ -65,25 +63,6 @@ def get_cuda_exe_name() -> str:
return "voicebox-server-cuda"
def is_cuda_download_supported() -> bool:
"""Return whether this platform has a matching CUDA release asset."""
return sys.platform == "win32"
def get_cuda_download_unsupported_reason() -> str | None:
"""Explain why this platform cannot use the release-download flow."""
if is_cuda_download_supported():
return None
return CUDA_DOWNLOAD_UNSUPPORTED_REASON
def ensure_cuda_download_supported() -> None:
"""Raise if downloading would fetch an asset built for another platform."""
reason = get_cuda_download_unsupported_reason()
if reason:
raise RuntimeError(reason)
def get_cuda_binary_path() -> Optional[Path]:
"""Return path to the CUDA executable if it exists inside the onedir."""
p = get_cuda_dir() / get_cuda_exe_name()
@@ -124,15 +103,12 @@ def get_cuda_status() -> dict:
cuda_path = get_cuda_binary_path()
progress = progress_manager.get_progress(PROGRESS_KEY)
cuda_libs_version = get_installed_cuda_libs_version()
unsupported_reason = get_cuda_download_unsupported_reason()
return {
"available": cuda_path is not None,
"active": is_cuda_active(),
"binary_path": str(cuda_path) if cuda_path else None,
"cuda_libs_version": cuda_libs_version,
"download_supported": unsupported_reason is None,
"unsupported_reason": unsupported_reason,
"downloading": progress is not None and progress.get("status") == "downloading",
"download_progress": progress,
}
@@ -281,8 +257,6 @@ async def download_cuda_binary(version: Optional[str] = None):
async def _download_cuda_binary_locked(version: Optional[str] = None):
"""Inner implementation of download_cuda_binary, called under _download_lock."""
ensure_cuda_download_supported()
import httpx
if version is None:
@@ -413,11 +387,6 @@ async def check_and_update_cuda_binary():
if not cuda_path:
return # No CUDA binary installed, nothing to update
unsupported_reason = get_cuda_download_unsupported_reason()
if unsupported_reason:
logger.info("Skipping CUDA backend auto-update: %s", unsupported_reason)
return
need_server = _needs_server_download()
need_libs = _needs_cuda_libs_download()
+4 -18
View File
@@ -48,14 +48,9 @@ async def run_generation(
This is the single entry point for all background generation work.
It is designed to be enqueued via ``services.task_queue.enqueue_generation``.
"""
from ..backends import (
engine_needs_trim,
engine_retries_runaway,
get_tts_backend_for_engine,
load_engine_model,
)
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
from ..utils.chunked_tts import generate_chunked
from ..utils.audio import has_tts_runaway, normalize_audio, save_audio, trim_tts_output
from ..utils.audio import normalize_audio, save_audio, trim_tts_output
task_manager = get_task_manager()
bg_db = next(get_db())
@@ -77,14 +72,12 @@ async def run_generation(
await history.update_generation_status(generation_id, "generating", bg_db)
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
gen_kwargs: dict = dict(
language=language,
seed=seed if mode != "regenerate" else None,
instruct=instruct,
trim_fn=trim_fn,
runaway_detector=runaway_detector,
)
if max_chunk_chars is not None:
gen_kwargs["max_chunk_chars"] = max_chunk_chars
@@ -274,14 +267,9 @@ async def generate_audio_sync(
normalize, then encodes in-memory via :func:`tts.audio_to_wav_bytes`
(same helper ``/generate/stream`` uses).
"""
from ..backends import (
engine_needs_trim,
engine_retries_runaway,
get_tts_backend_for_engine,
load_engine_model,
)
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
from ..utils.chunked_tts import generate_chunked
from ..utils.audio import has_tts_runaway, normalize_audio, trim_tts_output
from ..utils.audio import normalize_audio, trim_tts_output
from . import tts
bg_db = next(get_db())
@@ -299,14 +287,12 @@ async def generate_audio_sync(
bg_db.close()
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
gen_kwargs: dict = dict(
language=language,
seed=seed,
instruct=instruct,
trim_fn=trim_fn,
runaway_detector=runaway_detector,
)
if max_chunk_chars is not None:
gen_kwargs["max_chunk_chars"] = max_chunk_chars
+3 -15
View File
@@ -125,24 +125,12 @@ async def list_stories(
"""
stories = db.query(DBStory).order_by(DBStory.updated_at.desc()).all()
if not stories:
return []
# Batch-fetch all story item counts in one query to avoid an N+1 pattern
# (previously there was one COUNT query per story in the loop below).
story_ids = [s.id for s in stories]
count_rows = (
db.query(DBStoryItem.story_id, func.count(DBStoryItem.id).label("cnt"))
.filter(DBStoryItem.story_id.in_(story_ids))
.group_by(DBStoryItem.story_id)
.all()
)
item_counts = {row.story_id: row.cnt for row in count_rows}
result = []
for story in stories:
item_count = db.query(func.count(DBStoryItem.id)).filter(DBStoryItem.story_id == story.id).scalar()
response = StoryResponse.model_validate(story)
response.item_count = item_counts.get(story.id, 0)
response.item_count = item_count
result.append(response)
return result
@@ -1,164 +0,0 @@
"""
Regression tests for GET /audio/{generation_id} on failed generations.
A failed generation stores an empty ``audio_path``. Previously,
``config.resolve_storage_path("")`` resolved to the data directory itself,
which exists, so the route's 404 guard passed and ``FileResponse`` raised
``RuntimeError: File at path .../data is not a file`` — a 500 instead of
a clean 404.
Usage:
python -m pytest backend/tests/test_audio_failed_generation.py -v
"""
import sys
from pathlib import Path
import pytest
from fastapi import FastAPI
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from starlette.testclient import TestClient
# Repo root on sys.path so ``backend`` imports as a package (the audio
# routes use package-relative imports).
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
from backend import config
from backend.database import (
Base,
Generation,
GenerationVersion,
ProfileSample,
VoiceProfile,
get_db,
)
from backend.routes.audio import router as audio_router
def test_resolve_storage_path_empty_returns_none():
"""An empty stored path must not resolve to the data dir itself."""
assert config.resolve_storage_path("") is None
assert config.resolve_storage_path(None) is None
# Path("") is truthy, so it must be rejected via its (empty) parts.
assert config.resolve_storage_path(Path("")) is None
@pytest.fixture
def client(tmp_path, monkeypatch):
"""Minimal app with only the audio routes and a temp sqlite DB."""
monkeypatch.setattr(config, "_data_dir", tmp_path)
# An existing directory that a stored audio_path may wrongly point to.
(tmp_path / "somedir").mkdir()
engine = create_engine(
f"sqlite:///{tmp_path / 'test.db'}",
connect_args={"check_same_thread": False},
)
Base.metadata.create_all(bind=engine)
testing_session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
session = testing_session_local()
profile = VoiceProfile(id="profile-1", name="Test Profile")
session.add(profile)
session.add_all(
[
Generation(
id="gen-failed-empty",
profile_id="profile-1",
text="failed generation",
audio_path="",
status="failed",
error="engine exploded",
),
Generation(
id="gen-failed-null",
profile_id="profile-1",
text="failed generation",
audio_path=None,
status="failed",
),
Generation(
id="gen-missing-file",
profile_id="profile-1",
text="completed but file deleted",
audio_path="generations/does-not-exist.wav",
status="completed",
),
Generation(
id="gen-with-version",
profile_id="profile-1",
text="generation with a broken version",
audio_path="somedir",
status="completed",
),
GenerationVersion(
id="version-dir",
generation_id="gen-with-version",
label="original",
audio_path="somedir",
),
ProfileSample(
id="sample-dir",
profile_id="profile-1",
audio_path="somedir",
reference_text="sample pointing at a directory",
),
]
)
session.commit()
session.close()
app = FastAPI()
app.include_router(audio_router)
def override_get_db():
db = testing_session_local()
try:
yield db
finally:
db.close()
app.dependency_overrides[get_db] = override_get_db
return TestClient(app)
@pytest.mark.parametrize("generation_id", ["gen-failed-empty", "gen-failed-null"])
def test_failed_generation_returns_404(client, generation_id):
"""Failed generations (empty/null audio_path) get a clean 404, not a 500."""
response = client.get(f"/audio/{generation_id}")
assert response.status_code == 404
assert response.json()["detail"] == "Generation failed; no audio available"
def test_missing_audio_file_returns_404(client):
"""A completed generation whose file vanished still 404s."""
response = client.get("/audio/gen-missing-file")
assert response.status_code == 404
assert response.json()["detail"] == "Audio file not found"
def test_unknown_generation_returns_404(client):
response = client.get("/audio/no-such-generation")
assert response.status_code == 404
assert response.json()["detail"] == "Generation not found"
@pytest.mark.parametrize(
"url",
[
"/audio/gen-with-version",
"/audio/version/version-dir",
"/samples/sample-dir",
],
)
def test_audio_path_pointing_at_directory_returns_404(client, url):
"""A stored path resolving to an existing directory must 404, not 500.
Guards the is_file() checks: a directory passes exists() and would
crash FileResponse.
"""
response = client.get(url)
assert response.status_code == 404
assert response.json()["detail"] == "Audio file not found"
-123
View File
@@ -1,123 +0,0 @@
"""
Regression tests for issue #852: audioop removed from Python 3.13 stdlib.
Voice sample validation imports audioop transitively (librosa → audioread).
The audioop-lts backport must be declared in requirements and bundled in
PyInstaller builds on 3.13+.
"""
import re
import sys
from pathlib import Path
from unittest.mock import patch
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent))
from build_binary import build_server
@pytest.fixture
def backend_dir():
return Path(__file__).parent.parent
class TestAudioopRequirements:
def test_requirements_declare_audioop_lts_for_python_313(self, backend_dir):
content = (backend_dir / "requirements.txt").read_text()
assert re.search(
r"^audioop-lts.*python_version\s*>=\s*['\"]3\.13['\"]",
content,
re.MULTILINE,
), "requirements.txt must pin audioop-lts for Python 3.13+"
@pytest.mark.skipif(sys.version_info < (3, 13), reason="Python 3.13+ only")
class TestAudioopRuntime:
def test_audioop_importable(self):
import audioop # noqa: F401
def test_validate_reference_wav_does_not_fail_on_missing_audioop(self, tmp_path):
import numpy as np
import soundfile as sf
from utils.audio import validate_and_load_reference_audio
sr = 24000
t = np.arange(int(sr * 3), dtype=np.float32) / sr
audio = (0.3 * np.sin(2 * np.pi * 220 * t)).astype(np.float32)
path = tmp_path / "reference.wav"
sf.write(str(path), audio, sr)
ok, err, out_audio, out_sr = validate_and_load_reference_audio(str(path))
assert ok, err
assert out_audio is not None
assert out_sr == sr
assert "audioop" not in (err or "").lower()
class TestAudioopBuildArgs:
@staticmethod
def _hidden_imports(args):
imports = []
for i, arg in enumerate(args):
if arg == "--hidden-import" and i + 1 < len(args):
imports.append(args[i + 1])
return imports
def test_pyinstaller_includes_audioop_on_python_313(self):
class FakeVersionInfo(tuple):
@property
def major(self):
return self[0]
@property
def minor(self):
return self[1]
@property
def micro(self):
return self[2]
fake_313 = FakeVersionInfo((3, 13, 0, "final", 0))
with (
patch("build_binary.PyInstaller.__main__.run") as mock_run,
patch("build_binary.platform.system", return_value="Linux"),
patch("build_binary.is_apple_silicon", return_value=False),
patch("build_binary.os.chdir"),
patch("build_binary.sys.version_info", fake_313),
):
build_server()
args = mock_run.call_args[0][0]
assert "audioop" in self._hidden_imports(args)
def test_pyinstaller_omits_audioop_on_python_312(self):
class FakeVersionInfo(tuple):
@property
def major(self):
return self[0]
@property
def minor(self):
return self[1]
@property
def micro(self):
return self[2]
fake_312 = FakeVersionInfo((3, 12, 0, "final", 0))
with (
patch("build_binary.PyInstaller.__main__.run") as mock_run,
patch("build_binary.platform.system", return_value="Linux"),
patch("build_binary.is_apple_silicon", return_value=False),
patch("build_binary.os.chdir"),
patch("build_binary.sys.version_info", fake_312),
):
build_server()
args = mock_run.call_args[0][0]
assert "audioop" not in self._hidden_imports(args)
-32
View File
@@ -1,32 +0,0 @@
import sys as py_sys
import types
import pytest
from backend.services import cuda
def test_cuda_status_reports_unsupported_linux_download(monkeypatch, tmp_path):
monkeypatch.setattr(cuda.sys, "platform", "linux")
monkeypatch.setattr(cuda, "get_data_dir", lambda: tmp_path)
status = cuda.get_cuda_status()
assert status["available"] is False
assert status["download_supported"] is False
assert status["unsupported_reason"] == cuda.CUDA_DOWNLOAD_UNSUPPORTED_REASON
@pytest.mark.asyncio
async def test_cuda_download_rejects_linux_before_network(monkeypatch, tmp_path):
monkeypatch.setattr(cuda.sys, "platform", "linux")
monkeypatch.setattr(cuda, "get_data_dir", lambda: tmp_path)
class UnexpectedClient:
def __init__(self, *args, **kwargs):
raise AssertionError("unsupported platforms should not start a release download")
monkeypatch.setitem(py_sys.modules, "httpx", types.SimpleNamespace(AsyncClient=UnexpectedClient))
with pytest.raises(RuntimeError, match="currently only published for Windows"):
await cuda._download_cuda_binary_locked("v0.5.0")
@@ -1,68 +0,0 @@
"""Ensure TADA voice-prompt encoding disables autograd (#890)."""
from __future__ import annotations
from dataclasses import dataclass
from unittest.mock import AsyncMock
import numpy as np
import pytest
import soundfile as sf
import torch
from backend.backends.hume_backend import HumeTadaBackend
@dataclass
class _FakeEncoderOutput:
emb: torch.Tensor
class _GradTrackingEncoder:
"""Raises unless called under torch.inference_mode()."""
def __init__(self) -> None:
self.called_under_inference_mode = False
def __call__(self, audio, text=None, sample_rate=None):
self.called_under_inference_mode = torch.is_inference_mode_enabled()
if not self.called_under_inference_mode:
raise AssertionError("encoder forward must run under inference_mode")
# Touch a requires_grad tensor the way Snake1d alpha would.
alpha = torch.nn.Parameter(torch.ones(1, device=audio.device))
_ = audio.mean() * alpha
return _FakeEncoderOutput(emb=torch.zeros(1, 4, device=audio.device))
@pytest.mark.asyncio
async def test_create_voice_prompt_runs_encoder_under_inference_mode(tmp_path, monkeypatch):
wav = tmp_path / "ref.wav"
sf.write(str(wav), np.zeros(24000, dtype=np.float32), 24000)
backend = HumeTadaBackend()
backend.model = object() # mark loaded
backend.model_size = "1B"
backend._device = "cpu"
encoder = _GradTrackingEncoder()
backend.encoder = encoder
monkeypatch.setattr(backend, "load_model", AsyncMock(return_value=None))
monkeypatch.setattr(
"backend.backends.hume_backend.get_cached_voice_prompt",
lambda key: None,
)
monkeypatch.setattr(
"backend.backends.hume_backend.cache_voice_prompt",
lambda key, value: None,
)
prompt, from_cache = await backend.create_voice_prompt(
str(wav),
reference_text="hello world",
use_cache=False,
)
assert from_cache is False
assert encoder.called_under_inference_mode is True
assert isinstance(prompt["emb"], torch.Tensor)
assert prompt["emb"].device.type == "cpu"
-91
View File
@@ -1,91 +0,0 @@
"""Tests for the voicebox.speak MCP tool's ``model_size`` plumbing (issue #884).
The MCP speak path used to build its ``GenerationRequest`` without a
``model_size``, so every agent-triggered generation silently fell back to the
schema default ("1.7B") — there was no way to reach 0.6B (or TADA's 1B/3B)
through MCP. These tests pin the fix: ``_speak`` now forwards ``model_size``
straight into the request, matching the REST ``/generate`` surface.
"""
import pytest
from pydantic import ValidationError
import backend.routes.generations as generations
from backend.mcp_server import tools
class _FakeGeneration:
"""Minimal stand-in for GenerationResponse consumed by ``_speak_response``."""
def model_dump(self, mode="json"):
return {"id": "gen-test", "status": "generating"}
@pytest.fixture
def captured_request(monkeypatch):
"""Replace the real (torch-backed) generate_speech with a capturing stub.
``_speak`` imports ``generate_speech`` lazily from ``routes.generations``,
so patching the attribute on that module intercepts the call and lets us
inspect the ``GenerationRequest`` it would have run.
"""
captured = {}
async def fake_generate_speech(req, db):
captured["req"] = req
return _FakeGeneration()
monkeypatch.setattr(generations, "generate_speech", fake_generate_speech)
# Isolate the unit from the MCP event bus — _speak_response fires a
# speak-start event we don't care about here.
monkeypatch.setattr(tools.mcp_events, "publish", lambda *a, **k: None)
return captured
@pytest.mark.asyncio
async def test_speak_forwards_explicit_model_size(captured_request):
await tools._speak(
profile_id="p1",
profile_name="Morgan",
text="hello",
engine="qwen",
language="en",
personality=False,
model_size="0.6B",
db=None,
)
assert captured_request["req"].model_size == "0.6B"
@pytest.mark.asyncio
async def test_speak_omitted_model_size_is_none(captured_request):
# Omitted → None; generate_speech normalizes None to the engine default,
# so this reproduces the pre-fix behaviour for callers that don't ask.
await tools._speak(
profile_id="p1",
profile_name="Morgan",
text="hello",
engine="qwen",
language="en",
personality=False,
db=None,
)
assert captured_request["req"].model_size is None
@pytest.mark.asyncio
async def test_speak_rejects_invalid_model_size(captured_request):
# The GenerationRequest schema pattern is the single source of truth for
# valid sizes; a bad value is rejected before any generation runs.
with pytest.raises(ValidationError):
await tools._speak(
profile_id="p1",
profile_name="Morgan",
text="hello",
engine="qwen",
language="en",
personality=False,
model_size="9B",
db=None,
)
assert "req" not in captured_request
-55
View File
@@ -1,55 +0,0 @@
"""
Smoke test for the MLX backend dependencies on Apple Silicon.
Guards the `--no-deps` install of mlx-audio/mlx-lm done by `just setup-python`
and release.yml: those packages skip their declared dependencies (transformers
>=5.x conflict), so a missing transitive dep only surfaces at import time.
This test fails fast if the MLX STT/TTS entry points the backend uses stop
importing (e.g. the `miniaudio` regression from issue #505).
Usage:
python -m pytest backend/tests/test_mlx_smoke.py -v
"""
import platform
import sys
import pytest
pytestmark = pytest.mark.skipif(
not (sys.platform == "darwin" and platform.machine() == "arm64"),
reason="MLX packages are only installed on Apple Silicon macOS",
)
def test_mlx_core_runs():
"""The MLX runtime itself works (Metal array op)."""
import mlx.core as mx
assert mx.array([1, 2]).sum().item() == 3
def test_mlx_audio_tts_entry_point():
"""`from mlx_audio.tts import load` — used by MLXBackend.load_model_async."""
from mlx_audio.tts import load
assert callable(load)
def test_mlx_audio_stt_entry_point():
"""`from mlx_audio.stt import load` — used by the Whisper MLX STT path.
Importing mlx_audio.stt also pulls in miniaudio, so this catches the
ModuleNotFoundError from issue #505 on fresh installs.
"""
from mlx_audio.stt import load
assert callable(load)
def test_mlx_lm_entry_points():
"""`mlx_lm.load` / `mlx_lm.generate` — used by qwen_llm_backend."""
from mlx_lm import generate, load
assert callable(load)
assert callable(generate)
@@ -1,51 +0,0 @@
"""Errored downloads must not be reported as still downloading.
A failed download intentionally stays in the TaskManager with
``status="error"`` so ``/tasks/active`` can surface the error and retry
UI — but ``/models/status`` derives its ``downloading`` flag from the
same list. Without a status filter, one failed download shows the model
as "downloading" forever and masks its real cache state until the app
restarts (issue #925, symptom reports like #181).
"""
from backend.utils.tasks import TaskManager
def test_errored_download_is_not_pending():
tm = TaskManager()
tm.start_download("whisper-turbo")
assert [t.model_name for t in tm.get_pending_downloads()] == ["whisper-turbo"]
tm.error_download("whisper-turbo", "boom")
assert tm.get_pending_downloads() == []
# Still visible to /tasks/active for the error/retry UI.
active = tm.get_active_downloads()
assert [t.model_name for t in active] == ["whisper-turbo"]
assert active[0].status == "error"
assert active[0].error == "boom"
def test_retry_after_error_is_pending_again():
tm = TaskManager()
tm.start_download("qwen3-4b")
tm.error_download("qwen3-4b", "boom")
tm.start_download("qwen3-4b")
assert [t.model_name for t in tm.get_pending_downloads()] == ["qwen3-4b"]
def test_completed_download_is_removed_everywhere():
tm = TaskManager()
tm.start_download("whisper-turbo")
tm.complete_download("whisper-turbo")
assert tm.get_pending_downloads() == []
assert tm.get_active_downloads() == []
def test_cancel_dismisses_errored_download():
tm = TaskManager()
tm.start_download("whisper-turbo")
tm.error_download("whisper-turbo", "boom")
assert tm.cancel_download("whisper-turbo") is True
assert tm.get_active_downloads() == []
assert tm.get_pending_downloads() == []
-117
View File
@@ -1,117 +0,0 @@
"""Regression coverage for runaway MLX Qwen TTS output."""
from unittest.mock import patch
import numpy as np
import pytest
from backend.backends import engine_needs_trim, engine_retries_runaway
from backend.utils.audio import has_tts_runaway
from backend.utils.chunked_tts import generate_chunked
SAMPLE_RATE = 1000
def test_mlx_qwen_enables_runaway_retry_without_aggressive_trim():
with patch("backend.backends.get_backend_type", return_value="mlx"):
assert engine_needs_trim("qwen") is False
assert engine_retries_runaway("qwen") is True
def test_pytorch_qwen_keeps_runaway_retry_disabled():
with patch("backend.backends.get_backend_type", return_value="pytorch"):
assert engine_needs_trim("qwen") is False
assert engine_retries_runaway("qwen") is False
def test_detector_flags_long_internal_silence():
speech = np.full(2 * SAMPLE_RATE, 0.2, dtype=np.float32)
runaway_gap = np.zeros(2500, dtype=np.float32)
hallucinated_noise = np.full(2 * SAMPLE_RATE, 0.8, dtype=np.float32)
audio = np.concatenate([speech, runaway_gap, hallucinated_noise])
assert has_tts_runaway(audio, SAMPLE_RATE) is True
def test_detector_ignores_normal_internal_pause():
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
normal_pause = np.zeros(1200, dtype=np.float32)
audio = np.concatenate([speech, normal_pause, speech])
assert has_tts_runaway(audio, SAMPLE_RATE) is False
def test_trailing_silence_is_not_a_runaway():
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
trailing_silence = np.zeros(2 * SAMPLE_RATE, dtype=np.float32)
assert (
has_tts_runaway(
np.concatenate([speech, trailing_silence]),
SAMPLE_RATE,
)
is False
)
@pytest.mark.asyncio
async def test_runaway_chunk_is_retried_as_smaller_chunks():
class FakeBackend:
def __init__(self):
self.calls = []
async def generate(self, text, *_args):
self.calls.append(text)
if len(text) > 200:
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
silence = np.zeros(2500, dtype=np.float32)
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
return np.full(SAMPLE_RATE, 0.2, dtype=np.float32), SAMPLE_RATE
backend = FakeBackend()
text = f"{'A' * 119}. {'B' * 119}."
audio, sample_rate = await generate_chunked(
backend,
text,
{},
max_chunk_chars=800,
crossfade_ms=50,
runaway_detector=has_tts_runaway,
)
assert sample_rate == SAMPLE_RATE
assert backend.calls == [text, f"{'A' * 119}.", f"{'B' * 119}."]
assert len(audio) == 1950
@pytest.mark.asyncio
async def test_persistent_runaway_fails_instead_of_returning_corrupt_audio():
class AlwaysRunawayBackend:
def __init__(self):
self.calls = []
async def generate(self, text, *_args):
self.calls.append(text)
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
silence = np.zeros(2500, dtype=np.float32)
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
backend = AlwaysRunawayBackend()
text = f"{'A' * 119}. {'B' * 119}."
with pytest.raises(
RuntimeError,
match="remained unstable after retrying smaller text chunks",
):
await generate_chunked(
backend,
text,
{},
max_chunk_chars=800,
runaway_detector=has_tts_runaway,
)
assert [len(call) for call in backend.calls] == [241, 120, 100]
-37
View File
@@ -110,43 +110,6 @@ def save_audio(
raise OSError(f"Failed to save audio to {path}: {e}") from e
def has_tts_runaway(
audio: np.ndarray,
sample_rate: int = 24000,
frame_ms: int = 20,
silence_threshold_db: float = -40.0,
max_internal_silence_ms: int = 2000,
) -> bool:
"""Detect speech followed by a long silence and then more output.
This shape is a reliable signal that a TTS model missed EOS and resumed
with hallucinated speech or codec noise. Leading and trailing silence do
not count because they are not bounded by non-silent audio.
"""
frame_len = int(sample_rate * frame_ms / 1000)
if frame_len == 0 or len(audio) < frame_len:
return False
n_frames = len(audio) // frame_len
threshold_linear = 10 ** (silence_threshold_db / 20)
max_silence_frames = int(max_internal_silence_ms / frame_ms)
seen_speech = False
consecutive_silence = 0
for i in range(n_frames):
frame = audio[i * frame_len : (i + 1) * frame_len]
is_speech = np.sqrt(np.mean(frame**2)) >= threshold_linear
if is_speech:
if seen_speech and consecutive_silence >= max_silence_frames:
return True
seen_speech = True
consecutive_silence = 0
elif seen_speech:
consecutive_silence += 1
return False
def trim_tts_output(
audio: np.ndarray,
sample_rate: int = 24000,
+17 -65
View File
@@ -20,8 +20,6 @@ logger = logging.getLogger("voicebox.chunked-tts")
# Default chunk size in characters. Can be overridden per-request via
# the ``max_chunk_chars`` field on GenerationRequest.
DEFAULT_MAX_CHUNK_CHARS = 800
MAX_RUNAWAY_RETRIES = 2
MIN_RUNAWAY_RETRY_CHARS = 100
# Common abbreviations that should NOT be treated as sentence endings.
# Lowercase for case-insensitive matching.
@@ -213,7 +211,6 @@ async def generate_chunked(
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
crossfade_ms: int = 50,
trim_fn=None,
runaway_detector=None,
) -> Tuple[np.ndarray, int]:
"""Generate audio with automatic chunking for long text.
@@ -242,75 +239,25 @@ async def generate_chunked(
Optional ``(audio, sample_rate) -> audio`` post-processing
function applied to each chunk before concatenation (e.g.
``trim_tts_output`` for Chatterbox engines).
runaway_detector : callable | None
Optional ``(audio, sample_rate) -> bool`` detector. When it flags
unstable output, the affected text is split in half and retried.
Returns
-------
(audio, sample_rate) : Tuple[np.ndarray, int]
"""
async def generate_one(
chunk_text: str,
chunk_seed: int | None,
retry_depth: int = 0,
) -> tuple[np.ndarray, int]:
chunk_audio, chunk_sr = await backend.generate(
chunk_text,
voice_prompt,
language,
chunk_seed,
instruct,
)
if runaway_detector is not None and runaway_detector(chunk_audio, chunk_sr):
if retry_depth >= MAX_RUNAWAY_RETRIES or len(chunk_text) <= MIN_RUNAWAY_RETRY_CHARS:
raise RuntimeError(
"TTS output remained unstable after retrying smaller text chunks"
)
retry_max_chars = max(MIN_RUNAWAY_RETRY_CHARS, len(chunk_text) // 2)
retry_chunks = split_text_into_chunks(chunk_text, retry_max_chars)
if len(retry_chunks) <= 1:
raise RuntimeError("Unable to split unstable TTS output for retry")
logger.warning(
"Detected unstable TTS output for %d chars; retrying as %d smaller chunks",
len(chunk_text),
len(retry_chunks),
)
retry_audio: list[np.ndarray] = []
for i, retry_text in enumerate(retry_chunks):
retry_seed = (
chunk_seed + ((retry_depth + 1) * 1000) + i
if chunk_seed is not None
else None
)
audio, sample_rate = await generate_one(
retry_text,
retry_seed,
retry_depth + 1,
)
retry_audio.append(np.asarray(audio, dtype=np.float32))
return (
concatenate_audio_chunks(
retry_audio,
sample_rate,
crossfade_ms=crossfade_ms,
),
sample_rate,
)
if trim_fn is not None:
chunk_audio = trim_fn(chunk_audio, chunk_sr)
return np.asarray(chunk_audio, dtype=np.float32), chunk_sr
chunks = split_text_into_chunks(text, max_chunk_chars)
if len(chunks) <= 1:
# Short text — single-shot fast path
return await generate_one(text, seed)
audio, sample_rate = await backend.generate(
text,
voice_prompt,
language,
seed,
instruct,
)
if trim_fn is not None:
audio = trim_fn(audio, sample_rate)
return audio, sample_rate
# Long text — chunked generation
logger.info(
@@ -334,12 +281,17 @@ async def generate_chunked(
# always produces the same output.
chunk_seed = (seed + i) if seed is not None else None
chunk_audio, chunk_sr = await generate_one(
chunk_audio, chunk_sr = await backend.generate(
chunk_text,
voice_prompt,
language,
chunk_seed,
instruct,
)
if trim_fn is not None:
chunk_audio = trim_fn(chunk_audio, chunk_sr)
audio_chunks.append(chunk_audio)
audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
if sample_rate is None:
sample_rate = chunk_sr
-13
View File
@@ -67,19 +67,6 @@ class TaskManager:
def get_active_downloads(self) -> List[DownloadTask]:
"""Get all active downloads."""
return list(self._active_downloads.values())
def get_pending_downloads(self) -> List[DownloadTask]:
"""Get downloads that are still in flight.
Excludes errored tasks, which stay in the active list so the
error/retry UI can show them but must not be reported as
"downloading" by /models/status.
"""
return [
task
for task in self._active_downloads.values()
if task.status in ("downloading", "extracting")
]
def get_active_generations(self) -> List[GenerationTask]:
"""Get all active generations."""
+5
View File
@@ -57,6 +57,7 @@
"react-dom": "^18.3.0",
"react-hook-form": "^7.53.0",
"react-i18next": "^17.0.4",
"react-qr-code": "^2.0.18",
"react-sound-visualizer": "^1.4.0",
"tailwind-merge": "^2.5.4",
"wavesurfer.js": "^7.0.0",
@@ -1005,6 +1006,8 @@
"punycode": ["[email protected]", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
"qr.js": ["[email protected]", "", {}, "sha512-c4iYnWb+k2E+vYpRimHqSu575b1/wKl4XFeJGpFmrJQz5I88v9aY2czh7s0w36srfCM1sXgC/xpoJz5dJfq+OQ=="],
"queue-microtask": ["[email protected]", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="],
"react": ["[email protected]", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="],
@@ -1019,6 +1022,8 @@
"react-loaders": ["[email protected]", "", { "dependencies": { "classnames": "^2.2.3" }, "peerDependencies": { "prop-types": ">=15.6.0", "react": ">=15" } }, "sha512-4igMNqs9Fb3d4Z+0UHIGQNJsw/37gX0nUO8QxupnEKRn1dtyYC1LGwk5GuaoDciMQCQc/MmPwb4Fn6ZfdoX1FQ=="],
"react-qr-code": ["[email protected]", "", { "dependencies": { "prop-types": "^15.8.1", "qr.js": "0.0.0" }, "peerDependencies": { "react": "*" } }, "sha512-v1Jqz7urLMhkO6jkgJuBYhnqvXagzceg3qJUWayuCK/c6LTIonpWbwxR1f1APGd4xrW/QcQEovNrAojbUz65Tg=="],
"react-refresh": ["[email protected]", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
"react-remove-scroll": ["[email protected]", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="],
-12
View File
@@ -34,15 +34,3 @@ services:
# Tune the ROCm memory allocator
- PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,max_split_size_mb:512
# Redirect MIOpen kernel cache to a writable, persistent directory.
# Without this, MIOpen may fail to write its cache and throw
# miopenStatusUnknownError on fresh containers.
- MIOPEN_USER_DB_PATH=/app/data/cache/miopen_db
- MIOPEN_CUSTOM_CACHE_DIR=/app/data/cache/miopen_cache
# Use fast heuristics for kernel selection instead of exhaustive
# benchmarking. On RDNA4, exhaustive mode tries kernels that fail to
# allocate workspace memory (ptr: 0 size: 0), causing system stuttering
# on every generation even when the cache is present.
- MIOPEN_FIND_MODE=FAST
@@ -49,7 +49,6 @@ class ModelConfig:
model_size: str = "default"
size_mb: int = 0
needs_trim: bool = False
retries_runaway: bool = False
supports_instruct: bool = False
languages: list[str] = field(default_factory=lambda: ["en"])
```
@@ -60,7 +59,6 @@ Registry helpers in `backends/__init__.py` replace what used to be per-engine `i
- `get_tts_model_configs()` — only TTS variants
- `get_model_config(model_name)` — lookup by name
- `engine_needs_trim(engine)` — whether output should run through `trim_tts_output()`
- `engine_retries_runaway(engine)` — whether unstable output should be retried as smaller chunks
- `load_engine_model(engine, model_size)` — downloads + loads, handles engines with multiple sizes
- `get_tts_backend_for_engine(engine)` — thread-safe backend factory with double-checked locking
@@ -154,7 +152,7 @@ The request path from frontend to audio file:
6. **Inference** — the engine's `generate()` returns `(audio_array, sample_rate)`.
7. **Validate and post-process** — engines with `retries_runaway=True` retry unstable output as smaller chunks. If `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
7. **Post-process** — if `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
8. **Persist** — audio is written to the generations directory, a row is inserted into the `generations` table, and the response includes the generation metadata.
@@ -23,7 +23,7 @@ This page is for the cases where it doesn't:
| **Windows + NVIDIA** | PyTorch CUDA (cu128) | Auto-downloads the CUDA backend binary on first use |
| **Windows + Intel Arc** | PyTorch XPU (IPEX) | New in 0.4 — works with Arc A-series and B-series |
| **Windows generic GPU** | DirectML | Universal Windows GPU support; slower than CUDA |
| **Linux + NVIDIA** | PyTorch CUDA (cu128) | Use a local/remote Python backend with CUDA PyTorch |
| **Linux + NVIDIA** | PyTorch CUDA (cu128) | Same auto-download flow as Windows |
| **Linux + AMD** | PyTorch ROCm | Auto-configures `HSA_OVERRIDE_GFX_VERSION` |
| **Linux + Intel Arc** | PyTorch XPU (IPEX) | |
| **Any (no GPU)** | PyTorch CPU | Works everywhere; expect 5-50x slower than GPU |
@@ -46,7 +46,7 @@ On M-series Macs, Voicebox ships an MLX-optimized backend that uses the Apple Ne
The Whisper Turbo + MLX combo dropped transcription latency from ~20s to ~2-3s on M-series chips (see CHANGELOG entry for v0.1.10).
## Windows + NVIDIA — The CUDA Backend Swap
## Windows / Linux + NVIDIA — The CUDA Backend Swap
Voicebox doesn't bundle CUDA into the main installer (it would balloon downloads to multi-gigabyte territory for users who don't have an NVIDIA GPU). Instead, when you first need it, the app downloads a separate **CUDA backend binary** that contains the PyTorch + CUDA runtime.
+138
View File
@@ -0,0 +1,138 @@
---
title: "Hermes Agent"
description: "Use Voicebox as the voice and ears of Hermes Agent — spoken replies and voice-message transcription, fully local."
---
## Overview
[Hermes Agent](https://github.com/NousResearch/hermes-agent) is Nous
Research's open-source self-improving agent: a terminal CLI/TUI plus a
messaging gateway that connects one agent to Telegram, Discord, WhatsApp,
Slack, and Signal. It has first-class voice features — spoken replies,
voice-bubble delivery on chat platforms, push-to-talk dictation, and
automatic transcription of incoming voice messages — and every one of them
is pluggable.
Voicebox slots into both directions of that loop, entirely on-device:
- **Voice out** — Hermes speaks its replies in one of your cloned or preset
voices instead of a stock cloud voice.
- **Voice in** — voice messages and push-to-talk audio are transcribed by
the Whisper models already bundled with Voicebox. Audio never leaves your
machine.
There are two integration surfaces, and they compose — most people will
want both. Everything talks to the same local API
(`http://127.0.0.1:17493` while the Voicebox app is running).
<Callout type="info">
Running Voicebox in Docker instead of the desktop app? The API is on
`http://127.0.0.1:17600` — set `VOICEBOX_BASE_URL` accordingly wherever it
appears below. See [Docker](/overview/docker).
</Callout>
## MCP: agent-invoked voice tools
Hermes speaks MCP natively, and Voicebox ships a built-in
[MCP server](/overview/mcp-server). Voicebox is in Hermes's approved MCP
catalog, so:
```bash
hermes mcp install voicebox
```
(Or add the block manually to `~/.hermes/config.yaml`:)
```yaml
mcp_servers:
voicebox:
url: "http://127.0.0.1:17493/mcp"
headers:
X-Voicebox-Client-Id: "hermes"
```
Hermes discovers the tools — `voicebox.speak`, `voicebox.transcribe`,
`voicebox.list_profiles`, `voicebox.list_captures` — and the agent can now
*choose* to use them: "read me that summary in Morgan's voice" works
immediately, and the [per-client binding](/overview/mcp-server#per-client-bindings)
for `hermes` lets you pin its default voice from the Voicebox UI.
MCP makes Voicebox a set of tools the agent may call. It does **not**
reroute Hermes's own voice pipeline — spoken replies, voice bubbles, and
incoming voice-message transcription still use whatever `tts.provider` /
`stt.provider` are set to. That's the plugin's job.
## Provider plugin: Hermes's own voice pipeline
[`hermes-voicebox`](https://github.com/jamiepine/hermes-voicebox) registers
Voicebox as a Hermes **TTS provider** and **STT provider** via Hermes's
pluggable backend interfaces (`register_tts_provider` /
`register_transcription_provider` — see
[Build a Hermes Plugin](https://hermes-agent.nousresearch.com/docs/developer-guide/plugins)).
Once selected, the providers service the *entire* voice pipeline: every
spoken reply, every Telegram voice bubble, every incoming voice memo — plus
a bundled skill that teaches the agent when speaking aloud is appropriate
and to recall your dictated [Captures](/overview/captures) through MCP.
<Steps>
### Install the plugin
Into the same Python environment Hermes runs in:
```bash
pip install hermes-voicebox
```
No pip? Copy it in as a directory plugin instead:
```bash
git clone https://github.com/jamiepine/hermes-voicebox /tmp/hermes-voicebox
cp -r /tmp/hermes-voicebox/hermes_voicebox ~/.hermes/plugins/voicebox
hermes plugins enable voicebox
```
### Select the providers
In `~/.hermes/config.yaml`:
```yaml
tts:
provider: voicebox
stt:
provider: voicebox
```
### Try it
With the Voicebox app open, start `hermes chat` and ask it to say
something out loud — or send your Hermes bot a voice message on Telegram
and watch the transcript come back from your local Whisper.
</Steps>
## Behavior notes
- **Voicebox must be running.** The desktop app only serves the API while
it's open. Both providers implement availability as a live `/health`
check, so Hermes's provider picker reflects reality.
- **First generation is slower** while the TTS engine loads into memory;
subsequent calls are fast. Same for the first transcription with a new
Whisper size — Voicebox answers `202` while the model downloads, and the
plugin surfaces a friendly "try again in a minute".
- **Voice selection**: `tts.voice` in Hermes config (or the tool's `voice`
argument) accepts a Voicebox profile **name or id**. With no voice set,
the first profile is used.
- **Engines**: pass a Voicebox engine id (`qwen`, `kokoro`,
`chatterbox`, …) as the Hermes `model` to override the profile's
default engine.
## Next steps
- [MCP Server](/overview/mcp-server) — the tool-call route, per-client
bindings, and the speaking pill
- [Creating Voice Profiles](/overview/creating-voice-profiles) — clone the
voice Hermes will speak in
- [Remote Mode](/overview/remote-mode) — reaching a Voicebox instance on
another machine (read the security notes first: the API has no auth)
+1 -2
View File
@@ -75,8 +75,7 @@ No cloud fallback, no bring-your-own-API-key. Local is the product.
| Platform | Backend | Notes |
|----------|---------|-------|
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
| Windows (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
| Linux (NVIDIA) | PyTorch (CUDA) | Use a local/remote Python backend with CUDA PyTorch |
| Windows / Linux (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
| Windows (any GPU) | DirectML | Universal Windows GPU support |
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
+1
View File
@@ -13,6 +13,7 @@
"preset-voices",
"voice-personalities",
"mcp-server",
"hermes-agent",
"stories-editor",
"recording-transcription",
"generation-history",
+4 -4
View File
@@ -14,12 +14,12 @@ Make sure you have [installed Voicebox](/overview/installation) and launched the
Voice profiles are the foundation of Voicebox. Each profile contains voice samples that the AI uses to clone the voice.
<Steps>
<Step title="Navigate to Voices">
Click the **Voices** tab in the sidebar
<Step title="Navigate to Profiles">
Click the **Profiles** tab in the sidebar
</Step>
<Step title="Create New Voice">
Click the **+ New Voice** button
<Step title="Create New Profile">
Click the **+ New Profile** button
Fill in the details:
- **Name:** A descriptive name (e.g., "John Smith")
+7 -17
View File
@@ -72,12 +72,6 @@ setup-python:
if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
echo "Detected Apple Silicon — installing MLX dependencies..."
{{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt
# mlx-lm and mlx-audio declare transformers>=5.x, which conflicts with
# our transformers<=4.57.x cap, so install them --no-deps (their other
# runtime deps are covered by requirements.txt / requirements-mlx.txt —
# see the note in requirements-mlx.txt and .github/workflows/release.yml)
{{ pip }} install --no-deps mlx-lm==0.31.1
{{ pip }} install --no-deps mlx-audio==0.4.1
fi
{{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git
{{ pip }} install pyinstaller ruff pytest pytest-asyncio -q
@@ -95,10 +89,10 @@ setup-python:
}
Write-Host "Installing Python dependencies..."
& "{{ python }}" -m pip install --upgrade pip -q
$gpus = Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name; \
Write-Host "Detected GPUs: $($gpus -join ', ')"; \
$hasNvidia = ($gpus | Where-Object { $_ -match 'NVIDIA' }).Count -gt 0; \
$hasIntelArc = ($gpus | Where-Object { $_ -match 'Arc' }).Count -gt 0; \
$gpus = Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name
Write-Host "Detected GPUs: $($gpus -join ', ')"
$hasNvidia = ($gpus | Where-Object { $_ -match 'NVIDIA' }).Count -gt 0
$hasIntelArc = ($gpus | Where-Object { $_ -match 'Arc' }).Count -gt 0
if ($hasNvidia) { \
Write-Host "NVIDIA GPU detected — installing PyTorch with CUDA support..."; \
& "{{ pip }}" install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128; \
@@ -232,16 +226,12 @@ build-server: _ensure-venv
build-server: _ensure-venv
$ErrorActionPreference = "Stop"; \
$env:PATH = "{{ venv_bin }};$env:PATH"; \
$triple = (rustc --print host-tuple); \
New-Item -ItemType Directory -Path "{{ tauri_dir }}/src-tauri/binaries" -Force | Out-Null; \
& "{{ python }}" backend/build_binary.py; \
if ($LASTEXITCODE -ne 0) { throw "build_binary.py failed with exit code $LASTEXITCODE" }; \
$triple = (rustc --print host-tuple); \
New-Item -ItemType Directory -Path "{{ tauri_dir }}/src-tauri/binaries" -Force | Out-Null; \
Copy-Item "backend/dist/voicebox-server.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-server-$triple.exe" -Force; \
Write-Host "Copied sidecar: voicebox-server-$triple.exe"; \
& "{{ python }}" backend/build_binary.py --shim; \
if ($LASTEXITCODE -ne 0) { throw "build_binary.py --shim failed with exit code $LASTEXITCODE" }; \
Copy-Item "backend/dist/voicebox-mcp.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-mcp-$triple.exe" -Force; \
Write-Host "Copied sidecar: voicebox-mcp-$triple.exe"
Write-Host "Copied sidecar: voicebox-server-$triple.exe"
# Build CUDA server binary and place in app data dir for local testing
[windows]
+69 -45
View File
@@ -66,46 +66,13 @@ fn find_monitor_source_via_pactl() -> Option<String> {
None
}
/// Select the capture device: prefer an exact match against the monitor
/// source name reported by `pactl`, then fall back to any device whose name
/// contains "monitor", then the host's default input device.
fn select_capture_device(host: &cpal::Host, monitor_source: Option<&str>) -> Option<cpal::Device> {
let devices: Vec<cpal::Device> = host.input_devices().ok()?.collect();
if let Some(target) = monitor_source {
if let Some(pos) = devices
.iter()
.position(|d| d.name().map(|n| n == target).unwrap_or(false))
{
eprintln!(
"Linux audio capture: Using pactl monitor device: {}",
target
);
return devices.into_iter().nth(pos);
}
}
if let Some(pos) = devices.iter().position(|d| {
d.name()
.map(|n| n.to_lowercase().contains("monitor"))
.unwrap_or(false)
}) {
let name = devices[pos].name().unwrap_or_default();
eprintln!("Linux audio capture: Found monitor device by name: {}", name);
return devices.into_iter().nth(pos);
}
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
host.default_input_device()
}
/// Start capturing system audio on Linux using PulseAudio monitor sources.
///
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
/// monitor source via `pactl`, then select the matching cpal input device by
/// name. This avoids mutating the process environment (`PULSE_SOURCE`), which
/// is not thread-safe and would affect every thread in the process. If `pactl`
/// is unavailable, we fall back to searching cpal device names for "monitor".
/// monitor source via `pactl` and set the `PULSE_SOURCE` environment variable.
/// This tells PulseAudio's ALSA plugin to use the monitor as the default input
/// source for this process. If `pactl` is unavailable, we fall back to searching
/// cpal device names for "monitor".
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
@@ -134,16 +101,73 @@ pub async fn start_capture(
// Spawn capture on a dedicated thread
thread::spawn(move || {
let host = cpal::default_host();
// Try to set PULSE_SOURCE to a monitor before initializing cpal.
// This tells PulseAudio/PipeWire's ALSA plugin to use the monitor
// as the default input source for this process.
let monitor_source = find_monitor_source_via_pactl();
if let Some(ref source_name) = monitor_source {
eprintln!(
"Linux audio capture: Setting PULSE_SOURCE={}",
source_name
);
std::env::set_var("PULSE_SOURCE", source_name);
}
let device = match select_capture_device(&host, monitor_source.as_deref()) {
Some(d) => d,
None => {
let error_msg = "No audio input device available".to_string();
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
let host = cpal::default_host();
// Select the capture device.
// If PULSE_SOURCE was set, the default input device IS the monitor.
// Otherwise, fall back to searching device names for "monitor".
let device = if monitor_source.is_some() {
// PULSE_SOURCE was set — default input IS the monitor now
match host.default_input_device() {
Some(d) => {
let name = d.name().unwrap_or_default();
eprintln!(
"Linux audio capture: Using PULSE_SOURCE monitor device: {}",
name
);
d
}
None => {
let error_msg = "No audio input device available".to_string();
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
}
} else {
// pactl not available — try to find monitor by name (original approach)
let mut monitor_device = None;
if let Ok(devices) = host.input_devices() {
for d in devices {
if let Ok(name) = d.name() {
let name_lower = name.to_lowercase();
if name_lower.contains("monitor") {
eprintln!(
"Linux audio capture: Found monitor device by name: {}",
name
);
monitor_device = Some(d);
break;
}
}
}
}
match monitor_device {
Some(d) => d,
None => {
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
match host.default_input_device() {
Some(d) => d,
None => {
let error_msg = "No audio input device available".to_string();
eprintln!("{}", error_msg);
*error_arc.lock().unwrap() = Some(error_msg);
return;
}
}
}
}
};
-3
View File
@@ -264,9 +264,6 @@ fn apply_effect(app: &AppHandle, effect: Effect) {
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
}
}
// Skip on Linux: aborts if the window was never realized
// (see show_dictate_window in main.rs).
#[cfg(not(target_os = "linux"))]
let _ = window.set_ignore_cursor_events(false);
// Deliberately no set_focus() — taking key focus would yank
// it out of whatever app the user was typing in, which is
-1
View File
@@ -30,7 +30,6 @@ pub fn key_from_str(name: &str) -> Option<Key> {
"ShiftLeft" => Key::ShiftLeft,
"ShiftRight" => Key::ShiftRight,
"CapsLock" => Key::CapsLock,
"Function" => Key::Function,
// Whitespace / navigation
"Space" => Key::Space,
-4
View File
@@ -19,23 +19,19 @@
//! regardless of the active layout — most Windows apps treat that as
//! Ctrl+V. AutoHotkey relies on the same behaviour.
#[cfg(target_os = "macos")]
use std::sync::atomic::{AtomicU16, Ordering};
/// `kVK_ANSI_V` — the keycode for the physical V key on a US QWERTY
/// layout. Used as the fallback whenever live resolution can't produce a
/// better answer (no Unicode key layout data, lookup failure, non-macOS).
#[cfg(target_os = "macos")]
const FALLBACK_V_KEYCODE: u16 = 9;
#[cfg(target_os = "macos")]
static V_KEYCODE: AtomicU16 = AtomicU16::new(FALLBACK_V_KEYCODE);
/// Returns the keycode whose current-layout translation is `'v'`. Falls
/// back to `kVK_ANSI_V` when resolution hasn't run, the active input
/// source carries no Unicode key layout data, or no keycode in the layout
/// produces `v`.
#[cfg(target_os = "macos")]
pub fn paste_keycode_v() -> u16 {
V_KEYCODE.load(Ordering::Relaxed)
}
-7
View File
@@ -112,10 +112,6 @@ pub fn show_dictate_window(app: &tauri::AppHandle) {
let _ = window.set_position(PhysicalPosition::new(x, y));
}
}
// Skip on Linux: tao's CursorIgnoreEvents handler unwraps the GdkWindow,
// which is None until the window is first shown, aborting the process.
// The click-through toggle is a macOS workaround and is never set on Linux.
#[cfg(not(target_os = "linux"))]
let _ = window.set_ignore_cursor_events(false);
let _ = window.show();
}
@@ -1425,9 +1421,6 @@ pub fn run() {
let handle_for_hide = app.handle().clone();
app.handle().listen("dictate:hide", move |_event| {
if let Some(window) = handle_for_hide.get_webview_window(DICTATE_WINDOW_LABEL) {
// Skip on Linux: aborts if the window was never realized
// (see show_dictate_window).
#[cfg(not(target_os = "linux"))]
let _ = window.set_ignore_cursor_events(true);
let _ = window.set_position(PhysicalPosition::new(-10_000, -10_000));
let _ = window.hide();
+5 -20
View File
@@ -4,14 +4,10 @@
//! pipeline so the focused app performs its native paste action against
//! whatever the clipboard module has just staged.
//!
//! - **macOS** — Cmd down with Cmd flag, V down with Cmd flag, V up with
//! Cmd flag, Cmd up via `CGEventPost` at `kCGHIDEventTap`. The Cmd-down
//! event carries the Command flag so its `flagsChanged` representation
//! matches hardware — Electron/Chromium tracks modifier state from that
//! flag and drops the paste otherwise (see the note on the event table).
//! Accessibility permission is load-bearing: without it the system
//! swallows the events silently, so callers must gate on
//! [`crate::accessibility::is_trusted`].
//! - **macOS** — Cmd down, V down with Cmd flag, V up with Cmd flag, Cmd
//! up via `CGEventPost` at `kCGHIDEventTap`. Accessibility permission is
//! load-bearing: without it the system swallows the events silently, so
//! callers must gate on [`crate::accessibility::is_trusted`].
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
//! permission gate, but UAC/UIPI blocks delivery into elevated target
//! windows when we run non-elevated — nothing we can do short of also
@@ -105,18 +101,7 @@ pub fn send_paste() -> Result<(), String> {
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
let events = [
// The Cmd-down event must carry the Command flag itself. On real
// hardware the Cmd keyDown is a flagsChanged event whose flags
// already include Command; Chromium/Electron builds its tracked
// modifier state from that flag. Posting Cmd-down with flags = 0
// leaves that tracker showing "Command up", so the following V —
// even though its own flags carry Command — matches neither the
// Cmd+V accelerator (tracker says no modifier) nor plain-text
// insertion (event flags say Command), and Electron drops it
// silently. AppKit reads the V event's own flags and pastes
// regardless, which is why native apps worked but Electron
// targets (Slack, VS Code) silently no-op'd.
(KEYCODE_LEFT_CMD, true, K_CG_EVENT_FLAG_MASK_COMMAND),
(KEYCODE_LEFT_CMD, true, 0),
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
(KEYCODE_LEFT_CMD, false, 0),