mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-28 22:55:15 -07:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc553e94e5 | ||
|
|
68feec8305 | ||
|
|
126daf53a8 |
+1
-2
@@ -8,8 +8,7 @@ tauri/
|
|||||||
landing/
|
landing/
|
||||||
docs/
|
docs/
|
||||||
mlx-test/
|
mlx-test/
|
||||||
scripts/*
|
scripts/
|
||||||
!scripts/rocm-entrypoint.sh
|
|
||||||
|
|
||||||
# Dependencies & build artifacts (rebuilt in Docker)
|
# Dependencies & build artifacts (rebuilt in Docker)
|
||||||
node_modules/
|
node_modules/
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
package.json text eol=lf
|
|
||||||
scripts/*.sh text eol=lf
|
|
||||||
+1
-1
@@ -91,7 +91,7 @@ On Windows, to build with CUDA support for local testing:
|
|||||||
just build-local # Build CPU + CUDA server binaries + Tauri installer
|
just build-local # Build CPU + CUDA server binaries + Tauri installer
|
||||||
```
|
```
|
||||||
|
|
||||||
This builds the CPU sidecar (bundled with the app), the CUDA binary (placed in `%APPDATA%/sh.voicebox.app/backends/` for runtime GPU switching), and the installable Tauri app.
|
This builds the CPU sidecar (bundled with the app), the CUDA binary (placed in `%APPDATA%/com.voicebox.app/backends/` for runtime GPU switching), and the installable Tauri app.
|
||||||
|
|
||||||
Creates platform-specific installers (`.dmg`, `.msi`, `.AppImage`) in `tauri/src-tauri/target/release/bundle/`.
|
Creates platform-specific installers (`.dmg`, `.msi`, `.AppImage`) in `tauri/src-tauri/target/release/bundle/`.
|
||||||
|
|
||||||
|
|||||||
+3
-10
@@ -20,11 +20,8 @@ COPY package.json bun.lock CHANGELOG.md ./
|
|||||||
COPY app/ ./app/
|
COPY app/ ./app/
|
||||||
COPY web/ ./web/
|
COPY web/ ./web/
|
||||||
|
|
||||||
# Normalize line endings first (a Windows CRLF checkout would otherwise
|
# Strip workspaces not needed for web build, and fix trailing comma
|
||||||
# defeat the `-z 's/,\n ]/…/'` match below, since it's LF-anchored), then
|
RUN sed -i '/"tauri"/d; /"landing"/d' package.json && \
|
||||||
# strip workspaces not needed for web build, and fix trailing comma
|
|
||||||
RUN sed -i 's/\r$//' package.json && \
|
|
||||||
sed -i '/"tauri"/d; /"landing"/d' package.json && \
|
|
||||||
sed -i -z 's/,\n ]/\n ]/' package.json
|
sed -i -z 's/,\n ]/\n ]/' package.json
|
||||||
RUN bun install --no-save
|
RUN bun install --no-save
|
||||||
# Build frontend (skip tsc — upstream has pre-existing type errors)
|
# Build frontend (skip tsc — upstream has pre-existing type errors)
|
||||||
@@ -103,11 +100,7 @@ EXPOSE 17493
|
|||||||
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=60s \
|
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=60s \
|
||||||
CMD curl -f http://localhost:17493/health || exit 1
|
CMD curl -f http://localhost:17493/health || exit 1
|
||||||
|
|
||||||
# Entrypoint joins GPU groups then drops to the voicebox user.
|
# Entrypoint joins GPU groups then drops to the voicebox user
|
||||||
# Normalize CRLF (a Windows checkout otherwise leaves the shebang as
|
|
||||||
# `#!/bin/sh\r`, which Linux can't resolve — reported as a misleading
|
|
||||||
# "no such file or directory" even though the file exists).
|
|
||||||
COPY --chmod=755 scripts/rocm-entrypoint.sh /usr/local/bin/entrypoint.sh
|
COPY --chmod=755 scripts/rocm-entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||||
RUN sed -i 's/\r$//' /usr/local/bin/entrypoint.sh
|
|
||||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "17493"]
|
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "17493"]
|
||||||
|
|||||||
@@ -270,8 +270,7 @@ Use cases: agent dev loops (dictate a question, hear the answer in a cloned voic
|
|||||||
| Platform | Backend | Notes |
|
| Platform | Backend | Notes |
|
||||||
| ------------------------ | -------------- | ---------------------------------------------- |
|
| ------------------------ | -------------- | ---------------------------------------------- |
|
||||||
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
|
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
|
||||||
| Windows (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
|
| Windows / Linux (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
|
||||||
| Linux (NVIDIA) | PyTorch (CUDA) | Use a local/remote Python backend with CUDA PyTorch |
|
|
||||||
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
|
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
|
||||||
| Windows (any GPU) | DirectML | Universal Windows GPU support |
|
| Windows (any GPU) | DirectML | Universal Windows GPU support |
|
||||||
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
|
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
|
||||||
|
|||||||
@@ -139,7 +139,7 @@ export function EngineModelSelector({ form, compact, selectedProfile }: EngineMo
|
|||||||
<SelectValue />
|
<SelectValue />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
</FormControl>
|
</FormControl>
|
||||||
<SelectContent side={compact ? 'top' : undefined}>
|
<SelectContent>
|
||||||
{availableOptions.map((opt) => (
|
{availableOptions.map((opt) => (
|
||||||
<SelectItem key={opt.value} value={opt.value} className={itemClass}>
|
<SelectItem key={opt.value} value={opt.value} className={itemClass}>
|
||||||
{opt.label}
|
{opt.label}
|
||||||
|
|||||||
@@ -555,7 +555,7 @@ export function FloatingGenerateBox({
|
|||||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all w-full">
|
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all w-full">
|
||||||
<SelectValue placeholder={t('generation.voiceSelector.placeholder')} />
|
<SelectValue placeholder={t('generation.voiceSelector.placeholder')} />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent side="top">
|
<SelectContent>
|
||||||
{profiles?.map((profile) => (
|
{profiles?.map((profile) => (
|
||||||
<SelectItem key={profile.id} value={profile.id} className="text-xs">
|
<SelectItem key={profile.id} value={profile.id} className="text-xs">
|
||||||
{profile.name}
|
{profile.name}
|
||||||
@@ -582,7 +582,7 @@ export function FloatingGenerateBox({
|
|||||||
<SelectValue />
|
<SelectValue />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
</FormControl>
|
</FormControl>
|
||||||
<SelectContent side="top">
|
<SelectContent>
|
||||||
{engineLangs.map((lang) => (
|
{engineLangs.map((lang) => (
|
||||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||||
{lang.label}
|
{lang.label}
|
||||||
@@ -610,7 +610,7 @@ export function FloatingGenerateBox({
|
|||||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||||
<SelectValue placeholder={t('generation.effects.none')} />
|
<SelectValue placeholder={t('generation.effects.none')} />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent side="top">
|
<SelectContent>
|
||||||
<SelectItem value="none" className="text-xs">
|
<SelectItem value="none" className="text-xs">
|
||||||
{t('generation.effects.none')}
|
{t('generation.effects.none')}
|
||||||
</SelectItem>
|
</SelectItem>
|
||||||
|
|||||||
@@ -1,151 +0,0 @@
|
|||||||
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
|
|
||||||
import { Cloud, Loader2 } from 'lucide-react';
|
|
||||||
import { useEffect, useState } from 'react';
|
|
||||||
import { Button } from '@/components/ui/button';
|
|
||||||
import { useToast } from '@/components/ui/use-toast';
|
|
||||||
import { apiClient } from '@/lib/api/client';
|
|
||||||
import { SettingRow, SettingSection } from './SettingRow';
|
|
||||||
|
|
||||||
// "Log in with browser" device pairing. The backend opens the system browser
|
|
||||||
// and completes the code exchange; here we just kick it off and poll status
|
|
||||||
// until the link goes live. The API key never touches the frontend.
|
|
||||||
export function CloudSection() {
|
|
||||||
const { toast } = useToast();
|
|
||||||
const queryClient = useQueryClient();
|
|
||||||
const [polling, setPolling] = useState(false);
|
|
||||||
|
|
||||||
const { data: status } = useQuery({
|
|
||||||
queryKey: ['cloud-status'],
|
|
||||||
queryFn: () => apiClient.getCloudStatus(),
|
|
||||||
refetchInterval: polling ? 2000 : false,
|
|
||||||
});
|
|
||||||
|
|
||||||
const connected = status?.connected ?? false;
|
|
||||||
|
|
||||||
// Once the browser flow completes, stop polling and celebrate.
|
|
||||||
useEffect(() => {
|
|
||||||
if (connected && polling) {
|
|
||||||
setPolling(false);
|
|
||||||
toast({
|
|
||||||
title: 'Connected to Voicebox Cloud',
|
|
||||||
description: `Linked as ${status?.device_name ?? 'this device'}.`,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}, [connected, polling, status?.device_name, toast]);
|
|
||||||
|
|
||||||
// Give up after two minutes so an abandoned browser flow doesn't leave the
|
|
||||||
// button stuck on "Waiting for browser…". The backend state stays valid for
|
|
||||||
// ten, so the user can simply start again.
|
|
||||||
useEffect(() => {
|
|
||||||
if (!polling) return;
|
|
||||||
const timeoutId = window.setTimeout(() => {
|
|
||||||
setPolling(false);
|
|
||||||
toast({
|
|
||||||
title: 'Sign-in timed out',
|
|
||||||
description: 'The browser sign-in was not completed. Try again.',
|
|
||||||
variant: 'destructive',
|
|
||||||
});
|
|
||||||
}, 120_000);
|
|
||||||
return () => window.clearTimeout(timeoutId);
|
|
||||||
}, [polling, toast]);
|
|
||||||
|
|
||||||
const startLogin = useMutation({
|
|
||||||
mutationFn: () => apiClient.startCloudLogin(),
|
|
||||||
onSuccess: () => {
|
|
||||||
setPolling(true);
|
|
||||||
toast({
|
|
||||||
title: 'Continue in your browser',
|
|
||||||
description: 'Authorize this device, then return here.',
|
|
||||||
});
|
|
||||||
},
|
|
||||||
onError: (error: Error) =>
|
|
||||||
toast({
|
|
||||||
title: 'Could not start sign-in',
|
|
||||||
description: error.message,
|
|
||||||
variant: 'destructive',
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
const disconnect = useMutation({
|
|
||||||
mutationFn: () => apiClient.disconnectCloud(),
|
|
||||||
onSuccess: () => {
|
|
||||||
queryClient.invalidateQueries({ queryKey: ['cloud-status'] });
|
|
||||||
toast({
|
|
||||||
title: 'Disconnected',
|
|
||||||
description:
|
|
||||||
'This device is no longer linked. The key stays valid until revoked in your account.',
|
|
||||||
});
|
|
||||||
},
|
|
||||||
onError: (error: Error) =>
|
|
||||||
toast({ title: 'Could not disconnect', description: error.message, variant: 'destructive' }),
|
|
||||||
});
|
|
||||||
|
|
||||||
const busy = startLogin.isPending || polling;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<SettingSection
|
|
||||||
title="Voicebox Cloud"
|
|
||||||
description="End-to-end encrypted backup & sync across your devices."
|
|
||||||
>
|
|
||||||
<SettingRow
|
|
||||||
title={connected ? 'Connected' : 'Account'}
|
|
||||||
description={
|
|
||||||
connected
|
|
||||||
? `Linked as ${status?.device_name ?? 'this device'}${
|
|
||||||
status?.key_prefix ? ` · ${status.key_prefix}…` : ''
|
|
||||||
}`
|
|
||||||
: 'Log in to back up and sync your captures and generations.'
|
|
||||||
}
|
|
||||||
action={
|
|
||||||
connected ? (
|
|
||||||
<Button
|
|
||||||
disabled={disconnect.isPending}
|
|
||||||
onClick={() => disconnect.mutate()}
|
|
||||||
size="sm"
|
|
||||||
variant="outline"
|
|
||||||
>
|
|
||||||
{disconnect.isPending ? (
|
|
||||||
<>
|
|
||||||
<Loader2 className="h-3.5 w-3.5 mr-1.5 animate-spin" />
|
|
||||||
Disconnecting…
|
|
||||||
</>
|
|
||||||
) : (
|
|
||||||
'Disconnect'
|
|
||||||
)}
|
|
||||||
</Button>
|
|
||||||
) : (
|
|
||||||
<Button disabled={busy} onClick={() => startLogin.mutate()} size="sm">
|
|
||||||
{busy ? (
|
|
||||||
<>
|
|
||||||
<Loader2 className="h-3.5 w-3.5 mr-1.5 animate-spin" />
|
|
||||||
{polling ? 'Waiting for browser…' : 'Opening…'}
|
|
||||||
</>
|
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
<Cloud className="h-3.5 w-3.5 mr-1.5" />
|
|
||||||
Log in with browser
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</Button>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
|
|
||||||
{connected && (
|
|
||||||
<SettingRow
|
|
||||||
title="Manage"
|
|
||||||
description="Revoke this device, add API keys, or manage billing from your account."
|
|
||||||
>
|
|
||||||
<a
|
|
||||||
className="text-sm text-accent hover:underline"
|
|
||||||
href={status?.dashboard_url ?? 'https://voicebox.sh/account'}
|
|
||||||
rel="noopener noreferrer"
|
|
||||||
target="_blank"
|
|
||||||
>
|
|
||||||
Open account dashboard ↗
|
|
||||||
</a>
|
|
||||||
</SettingRow>
|
|
||||||
)}
|
|
||||||
</SettingSection>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -14,7 +14,6 @@ import { useAutoUpdater } from '@/hooks/useAutoUpdater';
|
|||||||
import { useServerHealth } from '@/lib/hooks/useServer';
|
import { useServerHealth } from '@/lib/hooks/useServer';
|
||||||
import { usePlatform } from '@/platform/PlatformContext';
|
import { usePlatform } from '@/platform/PlatformContext';
|
||||||
import { useServerStore } from '@/stores/serverStore';
|
import { useServerStore } from '@/stores/serverStore';
|
||||||
import { CloudSection } from './CloudSection';
|
|
||||||
import { LanguageSelect } from './LanguageSelect';
|
import { LanguageSelect } from './LanguageSelect';
|
||||||
import { SettingRow, SettingSection } from './SettingRow';
|
import { SettingRow, SettingSection } from './SettingRow';
|
||||||
import { ThemeSelect } from './ThemeSelect';
|
import { ThemeSelect } from './ThemeSelect';
|
||||||
@@ -208,8 +207,6 @@ export function GeneralPage() {
|
|||||||
/>
|
/>
|
||||||
</SettingSection>
|
</SettingSection>
|
||||||
|
|
||||||
<CloudSection />
|
|
||||||
|
|
||||||
<ApiReferenceCard serverUrl={serverUrl} />
|
<ApiReferenceCard serverUrl={serverUrl} />
|
||||||
|
|
||||||
{platform.metadata.isTauri && <UpdatesSection />}
|
{platform.metadata.isTauri && <UpdatesSection />}
|
||||||
|
|||||||
+1
-10
@@ -2,25 +2,19 @@ import i18n from 'i18next';
|
|||||||
import LanguageDetector from 'i18next-browser-languagedetector';
|
import LanguageDetector from 'i18next-browser-languagedetector';
|
||||||
import { initReactI18next } from 'react-i18next';
|
import { initReactI18next } from 'react-i18next';
|
||||||
import en from './locales/en/translation.json';
|
import en from './locales/en/translation.json';
|
||||||
import es from './locales/es/translation.json';
|
|
||||||
import fr from './locales/fr/translation.json';
|
|
||||||
import it from './locales/it/translation.json';
|
|
||||||
import ja from './locales/ja/translation.json';
|
import ja from './locales/ja/translation.json';
|
||||||
import ko from './locales/ko/translation.json';
|
|
||||||
import ptBR from './locales/pt-BR/translation.json';
|
import ptBR from './locales/pt-BR/translation.json';
|
||||||
import zhCN from './locales/zh-CN/translation.json';
|
import zhCN from './locales/zh-CN/translation.json';
|
||||||
import zhTW from './locales/zh-TW/translation.json';
|
import zhTW from './locales/zh-TW/translation.json';
|
||||||
|
import fr from './locales/fr/translation.json';
|
||||||
|
|
||||||
export const SUPPORTED_LANGUAGES = [
|
export const SUPPORTED_LANGUAGES = [
|
||||||
{ code: 'en', label: 'English' },
|
{ code: 'en', label: 'English' },
|
||||||
{ code: 'es', label: 'Español' },
|
|
||||||
{ code: 'pt-BR', label: 'Português (Brasil)' },
|
{ code: 'pt-BR', label: 'Português (Brasil)' },
|
||||||
{ code: 'ja', label: '日本語' },
|
{ code: 'ja', label: '日本語' },
|
||||||
{ code: 'ko', label: '한국어' },
|
|
||||||
{ code: 'zh-CN', label: '简体中文' },
|
{ code: 'zh-CN', label: '简体中文' },
|
||||||
{ code: 'zh-TW', label: '繁體中文' },
|
{ code: 'zh-TW', label: '繁體中文' },
|
||||||
{ code: 'fr', label: 'Français' },
|
{ code: 'fr', label: 'Français' },
|
||||||
{ code: 'it', label: 'Italiano' },
|
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
export type LanguageCode = (typeof SUPPORTED_LANGUAGES)[number]['code'];
|
export type LanguageCode = (typeof SUPPORTED_LANGUAGES)[number]['code'];
|
||||||
@@ -31,14 +25,11 @@ i18n
|
|||||||
.init({
|
.init({
|
||||||
resources: {
|
resources: {
|
||||||
en: { translation: en },
|
en: { translation: en },
|
||||||
es: { translation: es },
|
|
||||||
'pt-BR': { translation: ptBR },
|
'pt-BR': { translation: ptBR },
|
||||||
ja: { translation: ja },
|
ja: { translation: ja },
|
||||||
ko: { translation: ko },
|
|
||||||
'zh-CN': { translation: zhCN },
|
'zh-CN': { translation: zhCN },
|
||||||
'zh-TW': { translation: zhTW },
|
'zh-TW': { translation: zhTW },
|
||||||
fr: { translation: fr },
|
fr: { translation: fr },
|
||||||
it: { translation: it },
|
|
||||||
},
|
},
|
||||||
fallbackLng: 'en',
|
fallbackLng: 'en',
|
||||||
supportedLngs: SUPPORTED_LANGUAGES.map((l) => l.code),
|
supportedLngs: SUPPORTED_LANGUAGES.map((l) => l.code),
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -51,8 +51,6 @@ import type {
|
|||||||
MCPClientBinding,
|
MCPClientBinding,
|
||||||
MCPClientBindingListResponse,
|
MCPClientBindingListResponse,
|
||||||
MCPClientBindingUpsert,
|
MCPClientBindingUpsert,
|
||||||
CloudLoginStartResponse,
|
|
||||||
CloudStatus,
|
|
||||||
} from './types';
|
} from './types';
|
||||||
|
|
||||||
function formatErrorDetail(detail: unknown, fallback: string): string {
|
function formatErrorDetail(detail: unknown, fallback: string): string {
|
||||||
@@ -940,21 +938,6 @@ class ApiClient {
|
|||||||
|
|
||||||
return response.blob();
|
return response.blob();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cloud (backup & sync) — browser-based device login. startCloudLogin opens
|
|
||||||
// the system browser server-side; the UI then polls getCloudStatus until the
|
|
||||||
// backend completes the exchange and the link goes live.
|
|
||||||
async getCloudStatus(): Promise<CloudStatus> {
|
|
||||||
return this.request<CloudStatus>('/cloud/status');
|
|
||||||
}
|
|
||||||
|
|
||||||
async startCloudLogin(): Promise<CloudLoginStartResponse> {
|
|
||||||
return this.request<CloudLoginStartResponse>('/cloud/login/start', { method: 'POST' });
|
|
||||||
}
|
|
||||||
|
|
||||||
async disconnectCloud(): Promise<CloudStatus> {
|
|
||||||
return this.request<CloudStatus>('/cloud/disconnect', { method: 'POST' });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export const apiClient = new ApiClient();
|
export const apiClient = new ApiClient();
|
||||||
|
|||||||
@@ -287,10 +287,7 @@ export interface CudaDownloadProgress {
|
|||||||
export interface CudaStatus {
|
export interface CudaStatus {
|
||||||
available: boolean; // CUDA binary exists on disk
|
available: boolean; // CUDA binary exists on disk
|
||||||
active: boolean; // Currently running the CUDA binary
|
active: boolean; // Currently running the CUDA binary
|
||||||
binary_path: string | null;
|
binary_path?: string;
|
||||||
cuda_libs_version: string | null;
|
|
||||||
download_supported: boolean; // Platform has a matching release asset
|
|
||||||
unsupported_reason: string | null;
|
|
||||||
downloading: boolean; // Download in progress
|
downloading: boolean; // Download in progress
|
||||||
download_progress?: CudaDownloadProgress;
|
download_progress?: CudaDownloadProgress;
|
||||||
}
|
}
|
||||||
@@ -545,18 +542,3 @@ export interface MCPClientBindingUpsert {
|
|||||||
export interface MCPClientBindingListResponse {
|
export interface MCPClientBindingListResponse {
|
||||||
items: MCPClientBinding[];
|
items: MCPClientBinding[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ─── Cloud (backup & sync) ───────────────────────────────────────────── */
|
|
||||||
|
|
||||||
export interface CloudLoginStartResponse {
|
|
||||||
authorize_url: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CloudStatus {
|
|
||||||
connected: boolean;
|
|
||||||
device_name: string | null;
|
|
||||||
account_user_id: string | null;
|
|
||||||
key_prefix: string | null;
|
|
||||||
connected_at: string | null;
|
|
||||||
dashboard_url: string;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -47,14 +47,12 @@ export function useExportGeneration() {
|
|||||||
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
||||||
const blob = await apiClient.exportGeneration(generationId);
|
const blob = await apiClient.exportGeneration(generationId);
|
||||||
|
|
||||||
// Create safe filename from text. Append a short id so exports of
|
// Create safe filename from text
|
||||||
// similarly-worded generations don't collide on the same filename
|
|
||||||
// (the first 30 chars are frequently identical).
|
|
||||||
const safeText = text
|
const safeText = text
|
||||||
.substring(0, 30)
|
.substring(0, 30)
|
||||||
.replace(/[^a-z0-9]/gi, '-')
|
.replace(/[^a-z0-9]/gi, '-')
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
const filename = `generation-${safeText}-${generationId.substring(0, 8)}.voicebox.zip`;
|
const filename = `generation-${safeText}.voicebox.zip`;
|
||||||
|
|
||||||
await platform.filesystem.saveFile(filename, blob, [
|
await platform.filesystem.saveFile(filename, blob, [
|
||||||
{
|
{
|
||||||
@@ -75,14 +73,12 @@ export function useExportGenerationAudio() {
|
|||||||
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
||||||
const blob = await apiClient.exportGenerationAudio(generationId);
|
const blob = await apiClient.exportGenerationAudio(generationId);
|
||||||
|
|
||||||
// Create safe filename from text. Append a short id so exports of
|
// Create safe filename from text
|
||||||
// similarly-worded generations don't collide on the same filename
|
|
||||||
// (the first 30 chars are frequently identical).
|
|
||||||
const safeText = text
|
const safeText = text
|
||||||
.substring(0, 30)
|
.substring(0, 30)
|
||||||
.replace(/[^a-z0-9]/gi, '-')
|
.replace(/[^a-z0-9]/gi, '-')
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
const filename = `${safeText}-${generationId.substring(0, 8)}.wav`;
|
const filename = `${safeText}.wav`;
|
||||||
|
|
||||||
await platform.filesystem.saveFile(filename, blob, [
|
await platform.filesystem.saveFile(filename, blob, [
|
||||||
{
|
{
|
||||||
|
|||||||
+11
-14
@@ -1,5 +1,5 @@
|
|||||||
import { formatDistance } from 'date-fns';
|
import { formatDistance } from 'date-fns';
|
||||||
import { es, fr, ja, zhCN, zhTW } from 'date-fns/locale';
|
import { ja, zhCN, zhTW, fr } from 'date-fns/locale';
|
||||||
import i18n from '@/i18n';
|
import i18n from '@/i18n';
|
||||||
|
|
||||||
export function formatDuration(seconds: number): string {
|
export function formatDuration(seconds: number): string {
|
||||||
@@ -10,8 +10,6 @@ export function formatDuration(seconds: number): string {
|
|||||||
|
|
||||||
function getDateLocale() {
|
function getDateLocale() {
|
||||||
switch (i18n.language) {
|
switch (i18n.language) {
|
||||||
case 'es':
|
|
||||||
return es;
|
|
||||||
case 'ja':
|
case 'ja':
|
||||||
return ja;
|
return ja;
|
||||||
case 'zh-CN':
|
case 'zh-CN':
|
||||||
@@ -25,28 +23,27 @@ function getDateLocale() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backend timestamps are naive UTC — append `Z` so JS doesn't parse a
|
export function formatDate(date: string | Date): string {
|
||||||
// timezone-less date-time string as local time.
|
let dateObj: Date;
|
||||||
function parseServerDate(date: string | Date): Date {
|
if (typeof date === 'string') {
|
||||||
if (typeof date !== 'string') {
|
|
||||||
return date;
|
|
||||||
}
|
|
||||||
const dateStr = date.trim();
|
const dateStr = date.trim();
|
||||||
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
|
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
|
||||||
return new Date(`${dateStr}Z`);
|
dateObj = new Date(`${dateStr}Z`);
|
||||||
|
} else {
|
||||||
|
dateObj = new Date(dateStr);
|
||||||
}
|
}
|
||||||
return new Date(dateStr);
|
} else {
|
||||||
|
dateObj = date;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function formatDate(date: string | Date): string {
|
return formatDistance(dateObj, new Date(), {
|
||||||
return formatDistance(parseServerDate(date), new Date(), {
|
|
||||||
addSuffix: true,
|
addSuffix: true,
|
||||||
locale: getDateLocale(),
|
locale: getDateLocale(),
|
||||||
}).replace(/^about /i, '');
|
}).replace(/^about /i, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
export function formatAbsoluteDate(date: string | Date): string {
|
export function formatAbsoluteDate(date: string | Date): string {
|
||||||
const dateObj = parseServerDate(date);
|
const dateObj = typeof date === 'string' ? new Date(date) : date;
|
||||||
return dateObj.toLocaleString(i18n.language, {
|
return dateObj.toLocaleString(i18n.language, {
|
||||||
month: 'short',
|
month: 'short',
|
||||||
day: 'numeric',
|
day: 'numeric',
|
||||||
|
|||||||
@@ -38,13 +38,6 @@ logging.basicConfig(
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# An empty HSA_OVERRIDE_GFX_VERSION poisons the ROCm HSA runtime. It is
|
|
||||||
# treated as "force-empty" and no GPU is detected, even natively supported
|
|
||||||
# ones (e.g. gfx1201 / RX 9070 on ROCm 7.2). docker-compose can't
|
|
||||||
# conditionally omit an env var, so we clean it up here before torch loads.
|
|
||||||
if not os.environ.get("HSA_OVERRIDE_GFX_VERSION"):
|
|
||||||
os.environ.pop("HSA_OVERRIDE_GFX_VERSION", None)
|
|
||||||
|
|
||||||
# AMD GPU environment variables must be set before torch import
|
# AMD GPU environment variables must be set before torch import
|
||||||
# Only set HSA_OVERRIDE_GFX_VERSION for older GPUs that need it.
|
# Only set HSA_OVERRIDE_GFX_VERSION for older GPUs that need it.
|
||||||
# RDNA 3+ (gfx1100+) and RDNA 4 (gfx1200+) are natively supported by ROCm
|
# RDNA 3+ (gfx1100+) and RDNA 4 (gfx1200+) are natively supported by ROCm
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ class ModelConfig:
|
|||||||
model_size: str = "default"
|
model_size: str = "default"
|
||||||
size_mb: int = 0
|
size_mb: int = 0
|
||||||
needs_trim: bool = False
|
needs_trim: bool = False
|
||||||
retries_runaway: bool = False
|
|
||||||
supports_instruct: bool = False
|
supports_instruct: bool = False
|
||||||
languages: list[str] = field(default_factory=lambda: ["en"])
|
languages: list[str] = field(default_factory=lambda: ["en"])
|
||||||
|
|
||||||
@@ -233,10 +232,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
|||||||
repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||||
repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||||
|
|
||||||
# mlx-audio can continue after an EOS miss with silence followed by
|
|
||||||
# codec noise. Retry only the affected text as smaller chunks.
|
|
||||||
retries_runaway = backend_type == "mlx"
|
|
||||||
|
|
||||||
return [
|
return [
|
||||||
ModelConfig(
|
ModelConfig(
|
||||||
model_name="qwen-tts-1.7B",
|
model_name="qwen-tts-1.7B",
|
||||||
@@ -245,7 +240,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
|||||||
hf_repo_id=repo_1_7b,
|
hf_repo_id=repo_1_7b,
|
||||||
model_size="1.7B",
|
model_size="1.7B",
|
||||||
size_mb=3500,
|
size_mb=3500,
|
||||||
retries_runaway=retries_runaway,
|
|
||||||
supports_instruct=False, # Base model drops instruct silently
|
supports_instruct=False, # Base model drops instruct silently
|
||||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||||
),
|
),
|
||||||
@@ -256,7 +250,6 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
|||||||
hf_repo_id=repo_0_6b,
|
hf_repo_id=repo_0_6b,
|
||||||
model_size="0.6B",
|
model_size="0.6B",
|
||||||
size_mb=1200,
|
size_mb=1200,
|
||||||
retries_runaway=retries_runaway,
|
|
||||||
supports_instruct=False,
|
supports_instruct=False,
|
||||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||||
),
|
),
|
||||||
@@ -511,14 +504,6 @@ def engine_needs_trim(engine: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def engine_retries_runaway(engine: str) -> bool:
|
|
||||||
"""Whether unstable output should be retried in smaller chunks."""
|
|
||||||
for cfg in get_tts_model_configs():
|
|
||||||
if cfg.engine == engine:
|
|
||||||
return cfg.retries_runaway
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def engine_has_model_sizes(engine: str) -> bool:
|
def engine_has_model_sizes(engine: str) -> bool:
|
||||||
"""Whether this engine supports multiple model sizes (only Qwen currently)."""
|
"""Whether this engine supports multiple model sizes (only Qwen currently)."""
|
||||||
configs = [c for c in get_tts_model_configs() if c.engine == engine]
|
configs = [c for c in get_tts_model_configs() if c.engine == engine]
|
||||||
|
|||||||
@@ -248,12 +248,8 @@ class HumeTadaBackend:
|
|||||||
audio = audio.T # (samples, channels) -> (channels, samples)
|
audio = audio.T # (samples, channels) -> (channels, samples)
|
||||||
audio = audio.to(device)
|
audio = audio.to(device)
|
||||||
|
|
||||||
# Encode with forced alignment.
|
# Encode with forced alignment
|
||||||
# Must run under inference_mode: encoder params still require
|
|
||||||
# grad by default, and an autograd graph across the DAC/Snake
|
|
||||||
# stack can balloon VRAM far past the model footprint (#890).
|
|
||||||
text_arg = [reference_text] if reference_text else None
|
text_arg = [reference_text] if reference_text else None
|
||||||
with torch.inference_mode():
|
|
||||||
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
|
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
|
||||||
|
|
||||||
# Serialize EncoderOutput to a dict of CPU tensors for caching
|
# Serialize EncoderOutput to a dict of CPU tensors for caching
|
||||||
|
|||||||
@@ -96,16 +96,11 @@ KOKORO_VOICES = [
|
|||||||
("pf_dora", "Dora", "female", "pt"),
|
("pf_dora", "Dora", "female", "pt"),
|
||||||
("pm_alex", "Alex", "male", "pt"),
|
("pm_alex", "Alex", "male", "pt"),
|
||||||
("pm_santa", "Santa", "male", "pt"),
|
("pm_santa", "Santa", "male", "pt"),
|
||||||
# Chinese female
|
# Chinese
|
||||||
("zf_xiaobei", "Xiaobei", "female", "zh"),
|
("zf_xiaobei", "Xiaobei", "female", "zh"),
|
||||||
("zf_xiaoni", "Xiaoni", "female", "zh"),
|
("zf_xiaoni", "Xiaoni", "female", "zh"),
|
||||||
("zf_xiaoxiao", "Xiaoxiao", "female", "zh"),
|
("zf_xiaoxiao", "Xiaoxiao", "female", "zh"),
|
||||||
("zf_xiaoyi", "Xiaoyi", "female", "zh"),
|
("zf_xiaoyi", "Xiaoyi", "female", "zh"),
|
||||||
# Chinese male
|
|
||||||
("zm_yunjian", "Yunjian", "male", "zh"),
|
|
||||||
("zm_yunxi", "Yunxi", "male", "zh"),
|
|
||||||
("zm_yunxia", "Yunxia", "male", "zh"),
|
|
||||||
("zm_yunyang", "Yunyang", "male", "zh"),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
# Map our ISO language codes to Kokoro lang_code characters
|
# Map our ISO language codes to Kokoro lang_code characters
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from .base import (
|
|||||||
manual_seed,
|
manual_seed,
|
||||||
model_load_progress,
|
model_load_progress,
|
||||||
)
|
)
|
||||||
|
from ..utils.hf_offline_patch import force_offline_if_cached
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -102,11 +103,7 @@ class PyTorchQwenLLMBackend:
|
|||||||
|
|
||||||
with model_load_progress(progress_model_name, is_cached):
|
with model_load_progress(progress_model_name, is_cached):
|
||||||
logger.info("Loading Qwen3 %s on %s...", model_size, self.device)
|
logger.info("Loading Qwen3 %s on %s...", model_size, self.device)
|
||||||
# Loads run with the process's default HF_HUB_OFFLINE state.
|
with force_offline_if_cached(is_cached, progress_model_name):
|
||||||
# Forcing offline for cached models flips process-global state
|
|
||||||
# and silently switches every concurrent download/load on other
|
|
||||||
# threads to offline mode (issue #841) — the same regression
|
|
||||||
# removed app-wide in #524/#530.
|
|
||||||
self.tokenizer = AutoTokenizer.from_pretrained(repo)
|
self.tokenizer = AutoTokenizer.from_pretrained(repo)
|
||||||
dtype = torch.float16 if self.device in ("cuda", "mps") else torch.float32
|
dtype = torch.float16 if self.device in ("cuda", "mps") else torch.float32
|
||||||
self.model = AutoModelForCausalLM.from_pretrained(
|
self.model = AutoModelForCausalLM.from_pretrained(
|
||||||
@@ -226,7 +223,7 @@ class MLXQwenLLMBackend:
|
|||||||
|
|
||||||
with model_load_progress(progress_model_name, is_cached):
|
with model_load_progress(progress_model_name, is_cached):
|
||||||
logger.info("Loading Qwen3 %s via MLX...", model_size)
|
logger.info("Loading Qwen3 %s via MLX...", model_size)
|
||||||
# See the PyTorch loader comment — no offline forcing (issue #841).
|
with force_offline_if_cached(is_cached, progress_model_name):
|
||||||
loaded = mlx_load(repo)
|
loaded = mlx_load(repo)
|
||||||
|
|
||||||
# mlx_lm.load returns (model, tokenizer) by default and
|
# mlx_lm.load returns (model, tokenizer) by default and
|
||||||
|
|||||||
@@ -330,9 +330,6 @@ def build_server(cuda=False, rocm=False):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
if sys.version_info >= (3, 13):
|
|
||||||
args.extend(["--hidden-import", "audioop"])
|
|
||||||
|
|
||||||
# Add CUDA/ROCm-specific hidden imports
|
# Add CUDA/ROCm-specific hidden imports
|
||||||
if cuda or rocm:
|
if cuda or rocm:
|
||||||
variant = "ROCm" if rocm else "CUDA"
|
variant = "ROCm" if rocm else "CUDA"
|
||||||
|
|||||||
@@ -80,11 +80,6 @@ def resolve_storage_path(path: str | Path | None) -> Path | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
stored_path = Path(path)
|
stored_path = Path(path)
|
||||||
# Empty paths (e.g. failed generations) must not resolve to the data
|
|
||||||
# dir itself, which exists and would defeat the callers' 404 guards.
|
|
||||||
# Path("") is truthy, so check parts rather than the raw value.
|
|
||||||
if not stored_path.parts:
|
|
||||||
return None
|
|
||||||
if stored_path.is_absolute():
|
if stored_path.is_absolute():
|
||||||
rebased_path = _path_relative_to_any_data_dir(stored_path)
|
rebased_path = _path_relative_to_any_data_dir(stored_path)
|
||||||
if rebased_path is not None:
|
if rebased_path is not None:
|
||||||
@@ -143,17 +138,3 @@ def get_models_dir() -> Path:
|
|||||||
path = _data_dir / "models"
|
path = _data_dir / "models"
|
||||||
path.mkdir(parents=True, exist_ok=True)
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
# Voicebox Cloud (backup & sync). Two hosts: the web app owns auth + device
|
|
||||||
# pairing (voicebox.sh), the API owns sync + account endpoints
|
|
||||||
# (api.voicebox.sh). Override both for local development, e.g.
|
|
||||||
# VOICEBOX_CLOUD_URL=http://localhost:17592 VOICEBOX_CLOUD_API_URL=http://localhost:17593
|
|
||||||
def get_cloud_web_url() -> str:
|
|
||||||
"""Base URL of the Voicebox Cloud web app (auth + /connect + exchange)."""
|
|
||||||
return os.environ.get("VOICEBOX_CLOUD_URL", "https://voicebox.sh").rstrip("/")
|
|
||||||
|
|
||||||
|
|
||||||
def get_cloud_api_url() -> str:
|
|
||||||
"""Base URL of the Voicebox Cloud API (bearer-authenticated sync/account)."""
|
|
||||||
return os.environ.get("VOICEBOX_CLOUD_API_URL", "https://api.voicebox.sh").rstrip("/")
|
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from .models import (
|
|||||||
Capture,
|
Capture,
|
||||||
CaptureSettings,
|
CaptureSettings,
|
||||||
ChannelDeviceMapping,
|
ChannelDeviceMapping,
|
||||||
CloudSettings,
|
|
||||||
EffectPreset,
|
EffectPreset,
|
||||||
Generation,
|
Generation,
|
||||||
GenerationSettings,
|
GenerationSettings,
|
||||||
@@ -33,7 +32,6 @@ __all__ = [
|
|||||||
"Capture",
|
"Capture",
|
||||||
"CaptureSettings",
|
"CaptureSettings",
|
||||||
"ChannelDeviceMapping",
|
"ChannelDeviceMapping",
|
||||||
"CloudSettings",
|
|
||||||
"EffectPreset",
|
"EffectPreset",
|
||||||
"Generation",
|
"Generation",
|
||||||
"GenerationSettings",
|
"GenerationSettings",
|
||||||
|
|||||||
@@ -234,28 +234,6 @@ class GenerationSettings(Base):
|
|||||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||||
|
|
||||||
|
|
||||||
class CloudSettings(Base):
|
|
||||||
"""Singleton row holding the link to a Voicebox Cloud account.
|
|
||||||
|
|
||||||
Populated by the "Log in with browser" pairing flow (see services/cloud.py):
|
|
||||||
the browser hands back a one-time code, which the backend exchanges for an
|
|
||||||
``api_key`` it stores here. The key is a bearer credential for
|
|
||||||
api.voicebox.sh — auth only, never an encryption key (E2E key material lives
|
|
||||||
elsewhere). Stored in the local app database alongside the user's other data;
|
|
||||||
moving it to the OS keychain is a future hardening step. The ``id`` is
|
|
||||||
always 1; a null ``api_key`` means "not connected".
|
|
||||||
"""
|
|
||||||
|
|
||||||
__tablename__ = "cloud_settings"
|
|
||||||
|
|
||||||
id = Column(Integer, primary_key=True, default=1)
|
|
||||||
api_key = Column(String, nullable=True)
|
|
||||||
device_name = Column(String, nullable=True)
|
|
||||||
account_user_id = Column(String, nullable=True)
|
|
||||||
connected_at = Column(DateTime, nullable=True)
|
|
||||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
||||||
|
|
||||||
|
|
||||||
class MCPClientBinding(Base):
|
class MCPClientBinding(Base):
|
||||||
"""Per-MCP-client settings (voice profile, engine, personality default).
|
"""Per-MCP-client settings (voice profile, engine, personality default).
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import base64 as b64
|
|||||||
import logging
|
import logging
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal
|
from typing import Any
|
||||||
|
|
||||||
from fastmcp import FastMCP
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
@@ -49,7 +49,6 @@ def register_tools(mcp: FastMCP) -> None:
|
|||||||
engine: str | None = None,
|
engine: str | None = None,
|
||||||
personality: bool | None = None,
|
personality: bool | None = None,
|
||||||
language: str | None = None,
|
language: str | None = None,
|
||||||
model_size: Literal["1.7B", "0.6B", "1B", "3B"] | None = None,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Speak ``text`` in a voice profile.
|
"""Speak ``text`` in a voice profile.
|
||||||
|
|
||||||
@@ -62,12 +61,6 @@ def register_tools(mcp: FastMCP) -> None:
|
|||||||
LLM before TTS. When omitted, the per-client binding's
|
LLM before TTS. When omitted, the per-client binding's
|
||||||
``default_personality`` flag decides; when that is unset, the
|
``default_personality`` flag decides; when that is unset, the
|
||||||
default is plain TTS.
|
default is plain TTS.
|
||||||
|
|
||||||
``model_size`` selects a model variant for engines that ship more
|
|
||||||
than one — ``qwen`` and ``qwen_custom_voice`` accept "1.7B" (default)
|
|
||||||
or "0.6B"; ``tada`` accepts "1B" or "3B". Other engines ignore it.
|
|
||||||
Omit to use the engine default. Requesting a smaller variant (e.g.
|
|
||||||
"0.6B") is faster and avoids reloading a heavier model between calls.
|
|
||||||
"""
|
"""
|
||||||
from ..database.models import MCPClientBinding
|
from ..database.models import MCPClientBinding
|
||||||
|
|
||||||
@@ -106,7 +99,6 @@ def register_tools(mcp: FastMCP) -> None:
|
|||||||
engine=resolved_engine,
|
engine=resolved_engine,
|
||||||
language=language,
|
language=language,
|
||||||
personality=use_persona,
|
personality=use_persona,
|
||||||
model_size=model_size,
|
|
||||||
db=db,
|
db=db,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
@@ -236,23 +228,18 @@ async def _speak(
|
|||||||
engine: str | None,
|
engine: str | None,
|
||||||
language: str | None,
|
language: str | None,
|
||||||
personality: bool,
|
personality: bool,
|
||||||
model_size: str | None = None,
|
|
||||||
db,
|
db,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Delegate to POST /generate — the route handles personality-rewrite
|
"""Delegate to POST /generate — the route handles personality-rewrite
|
||||||
internally when ``personality=true`` and the profile has a prompt."""
|
internally when ``personality=true`` and the profile has a prompt."""
|
||||||
from ..routes.generations import generate_speech
|
from ..routes.generations import generate_speech
|
||||||
|
|
||||||
# model_size=None is intentional: generate_speech normalizes it to the
|
|
||||||
# engine default (see routes/generations.py), so an omitted size behaves
|
|
||||||
# exactly like the REST /generate endpoint with no model_size in the body.
|
|
||||||
req = models.GenerationRequest(
|
req = models.GenerationRequest(
|
||||||
profile_id=profile_id,
|
profile_id=profile_id,
|
||||||
text=text,
|
text=text,
|
||||||
language=language or "en",
|
language=language or "en",
|
||||||
engine=engine,
|
engine=engine,
|
||||||
personality=personality,
|
personality=personality,
|
||||||
model_size=model_size,
|
|
||||||
)
|
)
|
||||||
generation = await generate_speech(req, db)
|
generation = await generate_speech(req, db)
|
||||||
return _speak_response(generation, profile_name, source="mcp")
|
return _speak_response(generation, profile_name, source="mcp")
|
||||||
|
|||||||
@@ -794,24 +794,3 @@ class AvailableEffectsResponse(BaseModel):
|
|||||||
"""Response listing all available effect types."""
|
"""Response listing all available effect types."""
|
||||||
|
|
||||||
effects: List[AvailableEffect]
|
effects: List[AvailableEffect]
|
||||||
|
|
||||||
|
|
||||||
# ─── Cloud (backup & sync) ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
class CloudLoginStartResponse(BaseModel):
|
|
||||||
"""Returned when the desktop kicks off browser login. The backend has
|
|
||||||
already opened the browser; the URL is included for fallback/debugging."""
|
|
||||||
|
|
||||||
authorize_url: str
|
|
||||||
|
|
||||||
|
|
||||||
class CloudStatusResponse(BaseModel):
|
|
||||||
"""Current link between this device and a Voicebox Cloud account."""
|
|
||||||
|
|
||||||
connected: bool
|
|
||||||
device_name: Optional[str] = None
|
|
||||||
account_user_id: Optional[str] = None
|
|
||||||
key_prefix: Optional[str] = None
|
|
||||||
connected_at: Optional[datetime] = None
|
|
||||||
dashboard_url: str
|
|
||||||
|
|||||||
@@ -16,8 +16,7 @@ miniaudio>=1.59
|
|||||||
# mlx_audio.stt.load) works fine on transformers 4.57.x in practice.
|
# mlx_audio.stt.load) works fine on transformers 4.57.x in practice.
|
||||||
#
|
#
|
||||||
# Install it via `pip install --no-deps mlx-audio==0.4.1` after this file
|
# Install it via `pip install --no-deps mlx-audio==0.4.1` after this file
|
||||||
# (see .github/workflows/release.yml and the setup-python recipe in the
|
# (see .github/workflows/release.yml). Most other mlx-audio runtime deps
|
||||||
# justfile). Most other mlx-audio runtime deps
|
|
||||||
# (huggingface_hub, librosa, mlx-lm, numba, numpy, protobuf, pyloudnorm,
|
# (huggingface_hub, librosa, mlx-lm, numba, numpy, protobuf, pyloudnorm,
|
||||||
# sounddevice, tqdm) are already in requirements.txt or pulled in by
|
# sounddevice, tqdm) are already in requirements.txt or pulled in by
|
||||||
# other engines.
|
# other engines.
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_
|
|||||||
unidic-lite>=1.0.8
|
unidic-lite>=1.0.8
|
||||||
|
|
||||||
# Audio processing
|
# Audio processing
|
||||||
audioop-lts>=0.2.1; python_version >= "3.13"
|
|
||||||
librosa>=0.10.0
|
librosa>=0.10.0
|
||||||
soundfile>=0.12.0
|
soundfile>=0.12.0
|
||||||
numpy>=1.24.0,<2.0
|
numpy>=1.24.0,<2.0
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ def register_routers(app: FastAPI) -> None:
|
|||||||
from .speak import router as speak_router
|
from .speak import router as speak_router
|
||||||
from .mcp_bindings import router as mcp_bindings_router
|
from .mcp_bindings import router as mcp_bindings_router
|
||||||
from .events import router as events_router
|
from .events import router as events_router
|
||||||
from .cloud import router as cloud_router
|
|
||||||
|
|
||||||
app.include_router(health_router)
|
app.include_router(health_router)
|
||||||
app.include_router(profiles_router)
|
app.include_router(profiles_router)
|
||||||
@@ -45,4 +44,3 @@ def register_routers(app: FastAPI) -> None:
|
|||||||
app.include_router(speak_router)
|
app.include_router(speak_router)
|
||||||
app.include_router(mcp_bindings_router)
|
app.include_router(mcp_bindings_router)
|
||||||
app.include_router(events_router)
|
app.include_router(events_router)
|
||||||
app.include_router(cloud_router)
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ async def get_version_audio(version_id: str, db: Session = Depends(get_db)):
|
|||||||
raise HTTPException(status_code=404, detail="Version not found")
|
raise HTTPException(status_code=404, detail="Version not found")
|
||||||
|
|
||||||
audio_path = config.resolve_storage_path(version.audio_path)
|
audio_path = config.resolve_storage_path(version.audio_path)
|
||||||
if audio_path is None or not audio_path.is_file():
|
if audio_path is None or not audio_path.exists():
|
||||||
raise HTTPException(status_code=404, detail="Audio file not found")
|
raise HTTPException(status_code=404, detail="Audio file not found")
|
||||||
|
|
||||||
return FileResponse(
|
return FileResponse(
|
||||||
@@ -52,13 +52,8 @@ async def get_audio(generation_id: str, db: Session = Depends(get_db)):
|
|||||||
raise HTTPException(status_code=404, detail="Generation not found")
|
raise HTTPException(status_code=404, detail="Generation not found")
|
||||||
|
|
||||||
audio_path = config.resolve_storage_path(generation.audio_path)
|
audio_path = config.resolve_storage_path(generation.audio_path)
|
||||||
if audio_path is None or not audio_path.is_file():
|
if audio_path is None or not audio_path.exists():
|
||||||
detail = (
|
raise HTTPException(status_code=404, detail="Audio file not found")
|
||||||
"Generation failed; no audio available"
|
|
||||||
if generation.status == "failed"
|
|
||||||
else "Audio file not found"
|
|
||||||
)
|
|
||||||
raise HTTPException(status_code=404, detail=detail)
|
|
||||||
|
|
||||||
return FileResponse(
|
return FileResponse(
|
||||||
audio_path,
|
audio_path,
|
||||||
@@ -77,7 +72,7 @@ async def get_sample_audio(sample_id: str, db: Session = Depends(get_db)):
|
|||||||
raise HTTPException(status_code=404, detail="Sample not found")
|
raise HTTPException(status_code=404, detail="Sample not found")
|
||||||
|
|
||||||
audio_path = config.resolve_storage_path(sample.audio_path)
|
audio_path = config.resolve_storage_path(sample.audio_path)
|
||||||
if audio_path is None or not audio_path.is_file():
|
if audio_path is None or not audio_path.exists():
|
||||||
raise HTTPException(status_code=404, detail="Audio file not found")
|
raise HTTPException(status_code=404, detail="Audio file not found")
|
||||||
|
|
||||||
return FileResponse(
|
return FileResponse(
|
||||||
|
|||||||
@@ -1,75 +0,0 @@
|
|||||||
"""Voicebox Cloud device login routes.
|
|
||||||
|
|
||||||
The browser-based pairing flow:
|
|
||||||
1. POST /cloud/login/start — opens the browser to the cloud authorize page.
|
|
||||||
2. GET /cloud/callback — the browser lands here with a one-time code;
|
|
||||||
the backend exchanges it for an API key.
|
|
||||||
3. GET /cloud/status — the UI polls this to learn when it connected.
|
|
||||||
4. POST /cloud/disconnect — forget the local credential.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import socket
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
|
||||||
from fastapi.responses import HTMLResponse
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from .. import models
|
|
||||||
from ..database import get_db
|
|
||||||
from ..services import cloud as cloud_service
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/cloud", tags=["cloud"])
|
|
||||||
|
|
||||||
|
|
||||||
def _callback_url(request: Request) -> str:
|
|
||||||
# Always loopback — the cloud only redirects codes to 127.0.0.1/localhost.
|
|
||||||
port = request.url.port or 17493
|
|
||||||
return f"http://127.0.0.1:{port}/cloud/callback"
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/login/start", response_model=models.CloudLoginStartResponse)
|
|
||||||
async def start_cloud_login(request: Request):
|
|
||||||
device_name = socket.gethostname() or "Desktop"
|
|
||||||
authorize_url = cloud_service.start_login(_callback_url(request), device_name)
|
|
||||||
return models.CloudLoginStartResponse(authorize_url=authorize_url)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/callback", response_class=HTMLResponse)
|
|
||||||
async def cloud_callback(
|
|
||||||
request: Request,
|
|
||||||
code: str = "",
|
|
||||||
state: str = "",
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
):
|
|
||||||
ok, message = await cloud_service.handle_callback(db, code=code, state=state)
|
|
||||||
heading = "You're connected" if ok else "Couldn't connect"
|
|
||||||
accent = "#16a34a" if ok else "#dc2626"
|
|
||||||
sub = (
|
|
||||||
"Voicebox is now linked to your account. You can close this tab and return to the app."
|
|
||||||
if ok
|
|
||||||
else message
|
|
||||||
)
|
|
||||||
html = f"""<!doctype html>
|
|
||||||
<html lang="en"><head><meta charset="utf-8" />
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
||||||
<title>Voicebox Cloud</title>
|
|
||||||
<style>
|
|
||||||
body {{ margin:0; min-height:100vh; display:flex; align-items:center; justify-content:center;
|
|
||||||
font-family: ui-sans-serif, system-ui, -apple-system, sans-serif; background:#0b0b0d; color:#e7e7ea; }}
|
|
||||||
.card {{ max-width:28rem; padding:2.5rem; text-align:center; }}
|
|
||||||
h1 {{ font-size:1.5rem; margin:0 0 .5rem; color:{accent}; }}
|
|
||||||
p {{ color:#a1a1aa; line-height:1.5; }}
|
|
||||||
</style></head>
|
|
||||||
<body><div class="card"><h1>{heading}</h1><p>{sub}</p></div></body></html>"""
|
|
||||||
return HTMLResponse(content=html, status_code=200 if ok else 400)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/status", response_model=models.CloudStatusResponse)
|
|
||||||
async def cloud_status(db: Session = Depends(get_db)):
|
|
||||||
return models.CloudStatusResponse(**cloud_service.get_status(db))
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/disconnect", response_model=models.CloudStatusResponse)
|
|
||||||
async def cloud_disconnect(db: Session = Depends(get_db)):
|
|
||||||
cloud_service.disconnect(db)
|
|
||||||
return models.CloudStatusResponse(**cloud_service.get_status(db))
|
|
||||||
@@ -26,10 +26,6 @@ async def download_cuda_backend():
|
|||||||
"""Download the CUDA backend binary."""
|
"""Download the CUDA backend binary."""
|
||||||
from ..services import cuda
|
from ..services import cuda
|
||||||
|
|
||||||
unsupported_reason = cuda.get_cuda_download_unsupported_reason()
|
|
||||||
if unsupported_reason:
|
|
||||||
raise HTTPException(status_code=409, detail=unsupported_reason)
|
|
||||||
|
|
||||||
if cuda.get_cuda_binary_path() is not None:
|
if cuda.get_cuda_binary_path() is not None:
|
||||||
raise HTTPException(status_code=409, detail="CUDA backend already downloaded")
|
raise HTTPException(status_code=409, detail="CUDA backend already downloaded")
|
||||||
|
|
||||||
|
|||||||
@@ -321,13 +321,7 @@ async def stream_speech(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
"""Generate speech and stream the WAV audio directly without saving to disk."""
|
"""Generate speech and stream the WAV audio directly without saving to disk."""
|
||||||
from ..backends import (
|
from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim
|
||||||
engine_needs_trim,
|
|
||||||
engine_retries_runaway,
|
|
||||||
ensure_model_cached_or_raise,
|
|
||||||
get_tts_backend_for_engine,
|
|
||||||
load_engine_model,
|
|
||||||
)
|
|
||||||
|
|
||||||
profile = await profiles.get_profile(data.profile_id, db)
|
profile = await profiles.get_profile(data.profile_id, db)
|
||||||
if not profile:
|
if not profile:
|
||||||
@@ -353,15 +347,10 @@ async def stream_speech(
|
|||||||
from ..utils.chunked_tts import generate_chunked
|
from ..utils.chunked_tts import generate_chunked
|
||||||
|
|
||||||
trim_fn = None
|
trim_fn = None
|
||||||
runaway_detector = None
|
|
||||||
if engine_needs_trim(engine):
|
if engine_needs_trim(engine):
|
||||||
from ..utils.audio import trim_tts_output
|
from ..utils.audio import trim_tts_output
|
||||||
|
|
||||||
trim_fn = trim_tts_output
|
trim_fn = trim_tts_output
|
||||||
if engine_retries_runaway(engine):
|
|
||||||
from ..utils.audio import has_tts_runaway
|
|
||||||
|
|
||||||
runaway_detector = has_tts_runaway
|
|
||||||
|
|
||||||
audio, sample_rate = await generate_chunked(
|
audio, sample_rate = await generate_chunked(
|
||||||
tts_model,
|
tts_model,
|
||||||
@@ -373,7 +362,6 @@ async def stream_speech(
|
|||||||
max_chunk_chars=data.max_chunk_chars,
|
max_chunk_chars=data.max_chunk_chars,
|
||||||
crossfade_ms=data.crossfade_ms,
|
crossfade_ms=data.crossfade_ms,
|
||||||
trim_fn=trim_fn,
|
trim_fn=trim_fn,
|
||||||
runaway_detector=runaway_detector,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
effects_chain_config = None
|
effects_chain_config = None
|
||||||
|
|||||||
@@ -151,9 +151,7 @@ async def export_generation(
|
|||||||
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
||||||
if not safe_text:
|
if not safe_text:
|
||||||
safe_text = "generation"
|
safe_text = "generation"
|
||||||
# Append a short id so exports of similarly-worded generations don't collide
|
filename = f"generation-{safe_text}.voicebox.zip"
|
||||||
# on the same filename (the first 30 chars are frequently identical).
|
|
||||||
filename = f"generation-{safe_text}-{generation_id[:8]}.voicebox.zip"
|
|
||||||
|
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
io.BytesIO(zip_bytes),
|
io.BytesIO(zip_bytes),
|
||||||
@@ -182,9 +180,7 @@ async def export_generation_audio(
|
|||||||
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
||||||
if not safe_text:
|
if not safe_text:
|
||||||
safe_text = "generation"
|
safe_text = "generation"
|
||||||
# Append a short id so exports of similarly-worded generations don't collide
|
filename = f"{safe_text}.wav"
|
||||||
# on the same filename (the first 30 chars are frequently identical).
|
|
||||||
filename = f"{safe_text}-{generation_id[:8]}.wav"
|
|
||||||
|
|
||||||
return FileResponse(
|
return FileResponse(
|
||||||
audio_path,
|
audio_path,
|
||||||
|
|||||||
@@ -231,10 +231,7 @@ async def get_model_status():
|
|||||||
backend_type = get_backend_type()
|
backend_type = get_backend_type()
|
||||||
task_manager = get_task_manager()
|
task_manager = get_task_manager()
|
||||||
|
|
||||||
# Pending only — an errored task stays in the active list for the
|
active_download_names = {task.model_name for task in task_manager.get_active_downloads()}
|
||||||
# error/retry UI, but reporting it as "downloading" here would mask
|
|
||||||
# the model's real cache state until the app restarts (issue #925).
|
|
||||||
active_download_names = {task.model_name for task in task_manager.get_pending_downloads()}
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from huggingface_hub import scan_cache_dir
|
from huggingface_hub import scan_cache_dir
|
||||||
|
|||||||
@@ -232,7 +232,7 @@ async def upload_profile_avatar(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
"""Upload or update avatar image for a profile."""
|
"""Upload or update avatar image for a profile."""
|
||||||
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename or "").suffix) as tmp:
|
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename).suffix) as tmp:
|
||||||
content = await file.read()
|
content = await file.read()
|
||||||
tmp.write(content)
|
tmp.write(content)
|
||||||
tmp_path = tmp.name
|
tmp_path = tmp.name
|
||||||
|
|||||||
@@ -15,10 +15,6 @@ router = APIRouter()
|
|||||||
|
|
||||||
UPLOAD_CHUNK_SIZE = 1024 * 1024 # 1MB
|
UPLOAD_CHUNK_SIZE = 1024 * 1024 # 1MB
|
||||||
|
|
||||||
# Same set profiles.py accepts for voice samples. librosa picks its decoder from the
|
|
||||||
# file extension, so the temp file has to keep the uploaded one.
|
|
||||||
ALLOWED_AUDIO_EXTS = {".wav", ".mp3", ".m4a", ".ogg", ".flac", ".aac", ".webm", ".opus"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/transcribe", response_model=models.TranscriptionResponse)
|
@router.post("/transcribe", response_model=models.TranscriptionResponse)
|
||||||
async def transcribe_audio(
|
async def transcribe_audio(
|
||||||
@@ -27,33 +23,18 @@ async def transcribe_audio(
|
|||||||
model: str | None = Form(None),
|
model: str | None = Form(None),
|
||||||
):
|
):
|
||||||
"""Transcribe audio file to text."""
|
"""Transcribe audio file to text."""
|
||||||
uploaded_ext = Path(file.filename or "").suffix.lower()
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||||
file_suffix = uploaded_ext if uploaded_ext in ALLOWED_AUDIO_EXTS else ".wav"
|
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile(suffix=file_suffix, delete=False) as tmp:
|
|
||||||
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
|
while chunk := await file.read(UPLOAD_CHUNK_SIZE):
|
||||||
tmp.write(chunk)
|
tmp.write(chunk)
|
||||||
tmp_path = tmp.name
|
tmp_path = tmp.name
|
||||||
|
|
||||||
stt_path = tmp_path
|
|
||||||
try:
|
try:
|
||||||
from ..utils.audio import load_audio, save_audio
|
from ..utils.audio import load_audio
|
||||||
from ..backends import WHISPER_HF_REPOS
|
from ..backends import WHISPER_HF_REPOS
|
||||||
|
|
||||||
audio, sr = await asyncio.to_thread(load_audio, tmp_path)
|
audio, sr = await asyncio.to_thread(load_audio, tmp_path)
|
||||||
duration = len(audio) / sr
|
duration = len(audio) / sr
|
||||||
|
|
||||||
# The STT backend (mlx_audio.stt -> miniaudio) only decodes
|
|
||||||
# WAV/FLAC/MP3/Vorbis, so browser recordings uploaded as WebM/Opus
|
|
||||||
# fail with "unsupported file format" (issue: web-mode dictation).
|
|
||||||
# librosa already decoded the file above (it falls back to
|
|
||||||
# audioread/ffmpeg for exotic containers), so re-encode that PCM to a
|
|
||||||
# temp WAV and hand *that* to Whisper. WAV inputs pass through
|
|
||||||
# unchanged.
|
|
||||||
if file_suffix != ".wav":
|
|
||||||
stt_path = f"{tmp_path}.stt.wav"
|
|
||||||
await asyncio.to_thread(save_audio, audio, stt_path, sr)
|
|
||||||
|
|
||||||
whisper_model = transcribe.get_whisper_model()
|
whisper_model = transcribe.get_whisper_model()
|
||||||
model_size = model if model else whisper_model.model_size
|
model_size = model if model else whisper_model.model_size
|
||||||
|
|
||||||
@@ -88,7 +69,7 @@ async def transcribe_audio(
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
text = await whisper_model.transcribe(stt_path, language, model_size)
|
text = await whisper_model.transcribe(tmp_path, language, model_size)
|
||||||
|
|
||||||
return models.TranscriptionResponse(
|
return models.TranscriptionResponse(
|
||||||
text=text,
|
text=text,
|
||||||
@@ -101,5 +82,3 @@ async def transcribe_audio(
|
|||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
finally:
|
finally:
|
||||||
Path(tmp_path).unlink(missing_ok=True)
|
Path(tmp_path).unlink(missing_ok=True)
|
||||||
if stt_path != tmp_path:
|
|
||||||
Path(stt_path).unlink(missing_ok=True)
|
|
||||||
|
|||||||
@@ -1,183 +0,0 @@
|
|||||||
"""
|
|
||||||
Voicebox Cloud device login — the "Log in with browser" flow.
|
|
||||||
|
|
||||||
The desktop opens the browser to ``{web}/connect``; the user authorizes while
|
|
||||||
signed in; the cloud redirects a single-use code back to this backend's loopback
|
|
||||||
callback. We exchange that code (server-to-server, over TLS) for a ``voicebox_…``
|
|
||||||
API key, verify the key against the API, and store it locally. The key never
|
|
||||||
travels through a browser URL, and an unfinished flow leaves nothing behind.
|
|
||||||
|
|
||||||
The ``state`` we mint and round-trip prevents login-CSRF: a callback whose state
|
|
||||||
we didn't issue (e.g. an attacker tricking the user into hitting the loopback
|
|
||||||
callback with their own code) is rejected.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import secrets
|
|
||||||
import time
|
|
||||||
import webbrowser
|
|
||||||
from urllib.parse import urlencode
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from sqlalchemy.exc import IntegrityError
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from .. import config
|
|
||||||
from ..database import CloudSettings as DBCloudSettings
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
SINGLETON_ID = 1
|
|
||||||
PENDING_TTL_SECONDS = 600 # the whole browser flow must finish within 10 min
|
|
||||||
|
|
||||||
# state -> expiry epoch. In-memory: a single backend process owns the flow, and a
|
|
||||||
# dropped pairing should simply be restarted.
|
|
||||||
_pending: dict[str, float] = {}
|
|
||||||
|
|
||||||
|
|
||||||
def _prune() -> None:
|
|
||||||
now = time.time()
|
|
||||||
for state, expiry in list(_pending.items()):
|
|
||||||
if expiry < now:
|
|
||||||
_pending.pop(state, None)
|
|
||||||
|
|
||||||
|
|
||||||
def _json_dict(response: httpx.Response) -> dict | None:
|
|
||||||
"""Parsed JSON body, or None when it isn't a JSON object."""
|
|
||||||
try:
|
|
||||||
payload = response.json()
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return payload if isinstance(payload, dict) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _consume_state(state: str) -> bool:
|
|
||||||
"""Validate and single-use-consume a pending state."""
|
|
||||||
_prune()
|
|
||||||
expiry = _pending.pop(state, None)
|
|
||||||
return expiry is not None and expiry >= time.time()
|
|
||||||
|
|
||||||
|
|
||||||
def start_login(callback_url: str, device_name: str) -> str:
|
|
||||||
"""Mint a state, build the authorize URL, and open the browser.
|
|
||||||
|
|
||||||
Returns the authorize URL (also opened here) so the caller can surface it as
|
|
||||||
a fallback if the browser didn't open.
|
|
||||||
"""
|
|
||||||
state = secrets.token_urlsafe(24)
|
|
||||||
_prune()
|
|
||||||
_pending[state] = time.time() + PENDING_TTL_SECONDS
|
|
||||||
|
|
||||||
params = urlencode({"redirect_uri": callback_url, "state": state, "name": device_name})
|
|
||||||
authorize_url = f"{config.get_cloud_web_url()}/connect?{params}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
webbrowser.open(authorize_url)
|
|
||||||
except Exception: # pragma: no cover - platform dependent
|
|
||||||
logger.exception("failed to open browser for cloud login")
|
|
||||||
|
|
||||||
return authorize_url
|
|
||||||
|
|
||||||
|
|
||||||
async def handle_callback(db: Session, code: str, state: str) -> tuple[bool, str]:
|
|
||||||
"""Exchange the code for an API key and store it. Returns (ok, message)."""
|
|
||||||
if not _consume_state(state):
|
|
||||||
return False, "This sign-in link is invalid or has expired. Start again from the app."
|
|
||||||
if not code:
|
|
||||||
return False, "Missing authorization code."
|
|
||||||
|
|
||||||
web = config.get_cloud_web_url()
|
|
||||||
api = config.get_cloud_api_url()
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
||||||
exchanged = await client.post(f"{web}/api/connect/exchange", json={"code": code})
|
|
||||||
if exchanged.status_code != 200:
|
|
||||||
logger.warning("cloud exchange rejected code: %s", exchanged.status_code)
|
|
||||||
return False, "Could not complete sign-in — the code was rejected."
|
|
||||||
payload = _json_dict(exchanged)
|
|
||||||
if payload is None:
|
|
||||||
logger.warning("cloud exchange returned a non-JSON payload")
|
|
||||||
return False, "Voicebox Cloud returned an unexpected response."
|
|
||||||
api_key = payload.get("key")
|
|
||||||
device_name = payload.get("label")
|
|
||||||
if not api_key:
|
|
||||||
return False, "Voicebox Cloud did not return a key."
|
|
||||||
|
|
||||||
# Confirm the freshly minted key actually authenticates the API.
|
|
||||||
me = await client.get(
|
|
||||||
f"{api}/v1/account/me",
|
|
||||||
headers={"Authorization": f"Bearer {api_key}"},
|
|
||||||
)
|
|
||||||
if me.status_code != 200:
|
|
||||||
logger.warning("minted key failed verification: %s", me.status_code)
|
|
||||||
return False, "Sign-in succeeded but the key could not be verified."
|
|
||||||
# The 200 above proves the key works; the user id is best-effort.
|
|
||||||
data = (_json_dict(me) or {}).get("data")
|
|
||||||
account_user_id = data.get("userId") if isinstance(data, dict) else None
|
|
||||||
except httpx.HTTPError:
|
|
||||||
logger.exception("network error during cloud exchange")
|
|
||||||
return False, "Could not reach Voicebox Cloud. Check your connection and try again."
|
|
||||||
|
|
||||||
_store_key(db, api_key=api_key, device_name=device_name, account_user_id=account_user_id)
|
|
||||||
logger.info("connected to Voicebox Cloud as device %r", device_name)
|
|
||||||
return True, "Connected"
|
|
||||||
|
|
||||||
|
|
||||||
def _get_or_create_row(db: Session) -> DBCloudSettings:
|
|
||||||
row = db.query(DBCloudSettings).filter(DBCloudSettings.id == SINGLETON_ID).first()
|
|
||||||
if row is None:
|
|
||||||
row = DBCloudSettings(id=SINGLETON_ID)
|
|
||||||
db.add(row)
|
|
||||||
try:
|
|
||||||
db.commit()
|
|
||||||
except IntegrityError:
|
|
||||||
# Another request created the singleton concurrently.
|
|
||||||
db.rollback()
|
|
||||||
row = db.query(DBCloudSettings).filter(DBCloudSettings.id == SINGLETON_ID).one()
|
|
||||||
else:
|
|
||||||
db.refresh(row)
|
|
||||||
return row
|
|
||||||
|
|
||||||
|
|
||||||
def _store_key(db: Session, *, api_key: str, device_name: str | None, account_user_id: str | None):
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
row = _get_or_create_row(db)
|
|
||||||
row.api_key = api_key
|
|
||||||
row.device_name = device_name
|
|
||||||
row.account_user_id = account_user_id
|
|
||||||
row.connected_at = datetime.utcnow()
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def get_status(db: Session) -> dict:
|
|
||||||
"""Local view of the cloud link — never returns the full key."""
|
|
||||||
row = _get_or_create_row(db)
|
|
||||||
connected = bool(row.api_key)
|
|
||||||
# Prefix only: "voicebox_" (9) + 8 chars, matching the cloud's key_prefix.
|
|
||||||
key_prefix = row.api_key[:17] if row.api_key else None
|
|
||||||
return {
|
|
||||||
"connected": connected,
|
|
||||||
"device_name": row.device_name if connected else None,
|
|
||||||
"account_user_id": row.account_user_id if connected else None,
|
|
||||||
"key_prefix": key_prefix,
|
|
||||||
"connected_at": row.connected_at if connected else None,
|
|
||||||
"dashboard_url": f"{config.get_cloud_web_url()}/account",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def disconnect(db: Session) -> None:
|
|
||||||
"""Forget the local credential. The key remains valid on the server until
|
|
||||||
revoked from the account dashboard — surface that in the UI."""
|
|
||||||
row = _get_or_create_row(db)
|
|
||||||
row.api_key = None
|
|
||||||
row.device_name = None
|
|
||||||
row.account_user_id = None
|
|
||||||
row.connected_at = None
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def get_api_key(db: Session) -> str | None:
|
|
||||||
"""The stored bearer key, for the (future) sync client. None if not linked."""
|
|
||||||
row = _get_or_create_row(db)
|
|
||||||
return row.api_key
|
|
||||||
@@ -21,9 +21,9 @@ import tarfile
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from .. import __version__
|
|
||||||
from ..config import get_data_dir
|
from ..config import get_data_dir
|
||||||
from ..utils.progress import get_progress_manager
|
from ..utils.progress import get_progress_manager
|
||||||
|
from .. import __version__
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -31,8 +31,6 @@ GITHUB_RELEASES_URL = "https://github.com/jamiepine/voicebox/releases/download"
|
|||||||
|
|
||||||
PROGRESS_KEY = "cuda-backend"
|
PROGRESS_KEY = "cuda-backend"
|
||||||
|
|
||||||
CUDA_DOWNLOAD_UNSUPPORTED_REASON = "Downloadable CUDA backend releases are currently only published for Windows."
|
|
||||||
|
|
||||||
# The current expected CUDA libs version. Bump this when we change the
|
# The current expected CUDA libs version. Bump this when we change the
|
||||||
# CUDA toolkit version or torch's CUDA dependency changes (e.g. cu126 -> cu128).
|
# CUDA toolkit version or torch's CUDA dependency changes (e.g. cu126 -> cu128).
|
||||||
CUDA_LIBS_VERSION = "cu128-v1"
|
CUDA_LIBS_VERSION = "cu128-v1"
|
||||||
@@ -65,25 +63,6 @@ def get_cuda_exe_name() -> str:
|
|||||||
return "voicebox-server-cuda"
|
return "voicebox-server-cuda"
|
||||||
|
|
||||||
|
|
||||||
def is_cuda_download_supported() -> bool:
|
|
||||||
"""Return whether this platform has a matching CUDA release asset."""
|
|
||||||
return sys.platform == "win32"
|
|
||||||
|
|
||||||
|
|
||||||
def get_cuda_download_unsupported_reason() -> str | None:
|
|
||||||
"""Explain why this platform cannot use the release-download flow."""
|
|
||||||
if is_cuda_download_supported():
|
|
||||||
return None
|
|
||||||
return CUDA_DOWNLOAD_UNSUPPORTED_REASON
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_cuda_download_supported() -> None:
|
|
||||||
"""Raise if downloading would fetch an asset built for another platform."""
|
|
||||||
reason = get_cuda_download_unsupported_reason()
|
|
||||||
if reason:
|
|
||||||
raise RuntimeError(reason)
|
|
||||||
|
|
||||||
|
|
||||||
def get_cuda_binary_path() -> Optional[Path]:
|
def get_cuda_binary_path() -> Optional[Path]:
|
||||||
"""Return path to the CUDA executable if it exists inside the onedir."""
|
"""Return path to the CUDA executable if it exists inside the onedir."""
|
||||||
p = get_cuda_dir() / get_cuda_exe_name()
|
p = get_cuda_dir() / get_cuda_exe_name()
|
||||||
@@ -124,15 +103,12 @@ def get_cuda_status() -> dict:
|
|||||||
cuda_path = get_cuda_binary_path()
|
cuda_path = get_cuda_binary_path()
|
||||||
progress = progress_manager.get_progress(PROGRESS_KEY)
|
progress = progress_manager.get_progress(PROGRESS_KEY)
|
||||||
cuda_libs_version = get_installed_cuda_libs_version()
|
cuda_libs_version = get_installed_cuda_libs_version()
|
||||||
unsupported_reason = get_cuda_download_unsupported_reason()
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"available": cuda_path is not None,
|
"available": cuda_path is not None,
|
||||||
"active": is_cuda_active(),
|
"active": is_cuda_active(),
|
||||||
"binary_path": str(cuda_path) if cuda_path else None,
|
"binary_path": str(cuda_path) if cuda_path else None,
|
||||||
"cuda_libs_version": cuda_libs_version,
|
"cuda_libs_version": cuda_libs_version,
|
||||||
"download_supported": unsupported_reason is None,
|
|
||||||
"unsupported_reason": unsupported_reason,
|
|
||||||
"downloading": progress is not None and progress.get("status") == "downloading",
|
"downloading": progress is not None and progress.get("status") == "downloading",
|
||||||
"download_progress": progress,
|
"download_progress": progress,
|
||||||
}
|
}
|
||||||
@@ -281,8 +257,6 @@ async def download_cuda_binary(version: Optional[str] = None):
|
|||||||
|
|
||||||
async def _download_cuda_binary_locked(version: Optional[str] = None):
|
async def _download_cuda_binary_locked(version: Optional[str] = None):
|
||||||
"""Inner implementation of download_cuda_binary, called under _download_lock."""
|
"""Inner implementation of download_cuda_binary, called under _download_lock."""
|
||||||
ensure_cuda_download_supported()
|
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
if version is None:
|
if version is None:
|
||||||
@@ -413,11 +387,6 @@ async def check_and_update_cuda_binary():
|
|||||||
if not cuda_path:
|
if not cuda_path:
|
||||||
return # No CUDA binary installed, nothing to update
|
return # No CUDA binary installed, nothing to update
|
||||||
|
|
||||||
unsupported_reason = get_cuda_download_unsupported_reason()
|
|
||||||
if unsupported_reason:
|
|
||||||
logger.info("Skipping CUDA backend auto-update: %s", unsupported_reason)
|
|
||||||
return
|
|
||||||
|
|
||||||
need_server = _needs_server_download()
|
need_server = _needs_server_download()
|
||||||
need_libs = _needs_cuda_libs_download()
|
need_libs = _needs_cuda_libs_download()
|
||||||
|
|
||||||
|
|||||||
@@ -48,14 +48,9 @@ async def run_generation(
|
|||||||
This is the single entry point for all background generation work.
|
This is the single entry point for all background generation work.
|
||||||
It is designed to be enqueued via ``services.task_queue.enqueue_generation``.
|
It is designed to be enqueued via ``services.task_queue.enqueue_generation``.
|
||||||
"""
|
"""
|
||||||
from ..backends import (
|
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
|
||||||
engine_needs_trim,
|
|
||||||
engine_retries_runaway,
|
|
||||||
get_tts_backend_for_engine,
|
|
||||||
load_engine_model,
|
|
||||||
)
|
|
||||||
from ..utils.chunked_tts import generate_chunked
|
from ..utils.chunked_tts import generate_chunked
|
||||||
from ..utils.audio import has_tts_runaway, normalize_audio, save_audio, trim_tts_output
|
from ..utils.audio import normalize_audio, save_audio, trim_tts_output
|
||||||
|
|
||||||
task_manager = get_task_manager()
|
task_manager = get_task_manager()
|
||||||
bg_db = next(get_db())
|
bg_db = next(get_db())
|
||||||
@@ -77,14 +72,12 @@ async def run_generation(
|
|||||||
|
|
||||||
await history.update_generation_status(generation_id, "generating", bg_db)
|
await history.update_generation_status(generation_id, "generating", bg_db)
|
||||||
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
||||||
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
|
|
||||||
|
|
||||||
gen_kwargs: dict = dict(
|
gen_kwargs: dict = dict(
|
||||||
language=language,
|
language=language,
|
||||||
seed=seed if mode != "regenerate" else None,
|
seed=seed if mode != "regenerate" else None,
|
||||||
instruct=instruct,
|
instruct=instruct,
|
||||||
trim_fn=trim_fn,
|
trim_fn=trim_fn,
|
||||||
runaway_detector=runaway_detector,
|
|
||||||
)
|
)
|
||||||
if max_chunk_chars is not None:
|
if max_chunk_chars is not None:
|
||||||
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
||||||
@@ -274,14 +267,9 @@ async def generate_audio_sync(
|
|||||||
normalize, then encodes in-memory via :func:`tts.audio_to_wav_bytes`
|
normalize, then encodes in-memory via :func:`tts.audio_to_wav_bytes`
|
||||||
(same helper ``/generate/stream`` uses).
|
(same helper ``/generate/stream`` uses).
|
||||||
"""
|
"""
|
||||||
from ..backends import (
|
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
|
||||||
engine_needs_trim,
|
|
||||||
engine_retries_runaway,
|
|
||||||
get_tts_backend_for_engine,
|
|
||||||
load_engine_model,
|
|
||||||
)
|
|
||||||
from ..utils.chunked_tts import generate_chunked
|
from ..utils.chunked_tts import generate_chunked
|
||||||
from ..utils.audio import has_tts_runaway, normalize_audio, trim_tts_output
|
from ..utils.audio import normalize_audio, trim_tts_output
|
||||||
from . import tts
|
from . import tts
|
||||||
|
|
||||||
bg_db = next(get_db())
|
bg_db = next(get_db())
|
||||||
@@ -299,14 +287,12 @@ async def generate_audio_sync(
|
|||||||
bg_db.close()
|
bg_db.close()
|
||||||
|
|
||||||
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
||||||
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
|
|
||||||
|
|
||||||
gen_kwargs: dict = dict(
|
gen_kwargs: dict = dict(
|
||||||
language=language,
|
language=language,
|
||||||
seed=seed,
|
seed=seed,
|
||||||
instruct=instruct,
|
instruct=instruct,
|
||||||
trim_fn=trim_fn,
|
trim_fn=trim_fn,
|
||||||
runaway_detector=runaway_detector,
|
|
||||||
)
|
)
|
||||||
if max_chunk_chars is not None:
|
if max_chunk_chars is not None:
|
||||||
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
||||||
|
|||||||
@@ -125,24 +125,12 @@ async def list_stories(
|
|||||||
"""
|
"""
|
||||||
stories = db.query(DBStory).order_by(DBStory.updated_at.desc()).all()
|
stories = db.query(DBStory).order_by(DBStory.updated_at.desc()).all()
|
||||||
|
|
||||||
if not stories:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Batch-fetch all story item counts in one query to avoid an N+1 pattern
|
|
||||||
# (previously there was one COUNT query per story in the loop below).
|
|
||||||
story_ids = [s.id for s in stories]
|
|
||||||
count_rows = (
|
|
||||||
db.query(DBStoryItem.story_id, func.count(DBStoryItem.id).label("cnt"))
|
|
||||||
.filter(DBStoryItem.story_id.in_(story_ids))
|
|
||||||
.group_by(DBStoryItem.story_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
item_counts = {row.story_id: row.cnt for row in count_rows}
|
|
||||||
|
|
||||||
result = []
|
result = []
|
||||||
for story in stories:
|
for story in stories:
|
||||||
|
item_count = db.query(func.count(DBStoryItem.id)).filter(DBStoryItem.story_id == story.id).scalar()
|
||||||
|
|
||||||
response = StoryResponse.model_validate(story)
|
response = StoryResponse.model_validate(story)
|
||||||
response.item_count = item_counts.get(story.id, 0)
|
response.item_count = item_count
|
||||||
result.append(response)
|
result.append(response)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -1,164 +0,0 @@
|
|||||||
"""
|
|
||||||
Regression tests for GET /audio/{generation_id} on failed generations.
|
|
||||||
|
|
||||||
A failed generation stores an empty ``audio_path``. Previously,
|
|
||||||
``config.resolve_storage_path("")`` resolved to the data directory itself,
|
|
||||||
which exists, so the route's 404 guard passed and ``FileResponse`` raised
|
|
||||||
``RuntimeError: File at path .../data is not a file`` — a 500 instead of
|
|
||||||
a clean 404.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python -m pytest backend/tests/test_audio_failed_generation.py -v
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from fastapi import FastAPI
|
|
||||||
from sqlalchemy import create_engine
|
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
from starlette.testclient import TestClient
|
|
||||||
|
|
||||||
# Repo root on sys.path so ``backend`` imports as a package (the audio
|
|
||||||
# routes use package-relative imports).
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
||||||
|
|
||||||
from backend import config
|
|
||||||
from backend.database import (
|
|
||||||
Base,
|
|
||||||
Generation,
|
|
||||||
GenerationVersion,
|
|
||||||
ProfileSample,
|
|
||||||
VoiceProfile,
|
|
||||||
get_db,
|
|
||||||
)
|
|
||||||
from backend.routes.audio import router as audio_router
|
|
||||||
|
|
||||||
|
|
||||||
def test_resolve_storage_path_empty_returns_none():
|
|
||||||
"""An empty stored path must not resolve to the data dir itself."""
|
|
||||||
assert config.resolve_storage_path("") is None
|
|
||||||
assert config.resolve_storage_path(None) is None
|
|
||||||
# Path("") is truthy, so it must be rejected via its (empty) parts.
|
|
||||||
assert config.resolve_storage_path(Path("")) is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def client(tmp_path, monkeypatch):
|
|
||||||
"""Minimal app with only the audio routes and a temp sqlite DB."""
|
|
||||||
monkeypatch.setattr(config, "_data_dir", tmp_path)
|
|
||||||
# An existing directory that a stored audio_path may wrongly point to.
|
|
||||||
(tmp_path / "somedir").mkdir()
|
|
||||||
|
|
||||||
engine = create_engine(
|
|
||||||
f"sqlite:///{tmp_path / 'test.db'}",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(bind=engine)
|
|
||||||
testing_session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
|
||||||
|
|
||||||
session = testing_session_local()
|
|
||||||
profile = VoiceProfile(id="profile-1", name="Test Profile")
|
|
||||||
session.add(profile)
|
|
||||||
|
|
||||||
session.add_all(
|
|
||||||
[
|
|
||||||
Generation(
|
|
||||||
id="gen-failed-empty",
|
|
||||||
profile_id="profile-1",
|
|
||||||
text="failed generation",
|
|
||||||
audio_path="",
|
|
||||||
status="failed",
|
|
||||||
error="engine exploded",
|
|
||||||
),
|
|
||||||
Generation(
|
|
||||||
id="gen-failed-null",
|
|
||||||
profile_id="profile-1",
|
|
||||||
text="failed generation",
|
|
||||||
audio_path=None,
|
|
||||||
status="failed",
|
|
||||||
),
|
|
||||||
Generation(
|
|
||||||
id="gen-missing-file",
|
|
||||||
profile_id="profile-1",
|
|
||||||
text="completed but file deleted",
|
|
||||||
audio_path="generations/does-not-exist.wav",
|
|
||||||
status="completed",
|
|
||||||
),
|
|
||||||
Generation(
|
|
||||||
id="gen-with-version",
|
|
||||||
profile_id="profile-1",
|
|
||||||
text="generation with a broken version",
|
|
||||||
audio_path="somedir",
|
|
||||||
status="completed",
|
|
||||||
),
|
|
||||||
GenerationVersion(
|
|
||||||
id="version-dir",
|
|
||||||
generation_id="gen-with-version",
|
|
||||||
label="original",
|
|
||||||
audio_path="somedir",
|
|
||||||
),
|
|
||||||
ProfileSample(
|
|
||||||
id="sample-dir",
|
|
||||||
profile_id="profile-1",
|
|
||||||
audio_path="somedir",
|
|
||||||
reference_text="sample pointing at a directory",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
app = FastAPI()
|
|
||||||
app.include_router(audio_router)
|
|
||||||
|
|
||||||
def override_get_db():
|
|
||||||
db = testing_session_local()
|
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
app.dependency_overrides[get_db] = override_get_db
|
|
||||||
return TestClient(app)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("generation_id", ["gen-failed-empty", "gen-failed-null"])
|
|
||||||
def test_failed_generation_returns_404(client, generation_id):
|
|
||||||
"""Failed generations (empty/null audio_path) get a clean 404, not a 500."""
|
|
||||||
response = client.get(f"/audio/{generation_id}")
|
|
||||||
assert response.status_code == 404
|
|
||||||
assert response.json()["detail"] == "Generation failed; no audio available"
|
|
||||||
|
|
||||||
|
|
||||||
def test_missing_audio_file_returns_404(client):
|
|
||||||
"""A completed generation whose file vanished still 404s."""
|
|
||||||
response = client.get("/audio/gen-missing-file")
|
|
||||||
assert response.status_code == 404
|
|
||||||
assert response.json()["detail"] == "Audio file not found"
|
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_generation_returns_404(client):
|
|
||||||
response = client.get("/audio/no-such-generation")
|
|
||||||
assert response.status_code == 404
|
|
||||||
assert response.json()["detail"] == "Generation not found"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"url",
|
|
||||||
[
|
|
||||||
"/audio/gen-with-version",
|
|
||||||
"/audio/version/version-dir",
|
|
||||||
"/samples/sample-dir",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_audio_path_pointing_at_directory_returns_404(client, url):
|
|
||||||
"""A stored path resolving to an existing directory must 404, not 500.
|
|
||||||
|
|
||||||
Guards the is_file() checks: a directory passes exists() and would
|
|
||||||
crash FileResponse.
|
|
||||||
"""
|
|
||||||
response = client.get(url)
|
|
||||||
assert response.status_code == 404
|
|
||||||
assert response.json()["detail"] == "Audio file not found"
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
"""
|
|
||||||
Regression tests for issue #852: audioop removed from Python 3.13 stdlib.
|
|
||||||
|
|
||||||
Voice sample validation imports audioop transitively (librosa → audioread).
|
|
||||||
The audioop-lts backport must be declared in requirements and bundled in
|
|
||||||
PyInstaller builds on 3.13+.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
||||||
|
|
||||||
from build_binary import build_server
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def backend_dir():
|
|
||||||
return Path(__file__).parent.parent
|
|
||||||
|
|
||||||
|
|
||||||
class TestAudioopRequirements:
|
|
||||||
def test_requirements_declare_audioop_lts_for_python_313(self, backend_dir):
|
|
||||||
content = (backend_dir / "requirements.txt").read_text()
|
|
||||||
assert re.search(
|
|
||||||
r"^audioop-lts.*python_version\s*>=\s*['\"]3\.13['\"]",
|
|
||||||
content,
|
|
||||||
re.MULTILINE,
|
|
||||||
), "requirements.txt must pin audioop-lts for Python 3.13+"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(sys.version_info < (3, 13), reason="Python 3.13+ only")
|
|
||||||
class TestAudioopRuntime:
|
|
||||||
def test_audioop_importable(self):
|
|
||||||
import audioop # noqa: F401
|
|
||||||
|
|
||||||
def test_validate_reference_wav_does_not_fail_on_missing_audioop(self, tmp_path):
|
|
||||||
import numpy as np
|
|
||||||
import soundfile as sf
|
|
||||||
from utils.audio import validate_and_load_reference_audio
|
|
||||||
|
|
||||||
sr = 24000
|
|
||||||
t = np.arange(int(sr * 3), dtype=np.float32) / sr
|
|
||||||
audio = (0.3 * np.sin(2 * np.pi * 220 * t)).astype(np.float32)
|
|
||||||
path = tmp_path / "reference.wav"
|
|
||||||
sf.write(str(path), audio, sr)
|
|
||||||
|
|
||||||
ok, err, out_audio, out_sr = validate_and_load_reference_audio(str(path))
|
|
||||||
|
|
||||||
assert ok, err
|
|
||||||
assert out_audio is not None
|
|
||||||
assert out_sr == sr
|
|
||||||
assert "audioop" not in (err or "").lower()
|
|
||||||
|
|
||||||
|
|
||||||
class TestAudioopBuildArgs:
|
|
||||||
@staticmethod
|
|
||||||
def _hidden_imports(args):
|
|
||||||
imports = []
|
|
||||||
for i, arg in enumerate(args):
|
|
||||||
if arg == "--hidden-import" and i + 1 < len(args):
|
|
||||||
imports.append(args[i + 1])
|
|
||||||
return imports
|
|
||||||
|
|
||||||
def test_pyinstaller_includes_audioop_on_python_313(self):
|
|
||||||
class FakeVersionInfo(tuple):
|
|
||||||
@property
|
|
||||||
def major(self):
|
|
||||||
return self[0]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def minor(self):
|
|
||||||
return self[1]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def micro(self):
|
|
||||||
return self[2]
|
|
||||||
|
|
||||||
fake_313 = FakeVersionInfo((3, 13, 0, "final", 0))
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("build_binary.PyInstaller.__main__.run") as mock_run,
|
|
||||||
patch("build_binary.platform.system", return_value="Linux"),
|
|
||||||
patch("build_binary.is_apple_silicon", return_value=False),
|
|
||||||
patch("build_binary.os.chdir"),
|
|
||||||
patch("build_binary.sys.version_info", fake_313),
|
|
||||||
):
|
|
||||||
build_server()
|
|
||||||
args = mock_run.call_args[0][0]
|
|
||||||
|
|
||||||
assert "audioop" in self._hidden_imports(args)
|
|
||||||
|
|
||||||
def test_pyinstaller_omits_audioop_on_python_312(self):
|
|
||||||
class FakeVersionInfo(tuple):
|
|
||||||
@property
|
|
||||||
def major(self):
|
|
||||||
return self[0]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def minor(self):
|
|
||||||
return self[1]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def micro(self):
|
|
||||||
return self[2]
|
|
||||||
|
|
||||||
fake_312 = FakeVersionInfo((3, 12, 0, "final", 0))
|
|
||||||
|
|
||||||
with (
|
|
||||||
patch("build_binary.PyInstaller.__main__.run") as mock_run,
|
|
||||||
patch("build_binary.platform.system", return_value="Linux"),
|
|
||||||
patch("build_binary.is_apple_silicon", return_value=False),
|
|
||||||
patch("build_binary.os.chdir"),
|
|
||||||
patch("build_binary.sys.version_info", fake_312),
|
|
||||||
):
|
|
||||||
build_server()
|
|
||||||
args = mock_run.call_args[0][0]
|
|
||||||
|
|
||||||
assert "audioop" not in self._hidden_imports(args)
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
import sys as py_sys
|
|
||||||
import types
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from backend.services import cuda
|
|
||||||
|
|
||||||
|
|
||||||
def test_cuda_status_reports_unsupported_linux_download(monkeypatch, tmp_path):
|
|
||||||
monkeypatch.setattr(cuda.sys, "platform", "linux")
|
|
||||||
monkeypatch.setattr(cuda, "get_data_dir", lambda: tmp_path)
|
|
||||||
|
|
||||||
status = cuda.get_cuda_status()
|
|
||||||
|
|
||||||
assert status["available"] is False
|
|
||||||
assert status["download_supported"] is False
|
|
||||||
assert status["unsupported_reason"] == cuda.CUDA_DOWNLOAD_UNSUPPORTED_REASON
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_cuda_download_rejects_linux_before_network(monkeypatch, tmp_path):
|
|
||||||
monkeypatch.setattr(cuda.sys, "platform", "linux")
|
|
||||||
monkeypatch.setattr(cuda, "get_data_dir", lambda: tmp_path)
|
|
||||||
|
|
||||||
class UnexpectedClient:
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
raise AssertionError("unsupported platforms should not start a release download")
|
|
||||||
|
|
||||||
monkeypatch.setitem(py_sys.modules, "httpx", types.SimpleNamespace(AsyncClient=UnexpectedClient))
|
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match="currently only published for Windows"):
|
|
||||||
await cuda._download_cuda_binary_locked("v0.5.0")
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
"""Ensure TADA voice-prompt encoding disables autograd (#890)."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from unittest.mock import AsyncMock
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
import soundfile as sf
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from backend.backends.hume_backend import HumeTadaBackend
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class _FakeEncoderOutput:
|
|
||||||
emb: torch.Tensor
|
|
||||||
|
|
||||||
|
|
||||||
class _GradTrackingEncoder:
|
|
||||||
"""Raises unless called under torch.inference_mode()."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.called_under_inference_mode = False
|
|
||||||
|
|
||||||
def __call__(self, audio, text=None, sample_rate=None):
|
|
||||||
self.called_under_inference_mode = torch.is_inference_mode_enabled()
|
|
||||||
if not self.called_under_inference_mode:
|
|
||||||
raise AssertionError("encoder forward must run under inference_mode")
|
|
||||||
# Touch a requires_grad tensor the way Snake1d alpha would.
|
|
||||||
alpha = torch.nn.Parameter(torch.ones(1, device=audio.device))
|
|
||||||
_ = audio.mean() * alpha
|
|
||||||
return _FakeEncoderOutput(emb=torch.zeros(1, 4, device=audio.device))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_create_voice_prompt_runs_encoder_under_inference_mode(tmp_path, monkeypatch):
|
|
||||||
wav = tmp_path / "ref.wav"
|
|
||||||
sf.write(str(wav), np.zeros(24000, dtype=np.float32), 24000)
|
|
||||||
|
|
||||||
backend = HumeTadaBackend()
|
|
||||||
backend.model = object() # mark loaded
|
|
||||||
backend.model_size = "1B"
|
|
||||||
backend._device = "cpu"
|
|
||||||
encoder = _GradTrackingEncoder()
|
|
||||||
backend.encoder = encoder
|
|
||||||
|
|
||||||
monkeypatch.setattr(backend, "load_model", AsyncMock(return_value=None))
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"backend.backends.hume_backend.get_cached_voice_prompt",
|
|
||||||
lambda key: None,
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"backend.backends.hume_backend.cache_voice_prompt",
|
|
||||||
lambda key, value: None,
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt, from_cache = await backend.create_voice_prompt(
|
|
||||||
str(wav),
|
|
||||||
reference_text="hello world",
|
|
||||||
use_cache=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert from_cache is False
|
|
||||||
assert encoder.called_under_inference_mode is True
|
|
||||||
assert isinstance(prompt["emb"], torch.Tensor)
|
|
||||||
assert prompt["emb"].device.type == "cpu"
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
"""Tests for the voicebox.speak MCP tool's ``model_size`` plumbing (issue #884).
|
|
||||||
|
|
||||||
The MCP speak path used to build its ``GenerationRequest`` without a
|
|
||||||
``model_size``, so every agent-triggered generation silently fell back to the
|
|
||||||
schema default ("1.7B") — there was no way to reach 0.6B (or TADA's 1B/3B)
|
|
||||||
through MCP. These tests pin the fix: ``_speak`` now forwards ``model_size``
|
|
||||||
straight into the request, matching the REST ``/generate`` surface.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from pydantic import ValidationError
|
|
||||||
|
|
||||||
import backend.routes.generations as generations
|
|
||||||
from backend.mcp_server import tools
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeGeneration:
|
|
||||||
"""Minimal stand-in for GenerationResponse consumed by ``_speak_response``."""
|
|
||||||
|
|
||||||
def model_dump(self, mode="json"):
|
|
||||||
return {"id": "gen-test", "status": "generating"}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def captured_request(monkeypatch):
|
|
||||||
"""Replace the real (torch-backed) generate_speech with a capturing stub.
|
|
||||||
|
|
||||||
``_speak`` imports ``generate_speech`` lazily from ``routes.generations``,
|
|
||||||
so patching the attribute on that module intercepts the call and lets us
|
|
||||||
inspect the ``GenerationRequest`` it would have run.
|
|
||||||
"""
|
|
||||||
captured = {}
|
|
||||||
|
|
||||||
async def fake_generate_speech(req, db):
|
|
||||||
captured["req"] = req
|
|
||||||
return _FakeGeneration()
|
|
||||||
|
|
||||||
monkeypatch.setattr(generations, "generate_speech", fake_generate_speech)
|
|
||||||
# Isolate the unit from the MCP event bus — _speak_response fires a
|
|
||||||
# speak-start event we don't care about here.
|
|
||||||
monkeypatch.setattr(tools.mcp_events, "publish", lambda *a, **k: None)
|
|
||||||
return captured
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_speak_forwards_explicit_model_size(captured_request):
|
|
||||||
await tools._speak(
|
|
||||||
profile_id="p1",
|
|
||||||
profile_name="Morgan",
|
|
||||||
text="hello",
|
|
||||||
engine="qwen",
|
|
||||||
language="en",
|
|
||||||
personality=False,
|
|
||||||
model_size="0.6B",
|
|
||||||
db=None,
|
|
||||||
)
|
|
||||||
assert captured_request["req"].model_size == "0.6B"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_speak_omitted_model_size_is_none(captured_request):
|
|
||||||
# Omitted → None; generate_speech normalizes None to the engine default,
|
|
||||||
# so this reproduces the pre-fix behaviour for callers that don't ask.
|
|
||||||
await tools._speak(
|
|
||||||
profile_id="p1",
|
|
||||||
profile_name="Morgan",
|
|
||||||
text="hello",
|
|
||||||
engine="qwen",
|
|
||||||
language="en",
|
|
||||||
personality=False,
|
|
||||||
db=None,
|
|
||||||
)
|
|
||||||
assert captured_request["req"].model_size is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_speak_rejects_invalid_model_size(captured_request):
|
|
||||||
# The GenerationRequest schema pattern is the single source of truth for
|
|
||||||
# valid sizes; a bad value is rejected before any generation runs.
|
|
||||||
with pytest.raises(ValidationError):
|
|
||||||
await tools._speak(
|
|
||||||
profile_id="p1",
|
|
||||||
profile_name="Morgan",
|
|
||||||
text="hello",
|
|
||||||
engine="qwen",
|
|
||||||
language="en",
|
|
||||||
personality=False,
|
|
||||||
model_size="9B",
|
|
||||||
db=None,
|
|
||||||
)
|
|
||||||
assert "req" not in captured_request
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
"""
|
|
||||||
Smoke test for the MLX backend dependencies on Apple Silicon.
|
|
||||||
|
|
||||||
Guards the `--no-deps` install of mlx-audio/mlx-lm done by `just setup-python`
|
|
||||||
and release.yml: those packages skip their declared dependencies (transformers
|
|
||||||
>=5.x conflict), so a missing transitive dep only surfaces at import time.
|
|
||||||
This test fails fast if the MLX STT/TTS entry points the backend uses stop
|
|
||||||
importing (e.g. the `miniaudio` regression from issue #505).
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python -m pytest backend/tests/test_mlx_smoke.py -v
|
|
||||||
"""
|
|
||||||
|
|
||||||
import platform
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.skipif(
|
|
||||||
not (sys.platform == "darwin" and platform.machine() == "arm64"),
|
|
||||||
reason="MLX packages are only installed on Apple Silicon macOS",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_mlx_core_runs():
|
|
||||||
"""The MLX runtime itself works (Metal array op)."""
|
|
||||||
import mlx.core as mx
|
|
||||||
|
|
||||||
assert mx.array([1, 2]).sum().item() == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_mlx_audio_tts_entry_point():
|
|
||||||
"""`from mlx_audio.tts import load` — used by MLXBackend.load_model_async."""
|
|
||||||
from mlx_audio.tts import load
|
|
||||||
|
|
||||||
assert callable(load)
|
|
||||||
|
|
||||||
|
|
||||||
def test_mlx_audio_stt_entry_point():
|
|
||||||
"""`from mlx_audio.stt import load` — used by the Whisper MLX STT path.
|
|
||||||
|
|
||||||
Importing mlx_audio.stt also pulls in miniaudio, so this catches the
|
|
||||||
ModuleNotFoundError from issue #505 on fresh installs.
|
|
||||||
"""
|
|
||||||
from mlx_audio.stt import load
|
|
||||||
|
|
||||||
assert callable(load)
|
|
||||||
|
|
||||||
|
|
||||||
def test_mlx_lm_entry_points():
|
|
||||||
"""`mlx_lm.load` / `mlx_lm.generate` — used by qwen_llm_backend."""
|
|
||||||
from mlx_lm import generate, load
|
|
||||||
|
|
||||||
assert callable(load)
|
|
||||||
assert callable(generate)
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
"""Errored downloads must not be reported as still downloading.
|
|
||||||
|
|
||||||
A failed download intentionally stays in the TaskManager with
|
|
||||||
``status="error"`` so ``/tasks/active`` can surface the error and retry
|
|
||||||
UI — but ``/models/status`` derives its ``downloading`` flag from the
|
|
||||||
same list. Without a status filter, one failed download shows the model
|
|
||||||
as "downloading" forever and masks its real cache state until the app
|
|
||||||
restarts (issue #925, symptom reports like #181).
|
|
||||||
"""
|
|
||||||
|
|
||||||
from backend.utils.tasks import TaskManager
|
|
||||||
|
|
||||||
|
|
||||||
def test_errored_download_is_not_pending():
|
|
||||||
tm = TaskManager()
|
|
||||||
tm.start_download("whisper-turbo")
|
|
||||||
assert [t.model_name for t in tm.get_pending_downloads()] == ["whisper-turbo"]
|
|
||||||
|
|
||||||
tm.error_download("whisper-turbo", "boom")
|
|
||||||
|
|
||||||
assert tm.get_pending_downloads() == []
|
|
||||||
# Still visible to /tasks/active for the error/retry UI.
|
|
||||||
active = tm.get_active_downloads()
|
|
||||||
assert [t.model_name for t in active] == ["whisper-turbo"]
|
|
||||||
assert active[0].status == "error"
|
|
||||||
assert active[0].error == "boom"
|
|
||||||
|
|
||||||
|
|
||||||
def test_retry_after_error_is_pending_again():
|
|
||||||
tm = TaskManager()
|
|
||||||
tm.start_download("qwen3-4b")
|
|
||||||
tm.error_download("qwen3-4b", "boom")
|
|
||||||
tm.start_download("qwen3-4b")
|
|
||||||
assert [t.model_name for t in tm.get_pending_downloads()] == ["qwen3-4b"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_completed_download_is_removed_everywhere():
|
|
||||||
tm = TaskManager()
|
|
||||||
tm.start_download("whisper-turbo")
|
|
||||||
tm.complete_download("whisper-turbo")
|
|
||||||
assert tm.get_pending_downloads() == []
|
|
||||||
assert tm.get_active_downloads() == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_cancel_dismisses_errored_download():
|
|
||||||
tm = TaskManager()
|
|
||||||
tm.start_download("whisper-turbo")
|
|
||||||
tm.error_download("whisper-turbo", "boom")
|
|
||||||
assert tm.cancel_download("whisper-turbo") is True
|
|
||||||
assert tm.get_active_downloads() == []
|
|
||||||
assert tm.get_pending_downloads() == []
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
"""Regression coverage for runaway MLX Qwen TTS output."""
|
|
||||||
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from backend.backends import engine_needs_trim, engine_retries_runaway
|
|
||||||
from backend.utils.audio import has_tts_runaway
|
|
||||||
from backend.utils.chunked_tts import generate_chunked
|
|
||||||
|
|
||||||
SAMPLE_RATE = 1000
|
|
||||||
|
|
||||||
|
|
||||||
def test_mlx_qwen_enables_runaway_retry_without_aggressive_trim():
|
|
||||||
with patch("backend.backends.get_backend_type", return_value="mlx"):
|
|
||||||
assert engine_needs_trim("qwen") is False
|
|
||||||
assert engine_retries_runaway("qwen") is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_pytorch_qwen_keeps_runaway_retry_disabled():
|
|
||||||
with patch("backend.backends.get_backend_type", return_value="pytorch"):
|
|
||||||
assert engine_needs_trim("qwen") is False
|
|
||||||
assert engine_retries_runaway("qwen") is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_detector_flags_long_internal_silence():
|
|
||||||
speech = np.full(2 * SAMPLE_RATE, 0.2, dtype=np.float32)
|
|
||||||
runaway_gap = np.zeros(2500, dtype=np.float32)
|
|
||||||
hallucinated_noise = np.full(2 * SAMPLE_RATE, 0.8, dtype=np.float32)
|
|
||||||
audio = np.concatenate([speech, runaway_gap, hallucinated_noise])
|
|
||||||
|
|
||||||
assert has_tts_runaway(audio, SAMPLE_RATE) is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_detector_ignores_normal_internal_pause():
|
|
||||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
|
||||||
normal_pause = np.zeros(1200, dtype=np.float32)
|
|
||||||
audio = np.concatenate([speech, normal_pause, speech])
|
|
||||||
|
|
||||||
assert has_tts_runaway(audio, SAMPLE_RATE) is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_trailing_silence_is_not_a_runaway():
|
|
||||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
|
||||||
trailing_silence = np.zeros(2 * SAMPLE_RATE, dtype=np.float32)
|
|
||||||
|
|
||||||
assert (
|
|
||||||
has_tts_runaway(
|
|
||||||
np.concatenate([speech, trailing_silence]),
|
|
||||||
SAMPLE_RATE,
|
|
||||||
)
|
|
||||||
is False
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_runaway_chunk_is_retried_as_smaller_chunks():
|
|
||||||
class FakeBackend:
|
|
||||||
def __init__(self):
|
|
||||||
self.calls = []
|
|
||||||
|
|
||||||
async def generate(self, text, *_args):
|
|
||||||
self.calls.append(text)
|
|
||||||
if len(text) > 200:
|
|
||||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
|
||||||
silence = np.zeros(2500, dtype=np.float32)
|
|
||||||
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
|
|
||||||
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
|
|
||||||
return np.full(SAMPLE_RATE, 0.2, dtype=np.float32), SAMPLE_RATE
|
|
||||||
|
|
||||||
backend = FakeBackend()
|
|
||||||
text = f"{'A' * 119}. {'B' * 119}."
|
|
||||||
|
|
||||||
audio, sample_rate = await generate_chunked(
|
|
||||||
backend,
|
|
||||||
text,
|
|
||||||
{},
|
|
||||||
max_chunk_chars=800,
|
|
||||||
crossfade_ms=50,
|
|
||||||
runaway_detector=has_tts_runaway,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert sample_rate == SAMPLE_RATE
|
|
||||||
assert backend.calls == [text, f"{'A' * 119}.", f"{'B' * 119}."]
|
|
||||||
assert len(audio) == 1950
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_persistent_runaway_fails_instead_of_returning_corrupt_audio():
|
|
||||||
class AlwaysRunawayBackend:
|
|
||||||
def __init__(self):
|
|
||||||
self.calls = []
|
|
||||||
|
|
||||||
async def generate(self, text, *_args):
|
|
||||||
self.calls.append(text)
|
|
||||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
|
||||||
silence = np.zeros(2500, dtype=np.float32)
|
|
||||||
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
|
|
||||||
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
|
|
||||||
|
|
||||||
backend = AlwaysRunawayBackend()
|
|
||||||
text = f"{'A' * 119}. {'B' * 119}."
|
|
||||||
|
|
||||||
with pytest.raises(
|
|
||||||
RuntimeError,
|
|
||||||
match="remained unstable after retrying smaller text chunks",
|
|
||||||
):
|
|
||||||
await generate_chunked(
|
|
||||||
backend,
|
|
||||||
text,
|
|
||||||
{},
|
|
||||||
max_chunk_chars=800,
|
|
||||||
runaway_detector=has_tts_runaway,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert [len(call) for call in backend.calls] == [241, 120, 100]
|
|
||||||
@@ -110,43 +110,6 @@ def save_audio(
|
|||||||
raise OSError(f"Failed to save audio to {path}: {e}") from e
|
raise OSError(f"Failed to save audio to {path}: {e}") from e
|
||||||
|
|
||||||
|
|
||||||
def has_tts_runaway(
|
|
||||||
audio: np.ndarray,
|
|
||||||
sample_rate: int = 24000,
|
|
||||||
frame_ms: int = 20,
|
|
||||||
silence_threshold_db: float = -40.0,
|
|
||||||
max_internal_silence_ms: int = 2000,
|
|
||||||
) -> bool:
|
|
||||||
"""Detect speech followed by a long silence and then more output.
|
|
||||||
|
|
||||||
This shape is a reliable signal that a TTS model missed EOS and resumed
|
|
||||||
with hallucinated speech or codec noise. Leading and trailing silence do
|
|
||||||
not count because they are not bounded by non-silent audio.
|
|
||||||
"""
|
|
||||||
frame_len = int(sample_rate * frame_ms / 1000)
|
|
||||||
if frame_len == 0 or len(audio) < frame_len:
|
|
||||||
return False
|
|
||||||
|
|
||||||
n_frames = len(audio) // frame_len
|
|
||||||
threshold_linear = 10 ** (silence_threshold_db / 20)
|
|
||||||
max_silence_frames = int(max_internal_silence_ms / frame_ms)
|
|
||||||
seen_speech = False
|
|
||||||
consecutive_silence = 0
|
|
||||||
|
|
||||||
for i in range(n_frames):
|
|
||||||
frame = audio[i * frame_len : (i + 1) * frame_len]
|
|
||||||
is_speech = np.sqrt(np.mean(frame**2)) >= threshold_linear
|
|
||||||
if is_speech:
|
|
||||||
if seen_speech and consecutive_silence >= max_silence_frames:
|
|
||||||
return True
|
|
||||||
seen_speech = True
|
|
||||||
consecutive_silence = 0
|
|
||||||
elif seen_speech:
|
|
||||||
consecutive_silence += 1
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def trim_tts_output(
|
def trim_tts_output(
|
||||||
audio: np.ndarray,
|
audio: np.ndarray,
|
||||||
sample_rate: int = 24000,
|
sample_rate: int = 24000,
|
||||||
|
|||||||
@@ -20,8 +20,6 @@ logger = logging.getLogger("voicebox.chunked-tts")
|
|||||||
# Default chunk size in characters. Can be overridden per-request via
|
# Default chunk size in characters. Can be overridden per-request via
|
||||||
# the ``max_chunk_chars`` field on GenerationRequest.
|
# the ``max_chunk_chars`` field on GenerationRequest.
|
||||||
DEFAULT_MAX_CHUNK_CHARS = 800
|
DEFAULT_MAX_CHUNK_CHARS = 800
|
||||||
MAX_RUNAWAY_RETRIES = 2
|
|
||||||
MIN_RUNAWAY_RETRY_CHARS = 100
|
|
||||||
|
|
||||||
# Common abbreviations that should NOT be treated as sentence endings.
|
# Common abbreviations that should NOT be treated as sentence endings.
|
||||||
# Lowercase for case-insensitive matching.
|
# Lowercase for case-insensitive matching.
|
||||||
@@ -213,7 +211,6 @@ async def generate_chunked(
|
|||||||
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
|
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
|
||||||
crossfade_ms: int = 50,
|
crossfade_ms: int = 50,
|
||||||
trim_fn=None,
|
trim_fn=None,
|
||||||
runaway_detector=None,
|
|
||||||
) -> Tuple[np.ndarray, int]:
|
) -> Tuple[np.ndarray, int]:
|
||||||
"""Generate audio with automatic chunking for long text.
|
"""Generate audio with automatic chunking for long text.
|
||||||
|
|
||||||
@@ -242,75 +239,25 @@ async def generate_chunked(
|
|||||||
Optional ``(audio, sample_rate) -> audio`` post-processing
|
Optional ``(audio, sample_rate) -> audio`` post-processing
|
||||||
function applied to each chunk before concatenation (e.g.
|
function applied to each chunk before concatenation (e.g.
|
||||||
``trim_tts_output`` for Chatterbox engines).
|
``trim_tts_output`` for Chatterbox engines).
|
||||||
runaway_detector : callable | None
|
|
||||||
Optional ``(audio, sample_rate) -> bool`` detector. When it flags
|
|
||||||
unstable output, the affected text is split in half and retried.
|
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
(audio, sample_rate) : Tuple[np.ndarray, int]
|
(audio, sample_rate) : Tuple[np.ndarray, int]
|
||||||
"""
|
"""
|
||||||
async def generate_one(
|
|
||||||
chunk_text: str,
|
|
||||||
chunk_seed: int | None,
|
|
||||||
retry_depth: int = 0,
|
|
||||||
) -> tuple[np.ndarray, int]:
|
|
||||||
chunk_audio, chunk_sr = await backend.generate(
|
|
||||||
chunk_text,
|
|
||||||
voice_prompt,
|
|
||||||
language,
|
|
||||||
chunk_seed,
|
|
||||||
instruct,
|
|
||||||
)
|
|
||||||
|
|
||||||
if runaway_detector is not None and runaway_detector(chunk_audio, chunk_sr):
|
|
||||||
if retry_depth >= MAX_RUNAWAY_RETRIES or len(chunk_text) <= MIN_RUNAWAY_RETRY_CHARS:
|
|
||||||
raise RuntimeError(
|
|
||||||
"TTS output remained unstable after retrying smaller text chunks"
|
|
||||||
)
|
|
||||||
|
|
||||||
retry_max_chars = max(MIN_RUNAWAY_RETRY_CHARS, len(chunk_text) // 2)
|
|
||||||
retry_chunks = split_text_into_chunks(chunk_text, retry_max_chars)
|
|
||||||
if len(retry_chunks) <= 1:
|
|
||||||
raise RuntimeError("Unable to split unstable TTS output for retry")
|
|
||||||
|
|
||||||
logger.warning(
|
|
||||||
"Detected unstable TTS output for %d chars; retrying as %d smaller chunks",
|
|
||||||
len(chunk_text),
|
|
||||||
len(retry_chunks),
|
|
||||||
)
|
|
||||||
retry_audio: list[np.ndarray] = []
|
|
||||||
for i, retry_text in enumerate(retry_chunks):
|
|
||||||
retry_seed = (
|
|
||||||
chunk_seed + ((retry_depth + 1) * 1000) + i
|
|
||||||
if chunk_seed is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
audio, sample_rate = await generate_one(
|
|
||||||
retry_text,
|
|
||||||
retry_seed,
|
|
||||||
retry_depth + 1,
|
|
||||||
)
|
|
||||||
retry_audio.append(np.asarray(audio, dtype=np.float32))
|
|
||||||
|
|
||||||
return (
|
|
||||||
concatenate_audio_chunks(
|
|
||||||
retry_audio,
|
|
||||||
sample_rate,
|
|
||||||
crossfade_ms=crossfade_ms,
|
|
||||||
),
|
|
||||||
sample_rate,
|
|
||||||
)
|
|
||||||
|
|
||||||
if trim_fn is not None:
|
|
||||||
chunk_audio = trim_fn(chunk_audio, chunk_sr)
|
|
||||||
return np.asarray(chunk_audio, dtype=np.float32), chunk_sr
|
|
||||||
|
|
||||||
chunks = split_text_into_chunks(text, max_chunk_chars)
|
chunks = split_text_into_chunks(text, max_chunk_chars)
|
||||||
|
|
||||||
if len(chunks) <= 1:
|
if len(chunks) <= 1:
|
||||||
# Short text — single-shot fast path
|
# Short text — single-shot fast path
|
||||||
return await generate_one(text, seed)
|
audio, sample_rate = await backend.generate(
|
||||||
|
text,
|
||||||
|
voice_prompt,
|
||||||
|
language,
|
||||||
|
seed,
|
||||||
|
instruct,
|
||||||
|
)
|
||||||
|
if trim_fn is not None:
|
||||||
|
audio = trim_fn(audio, sample_rate)
|
||||||
|
return audio, sample_rate
|
||||||
|
|
||||||
# Long text — chunked generation
|
# Long text — chunked generation
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -334,12 +281,17 @@ async def generate_chunked(
|
|||||||
# always produces the same output.
|
# always produces the same output.
|
||||||
chunk_seed = (seed + i) if seed is not None else None
|
chunk_seed = (seed + i) if seed is not None else None
|
||||||
|
|
||||||
chunk_audio, chunk_sr = await generate_one(
|
chunk_audio, chunk_sr = await backend.generate(
|
||||||
chunk_text,
|
chunk_text,
|
||||||
|
voice_prompt,
|
||||||
|
language,
|
||||||
chunk_seed,
|
chunk_seed,
|
||||||
|
instruct,
|
||||||
)
|
)
|
||||||
|
if trim_fn is not None:
|
||||||
|
chunk_audio = trim_fn(chunk_audio, chunk_sr)
|
||||||
|
|
||||||
audio_chunks.append(chunk_audio)
|
audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
|
||||||
if sample_rate is None:
|
if sample_rate is None:
|
||||||
sample_rate = chunk_sr
|
sample_rate = chunk_sr
|
||||||
|
|
||||||
|
|||||||
@@ -68,19 +68,6 @@ class TaskManager:
|
|||||||
"""Get all active downloads."""
|
"""Get all active downloads."""
|
||||||
return list(self._active_downloads.values())
|
return list(self._active_downloads.values())
|
||||||
|
|
||||||
def get_pending_downloads(self) -> List[DownloadTask]:
|
|
||||||
"""Get downloads that are still in flight.
|
|
||||||
|
|
||||||
Excludes errored tasks, which stay in the active list so the
|
|
||||||
error/retry UI can show them but must not be reported as
|
|
||||||
"downloading" by /models/status.
|
|
||||||
"""
|
|
||||||
return [
|
|
||||||
task
|
|
||||||
for task in self._active_downloads.values()
|
|
||||||
if task.status in ("downloading", "extracting")
|
|
||||||
]
|
|
||||||
|
|
||||||
def get_active_generations(self) -> List[GenerationTask]:
|
def get_active_generations(self) -> List[GenerationTask]:
|
||||||
"""Get all active generations."""
|
"""Get all active generations."""
|
||||||
return list(self._active_generations.values())
|
return list(self._active_generations.values())
|
||||||
|
|||||||
@@ -57,6 +57,7 @@
|
|||||||
"react-dom": "^18.3.0",
|
"react-dom": "^18.3.0",
|
||||||
"react-hook-form": "^7.53.0",
|
"react-hook-form": "^7.53.0",
|
||||||
"react-i18next": "^17.0.4",
|
"react-i18next": "^17.0.4",
|
||||||
|
"react-qr-code": "^2.0.18",
|
||||||
"react-sound-visualizer": "^1.4.0",
|
"react-sound-visualizer": "^1.4.0",
|
||||||
"tailwind-merge": "^2.5.4",
|
"tailwind-merge": "^2.5.4",
|
||||||
"wavesurfer.js": "^7.0.0",
|
"wavesurfer.js": "^7.0.0",
|
||||||
@@ -1005,6 +1006,8 @@
|
|||||||
|
|
||||||
"punycode": ["[email protected]", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
|
"punycode": ["[email protected]", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="],
|
||||||
|
|
||||||
|
"qr.js": ["[email protected]", "", {}, "sha512-c4iYnWb+k2E+vYpRimHqSu575b1/wKl4XFeJGpFmrJQz5I88v9aY2czh7s0w36srfCM1sXgC/xpoJz5dJfq+OQ=="],
|
||||||
|
|
||||||
"queue-microtask": ["[email protected]", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="],
|
"queue-microtask": ["[email protected]", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="],
|
||||||
|
|
||||||
"react": ["[email protected]", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="],
|
"react": ["[email protected]", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="],
|
||||||
@@ -1019,6 +1022,8 @@
|
|||||||
|
|
||||||
"react-loaders": ["[email protected]", "", { "dependencies": { "classnames": "^2.2.3" }, "peerDependencies": { "prop-types": ">=15.6.0", "react": ">=15" } }, "sha512-4igMNqs9Fb3d4Z+0UHIGQNJsw/37gX0nUO8QxupnEKRn1dtyYC1LGwk5GuaoDciMQCQc/MmPwb4Fn6ZfdoX1FQ=="],
|
"react-loaders": ["[email protected]", "", { "dependencies": { "classnames": "^2.2.3" }, "peerDependencies": { "prop-types": ">=15.6.0", "react": ">=15" } }, "sha512-4igMNqs9Fb3d4Z+0UHIGQNJsw/37gX0nUO8QxupnEKRn1dtyYC1LGwk5GuaoDciMQCQc/MmPwb4Fn6ZfdoX1FQ=="],
|
||||||
|
|
||||||
|
"react-qr-code": ["[email protected]", "", { "dependencies": { "prop-types": "^15.8.1", "qr.js": "0.0.0" }, "peerDependencies": { "react": "*" } }, "sha512-v1Jqz7urLMhkO6jkgJuBYhnqvXagzceg3qJUWayuCK/c6LTIonpWbwxR1f1APGd4xrW/QcQEovNrAojbUz65Tg=="],
|
||||||
|
|
||||||
"react-refresh": ["[email protected]", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
"react-refresh": ["[email protected]", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
||||||
|
|
||||||
"react-remove-scroll": ["[email protected]", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="],
|
"react-remove-scroll": ["[email protected]", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="],
|
||||||
|
|||||||
@@ -34,15 +34,3 @@ services:
|
|||||||
|
|
||||||
# Tune the ROCm memory allocator
|
# Tune the ROCm memory allocator
|
||||||
- PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,max_split_size_mb:512
|
- PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,max_split_size_mb:512
|
||||||
|
|
||||||
# Redirect MIOpen kernel cache to a writable, persistent directory.
|
|
||||||
# Without this, MIOpen may fail to write its cache and throw
|
|
||||||
# miopenStatusUnknownError on fresh containers.
|
|
||||||
- MIOPEN_USER_DB_PATH=/app/data/cache/miopen_db
|
|
||||||
- MIOPEN_CUSTOM_CACHE_DIR=/app/data/cache/miopen_cache
|
|
||||||
|
|
||||||
# Use fast heuristics for kernel selection instead of exhaustive
|
|
||||||
# benchmarking. On RDNA4, exhaustive mode tries kernels that fail to
|
|
||||||
# allocate workspace memory (ptr: 0 size: 0), causing system stuttering
|
|
||||||
# on every generation even when the cache is present.
|
|
||||||
- MIOPEN_FIND_MODE=FAST
|
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ class ModelConfig:
|
|||||||
model_size: str = "default"
|
model_size: str = "default"
|
||||||
size_mb: int = 0
|
size_mb: int = 0
|
||||||
needs_trim: bool = False
|
needs_trim: bool = False
|
||||||
retries_runaway: bool = False
|
|
||||||
supports_instruct: bool = False
|
supports_instruct: bool = False
|
||||||
languages: list[str] = field(default_factory=lambda: ["en"])
|
languages: list[str] = field(default_factory=lambda: ["en"])
|
||||||
```
|
```
|
||||||
@@ -60,7 +59,6 @@ Registry helpers in `backends/__init__.py` replace what used to be per-engine `i
|
|||||||
- `get_tts_model_configs()` — only TTS variants
|
- `get_tts_model_configs()` — only TTS variants
|
||||||
- `get_model_config(model_name)` — lookup by name
|
- `get_model_config(model_name)` — lookup by name
|
||||||
- `engine_needs_trim(engine)` — whether output should run through `trim_tts_output()`
|
- `engine_needs_trim(engine)` — whether output should run through `trim_tts_output()`
|
||||||
- `engine_retries_runaway(engine)` — whether unstable output should be retried as smaller chunks
|
|
||||||
- `load_engine_model(engine, model_size)` — downloads + loads, handles engines with multiple sizes
|
- `load_engine_model(engine, model_size)` — downloads + loads, handles engines with multiple sizes
|
||||||
- `get_tts_backend_for_engine(engine)` — thread-safe backend factory with double-checked locking
|
- `get_tts_backend_for_engine(engine)` — thread-safe backend factory with double-checked locking
|
||||||
|
|
||||||
@@ -154,7 +152,7 @@ The request path from frontend to audio file:
|
|||||||
|
|
||||||
6. **Inference** — the engine's `generate()` returns `(audio_array, sample_rate)`.
|
6. **Inference** — the engine's `generate()` returns `(audio_array, sample_rate)`.
|
||||||
|
|
||||||
7. **Validate and post-process** — engines with `retries_runaway=True` retry unstable output as smaller chunks. If `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
|
7. **Post-process** — if `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
|
||||||
|
|
||||||
8. **Persist** — audio is written to the generations directory, a row is inserted into the `generations` table, and the response includes the generation metadata.
|
8. **Persist** — audio is written to the generations directory, a row is inserted into the `generations` table, and the response includes the generation metadata.
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ This page is for the cases where it doesn't:
|
|||||||
| **Windows + NVIDIA** | PyTorch CUDA (cu128) | Auto-downloads the CUDA backend binary on first use |
|
| **Windows + NVIDIA** | PyTorch CUDA (cu128) | Auto-downloads the CUDA backend binary on first use |
|
||||||
| **Windows + Intel Arc** | PyTorch XPU (IPEX) | New in 0.4 — works with Arc A-series and B-series |
|
| **Windows + Intel Arc** | PyTorch XPU (IPEX) | New in 0.4 — works with Arc A-series and B-series |
|
||||||
| **Windows generic GPU** | DirectML | Universal Windows GPU support; slower than CUDA |
|
| **Windows generic GPU** | DirectML | Universal Windows GPU support; slower than CUDA |
|
||||||
| **Linux + NVIDIA** | PyTorch CUDA (cu128) | Use a local/remote Python backend with CUDA PyTorch |
|
| **Linux + NVIDIA** | PyTorch CUDA (cu128) | Same auto-download flow as Windows |
|
||||||
| **Linux + AMD** | PyTorch ROCm | Auto-configures `HSA_OVERRIDE_GFX_VERSION` |
|
| **Linux + AMD** | PyTorch ROCm | Auto-configures `HSA_OVERRIDE_GFX_VERSION` |
|
||||||
| **Linux + Intel Arc** | PyTorch XPU (IPEX) | |
|
| **Linux + Intel Arc** | PyTorch XPU (IPEX) | |
|
||||||
| **Any (no GPU)** | PyTorch CPU | Works everywhere; expect 5-50x slower than GPU |
|
| **Any (no GPU)** | PyTorch CPU | Works everywhere; expect 5-50x slower than GPU |
|
||||||
@@ -46,7 +46,7 @@ On M-series Macs, Voicebox ships an MLX-optimized backend that uses the Apple Ne
|
|||||||
|
|
||||||
The Whisper Turbo + MLX combo dropped transcription latency from ~20s to ~2-3s on M-series chips (see CHANGELOG entry for v0.1.10).
|
The Whisper Turbo + MLX combo dropped transcription latency from ~20s to ~2-3s on M-series chips (see CHANGELOG entry for v0.1.10).
|
||||||
|
|
||||||
## Windows + NVIDIA — The CUDA Backend Swap
|
## Windows / Linux + NVIDIA — The CUDA Backend Swap
|
||||||
|
|
||||||
Voicebox doesn't bundle CUDA into the main installer (it would balloon downloads to multi-gigabyte territory for users who don't have an NVIDIA GPU). Instead, when you first need it, the app downloads a separate **CUDA backend binary** that contains the PyTorch + CUDA runtime.
|
Voicebox doesn't bundle CUDA into the main installer (it would balloon downloads to multi-gigabyte territory for users who don't have an NVIDIA GPU). Instead, when you first need it, the app downloads a separate **CUDA backend binary** that contains the PyTorch + CUDA runtime.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
---
|
||||||
|
title: "Hermes Agent"
|
||||||
|
description: "Use Voicebox as the voice and ears of Hermes Agent — spoken replies and voice-message transcription, fully local."
|
||||||
|
---
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
[Hermes Agent](https://github.com/NousResearch/hermes-agent) is Nous
|
||||||
|
Research's open-source self-improving agent: a terminal CLI/TUI plus a
|
||||||
|
messaging gateway that connects one agent to Telegram, Discord, WhatsApp,
|
||||||
|
Slack, and Signal. It has first-class voice features — spoken replies,
|
||||||
|
voice-bubble delivery on chat platforms, push-to-talk dictation, and
|
||||||
|
automatic transcription of incoming voice messages — and every one of them
|
||||||
|
is pluggable.
|
||||||
|
|
||||||
|
Voicebox slots into both directions of that loop, entirely on-device:
|
||||||
|
|
||||||
|
- **Voice out** — Hermes speaks its replies in one of your cloned or preset
|
||||||
|
voices instead of a stock cloud voice.
|
||||||
|
- **Voice in** — voice messages and push-to-talk audio are transcribed by
|
||||||
|
the Whisper models already bundled with Voicebox. Audio never leaves your
|
||||||
|
machine.
|
||||||
|
|
||||||
|
There are two integration surfaces, and they compose — most people will
|
||||||
|
want both. Everything talks to the same local API
|
||||||
|
(`http://127.0.0.1:17493` while the Voicebox app is running).
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
Running Voicebox in Docker instead of the desktop app? The API is on
|
||||||
|
`http://127.0.0.1:17600` — set `VOICEBOX_BASE_URL` accordingly wherever it
|
||||||
|
appears below. See [Docker](/overview/docker).
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
## MCP: agent-invoked voice tools
|
||||||
|
|
||||||
|
Hermes speaks MCP natively, and Voicebox ships a built-in
|
||||||
|
[MCP server](/overview/mcp-server). Voicebox is in Hermes's approved MCP
|
||||||
|
catalog, so:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hermes mcp install voicebox
|
||||||
|
```
|
||||||
|
|
||||||
|
(Or add the block manually to `~/.hermes/config.yaml`:)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
mcp_servers:
|
||||||
|
voicebox:
|
||||||
|
url: "http://127.0.0.1:17493/mcp"
|
||||||
|
headers:
|
||||||
|
X-Voicebox-Client-Id: "hermes"
|
||||||
|
```
|
||||||
|
|
||||||
|
Hermes discovers the tools — `voicebox.speak`, `voicebox.transcribe`,
|
||||||
|
`voicebox.list_profiles`, `voicebox.list_captures` — and the agent can now
|
||||||
|
*choose* to use them: "read me that summary in Morgan's voice" works
|
||||||
|
immediately, and the [per-client binding](/overview/mcp-server#per-client-bindings)
|
||||||
|
for `hermes` lets you pin its default voice from the Voicebox UI.
|
||||||
|
|
||||||
|
MCP makes Voicebox a set of tools the agent may call. It does **not**
|
||||||
|
reroute Hermes's own voice pipeline — spoken replies, voice bubbles, and
|
||||||
|
incoming voice-message transcription still use whatever `tts.provider` /
|
||||||
|
`stt.provider` are set to. That's the plugin's job.
|
||||||
|
|
||||||
|
## Provider plugin: Hermes's own voice pipeline
|
||||||
|
|
||||||
|
[`hermes-voicebox`](https://github.com/jamiepine/hermes-voicebox) registers
|
||||||
|
Voicebox as a Hermes **TTS provider** and **STT provider** via Hermes's
|
||||||
|
pluggable backend interfaces (`register_tts_provider` /
|
||||||
|
`register_transcription_provider` — see
|
||||||
|
[Build a Hermes Plugin](https://hermes-agent.nousresearch.com/docs/developer-guide/plugins)).
|
||||||
|
Once selected, the providers service the *entire* voice pipeline: every
|
||||||
|
spoken reply, every Telegram voice bubble, every incoming voice memo — plus
|
||||||
|
a bundled skill that teaches the agent when speaking aloud is appropriate
|
||||||
|
and to recall your dictated [Captures](/overview/captures) through MCP.
|
||||||
|
|
||||||
|
<Steps>
|
||||||
|
|
||||||
|
### Install the plugin
|
||||||
|
|
||||||
|
Into the same Python environment Hermes runs in:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install hermes-voicebox
|
||||||
|
```
|
||||||
|
|
||||||
|
No pip? Copy it in as a directory plugin instead:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/jamiepine/hermes-voicebox /tmp/hermes-voicebox
|
||||||
|
cp -r /tmp/hermes-voicebox/hermes_voicebox ~/.hermes/plugins/voicebox
|
||||||
|
hermes plugins enable voicebox
|
||||||
|
```
|
||||||
|
|
||||||
|
### Select the providers
|
||||||
|
|
||||||
|
In `~/.hermes/config.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
tts:
|
||||||
|
provider: voicebox
|
||||||
|
|
||||||
|
stt:
|
||||||
|
provider: voicebox
|
||||||
|
```
|
||||||
|
|
||||||
|
### Try it
|
||||||
|
|
||||||
|
With the Voicebox app open, start `hermes chat` and ask it to say
|
||||||
|
something out loud — or send your Hermes bot a voice message on Telegram
|
||||||
|
and watch the transcript come back from your local Whisper.
|
||||||
|
|
||||||
|
</Steps>
|
||||||
|
|
||||||
|
## Behavior notes
|
||||||
|
|
||||||
|
- **Voicebox must be running.** The desktop app only serves the API while
|
||||||
|
it's open. Both providers implement availability as a live `/health`
|
||||||
|
check, so Hermes's provider picker reflects reality.
|
||||||
|
- **First generation is slower** while the TTS engine loads into memory;
|
||||||
|
subsequent calls are fast. Same for the first transcription with a new
|
||||||
|
Whisper size — Voicebox answers `202` while the model downloads, and the
|
||||||
|
plugin surfaces a friendly "try again in a minute".
|
||||||
|
- **Voice selection**: `tts.voice` in Hermes config (or the tool's `voice`
|
||||||
|
argument) accepts a Voicebox profile **name or id**. With no voice set,
|
||||||
|
the first profile is used.
|
||||||
|
- **Engines**: pass a Voicebox engine id (`qwen`, `kokoro`,
|
||||||
|
`chatterbox`, …) as the Hermes `model` to override the profile's
|
||||||
|
default engine.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
|
||||||
|
- [MCP Server](/overview/mcp-server) — the tool-call route, per-client
|
||||||
|
bindings, and the speaking pill
|
||||||
|
- [Creating Voice Profiles](/overview/creating-voice-profiles) — clone the
|
||||||
|
voice Hermes will speak in
|
||||||
|
- [Remote Mode](/overview/remote-mode) — reaching a Voicebox instance on
|
||||||
|
another machine (read the security notes first: the API has no auth)
|
||||||
@@ -75,8 +75,7 @@ No cloud fallback, no bring-your-own-API-key. Local is the product.
|
|||||||
| Platform | Backend | Notes |
|
| Platform | Backend | Notes |
|
||||||
|----------|---------|-------|
|
|----------|---------|-------|
|
||||||
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
|
| macOS (Apple Silicon) | MLX (Metal) | 4-5x faster via Neural Engine |
|
||||||
| Windows (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
|
| Windows / Linux (NVIDIA) | PyTorch (CUDA) | Auto-downloads CUDA binary from within the app |
|
||||||
| Linux (NVIDIA) | PyTorch (CUDA) | Use a local/remote Python backend with CUDA PyTorch |
|
|
||||||
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
|
| Linux (AMD) | PyTorch (ROCm) | Auto-configures HSA_OVERRIDE_GFX_VERSION |
|
||||||
| Windows (any GPU) | DirectML | Universal Windows GPU support |
|
| Windows (any GPU) | DirectML | Universal Windows GPU support |
|
||||||
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
|
| Intel Arc | IPEX/XPU | Intel discrete GPU acceleration |
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
"preset-voices",
|
"preset-voices",
|
||||||
"voice-personalities",
|
"voice-personalities",
|
||||||
"mcp-server",
|
"mcp-server",
|
||||||
|
"hermes-agent",
|
||||||
"stories-editor",
|
"stories-editor",
|
||||||
"recording-transcription",
|
"recording-transcription",
|
||||||
"generation-history",
|
"generation-history",
|
||||||
|
|||||||
@@ -14,12 +14,12 @@ Make sure you have [installed Voicebox](/overview/installation) and launched the
|
|||||||
Voice profiles are the foundation of Voicebox. Each profile contains voice samples that the AI uses to clone the voice.
|
Voice profiles are the foundation of Voicebox. Each profile contains voice samples that the AI uses to clone the voice.
|
||||||
|
|
||||||
<Steps>
|
<Steps>
|
||||||
<Step title="Navigate to Voices">
|
<Step title="Navigate to Profiles">
|
||||||
Click the **Voices** tab in the sidebar
|
Click the **Profiles** tab in the sidebar
|
||||||
</Step>
|
</Step>
|
||||||
|
|
||||||
<Step title="Create New Voice">
|
<Step title="Create New Profile">
|
||||||
Click the **+ New Voice** button
|
Click the **+ New Profile** button
|
||||||
|
|
||||||
Fill in the details:
|
Fill in the details:
|
||||||
- **Name:** A descriptive name (e.g., "John Smith")
|
- **Name:** A descriptive name (e.g., "John Smith")
|
||||||
|
|||||||
@@ -72,12 +72,6 @@ setup-python:
|
|||||||
if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
|
if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then
|
||||||
echo "Detected Apple Silicon — installing MLX dependencies..."
|
echo "Detected Apple Silicon — installing MLX dependencies..."
|
||||||
{{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt
|
{{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt
|
||||||
# mlx-lm and mlx-audio declare transformers>=5.x, which conflicts with
|
|
||||||
# our transformers<=4.57.x cap, so install them --no-deps (their other
|
|
||||||
# runtime deps are covered by requirements.txt / requirements-mlx.txt —
|
|
||||||
# see the note in requirements-mlx.txt and .github/workflows/release.yml)
|
|
||||||
{{ pip }} install --no-deps mlx-lm==0.31.1
|
|
||||||
{{ pip }} install --no-deps mlx-audio==0.4.1
|
|
||||||
fi
|
fi
|
||||||
{{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git
|
{{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git
|
||||||
{{ pip }} install pyinstaller ruff pytest pytest-asyncio -q
|
{{ pip }} install pyinstaller ruff pytest pytest-asyncio -q
|
||||||
@@ -95,10 +89,10 @@ setup-python:
|
|||||||
}
|
}
|
||||||
Write-Host "Installing Python dependencies..."
|
Write-Host "Installing Python dependencies..."
|
||||||
& "{{ python }}" -m pip install --upgrade pip -q
|
& "{{ python }}" -m pip install --upgrade pip -q
|
||||||
$gpus = Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name; \
|
$gpus = Get-CimInstance Win32_VideoController | Select-Object -ExpandProperty Name
|
||||||
Write-Host "Detected GPUs: $($gpus -join ', ')"; \
|
Write-Host "Detected GPUs: $($gpus -join ', ')"
|
||||||
$hasNvidia = ($gpus | Where-Object { $_ -match 'NVIDIA' }).Count -gt 0; \
|
$hasNvidia = ($gpus | Where-Object { $_ -match 'NVIDIA' }).Count -gt 0
|
||||||
$hasIntelArc = ($gpus | Where-Object { $_ -match 'Arc' }).Count -gt 0; \
|
$hasIntelArc = ($gpus | Where-Object { $_ -match 'Arc' }).Count -gt 0
|
||||||
if ($hasNvidia) { \
|
if ($hasNvidia) { \
|
||||||
Write-Host "NVIDIA GPU detected — installing PyTorch with CUDA support..."; \
|
Write-Host "NVIDIA GPU detected — installing PyTorch with CUDA support..."; \
|
||||||
& "{{ pip }}" install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128; \
|
& "{{ pip }}" install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128; \
|
||||||
@@ -232,16 +226,12 @@ build-server: _ensure-venv
|
|||||||
build-server: _ensure-venv
|
build-server: _ensure-venv
|
||||||
$ErrorActionPreference = "Stop"; \
|
$ErrorActionPreference = "Stop"; \
|
||||||
$env:PATH = "{{ venv_bin }};$env:PATH"; \
|
$env:PATH = "{{ venv_bin }};$env:PATH"; \
|
||||||
$triple = (rustc --print host-tuple); \
|
|
||||||
New-Item -ItemType Directory -Path "{{ tauri_dir }}/src-tauri/binaries" -Force | Out-Null; \
|
|
||||||
& "{{ python }}" backend/build_binary.py; \
|
& "{{ python }}" backend/build_binary.py; \
|
||||||
if ($LASTEXITCODE -ne 0) { throw "build_binary.py failed with exit code $LASTEXITCODE" }; \
|
if ($LASTEXITCODE -ne 0) { throw "build_binary.py failed with exit code $LASTEXITCODE" }; \
|
||||||
|
$triple = (rustc --print host-tuple); \
|
||||||
|
New-Item -ItemType Directory -Path "{{ tauri_dir }}/src-tauri/binaries" -Force | Out-Null; \
|
||||||
Copy-Item "backend/dist/voicebox-server.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-server-$triple.exe" -Force; \
|
Copy-Item "backend/dist/voicebox-server.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-server-$triple.exe" -Force; \
|
||||||
Write-Host "Copied sidecar: voicebox-server-$triple.exe"; \
|
Write-Host "Copied sidecar: voicebox-server-$triple.exe"
|
||||||
& "{{ python }}" backend/build_binary.py --shim; \
|
|
||||||
if ($LASTEXITCODE -ne 0) { throw "build_binary.py --shim failed with exit code $LASTEXITCODE" }; \
|
|
||||||
Copy-Item "backend/dist/voicebox-mcp.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-mcp-$triple.exe" -Force; \
|
|
||||||
Write-Host "Copied sidecar: voicebox-mcp-$triple.exe"
|
|
||||||
|
|
||||||
# Build CUDA server binary and place in app data dir for local testing
|
# Build CUDA server binary and place in app data dir for local testing
|
||||||
[windows]
|
[windows]
|
||||||
|
|||||||
@@ -66,46 +66,13 @@ fn find_monitor_source_via_pactl() -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Select the capture device: prefer an exact match against the monitor
|
|
||||||
/// source name reported by `pactl`, then fall back to any device whose name
|
|
||||||
/// contains "monitor", then the host's default input device.
|
|
||||||
fn select_capture_device(host: &cpal::Host, monitor_source: Option<&str>) -> Option<cpal::Device> {
|
|
||||||
let devices: Vec<cpal::Device> = host.input_devices().ok()?.collect();
|
|
||||||
|
|
||||||
if let Some(target) = monitor_source {
|
|
||||||
if let Some(pos) = devices
|
|
||||||
.iter()
|
|
||||||
.position(|d| d.name().map(|n| n == target).unwrap_or(false))
|
|
||||||
{
|
|
||||||
eprintln!(
|
|
||||||
"Linux audio capture: Using pactl monitor device: {}",
|
|
||||||
target
|
|
||||||
);
|
|
||||||
return devices.into_iter().nth(pos);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pos) = devices.iter().position(|d| {
|
|
||||||
d.name()
|
|
||||||
.map(|n| n.to_lowercase().contains("monitor"))
|
|
||||||
.unwrap_or(false)
|
|
||||||
}) {
|
|
||||||
let name = devices[pos].name().unwrap_or_default();
|
|
||||||
eprintln!("Linux audio capture: Found monitor device by name: {}", name);
|
|
||||||
return devices.into_iter().nth(pos);
|
|
||||||
}
|
|
||||||
|
|
||||||
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
|
||||||
host.default_input_device()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Start capturing system audio on Linux using PulseAudio monitor sources.
|
/// Start capturing system audio on Linux using PulseAudio monitor sources.
|
||||||
///
|
///
|
||||||
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
|
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
|
||||||
/// monitor source via `pactl`, then select the matching cpal input device by
|
/// monitor source via `pactl` and set the `PULSE_SOURCE` environment variable.
|
||||||
/// name. This avoids mutating the process environment (`PULSE_SOURCE`), which
|
/// This tells PulseAudio's ALSA plugin to use the monitor as the default input
|
||||||
/// is not thread-safe and would affect every thread in the process. If `pactl`
|
/// source for this process. If `pactl` is unavailable, we fall back to searching
|
||||||
/// is unavailable, we fall back to searching cpal device names for "monitor".
|
/// cpal device names for "monitor".
|
||||||
pub async fn start_capture(
|
pub async fn start_capture(
|
||||||
state: &AudioCaptureState,
|
state: &AudioCaptureState,
|
||||||
max_duration_secs: u32,
|
max_duration_secs: u32,
|
||||||
@@ -134,10 +101,64 @@ pub async fn start_capture(
|
|||||||
|
|
||||||
// Spawn capture on a dedicated thread
|
// Spawn capture on a dedicated thread
|
||||||
thread::spawn(move || {
|
thread::spawn(move || {
|
||||||
let host = cpal::default_host();
|
// Try to set PULSE_SOURCE to a monitor before initializing cpal.
|
||||||
|
// This tells PulseAudio/PipeWire's ALSA plugin to use the monitor
|
||||||
|
// as the default input source for this process.
|
||||||
let monitor_source = find_monitor_source_via_pactl();
|
let monitor_source = find_monitor_source_via_pactl();
|
||||||
|
if let Some(ref source_name) = monitor_source {
|
||||||
|
eprintln!(
|
||||||
|
"Linux audio capture: Setting PULSE_SOURCE={}",
|
||||||
|
source_name
|
||||||
|
);
|
||||||
|
std::env::set_var("PULSE_SOURCE", source_name);
|
||||||
|
}
|
||||||
|
|
||||||
let device = match select_capture_device(&host, monitor_source.as_deref()) {
|
let host = cpal::default_host();
|
||||||
|
|
||||||
|
// Select the capture device.
|
||||||
|
// If PULSE_SOURCE was set, the default input device IS the monitor.
|
||||||
|
// Otherwise, fall back to searching device names for "monitor".
|
||||||
|
let device = if monitor_source.is_some() {
|
||||||
|
// PULSE_SOURCE was set — default input IS the monitor now
|
||||||
|
match host.default_input_device() {
|
||||||
|
Some(d) => {
|
||||||
|
let name = d.name().unwrap_or_default();
|
||||||
|
eprintln!(
|
||||||
|
"Linux audio capture: Using PULSE_SOURCE monitor device: {}",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
d
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
let error_msg = "No audio input device available".to_string();
|
||||||
|
eprintln!("{}", error_msg);
|
||||||
|
*error_arc.lock().unwrap() = Some(error_msg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// pactl not available — try to find monitor by name (original approach)
|
||||||
|
let mut monitor_device = None;
|
||||||
|
if let Ok(devices) = host.input_devices() {
|
||||||
|
for d in devices {
|
||||||
|
if let Ok(name) = d.name() {
|
||||||
|
let name_lower = name.to_lowercase();
|
||||||
|
if name_lower.contains("monitor") {
|
||||||
|
eprintln!(
|
||||||
|
"Linux audio capture: Found monitor device by name: {}",
|
||||||
|
name
|
||||||
|
);
|
||||||
|
monitor_device = Some(d);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match monitor_device {
|
||||||
|
Some(d) => d,
|
||||||
|
None => {
|
||||||
|
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
||||||
|
match host.default_input_device() {
|
||||||
Some(d) => d,
|
Some(d) => d,
|
||||||
None => {
|
None => {
|
||||||
let error_msg = "No audio input device available".to_string();
|
let error_msg = "No audio input device available".to_string();
|
||||||
@@ -145,6 +166,9 @@ pub async fn start_capture(
|
|||||||
*error_arc.lock().unwrap() = Some(error_msg);
|
*error_arc.lock().unwrap() = Some(error_msg);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
let device_name = device.name().unwrap_or_else(|_| "unknown".to_string());
|
||||||
|
|||||||
@@ -264,9 +264,6 @@ fn apply_effect(app: &AppHandle, effect: Effect) {
|
|||||||
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
|
let _ = window.set_position(tauri::PhysicalPosition::new(x, y));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Skip on Linux: aborts if the window was never realized
|
|
||||||
// (see show_dictate_window in main.rs).
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
|
||||||
let _ = window.set_ignore_cursor_events(false);
|
let _ = window.set_ignore_cursor_events(false);
|
||||||
// Deliberately no set_focus() — taking key focus would yank
|
// Deliberately no set_focus() — taking key focus would yank
|
||||||
// it out of whatever app the user was typing in, which is
|
// it out of whatever app the user was typing in, which is
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ pub fn key_from_str(name: &str) -> Option<Key> {
|
|||||||
"ShiftLeft" => Key::ShiftLeft,
|
"ShiftLeft" => Key::ShiftLeft,
|
||||||
"ShiftRight" => Key::ShiftRight,
|
"ShiftRight" => Key::ShiftRight,
|
||||||
"CapsLock" => Key::CapsLock,
|
"CapsLock" => Key::CapsLock,
|
||||||
"Function" => Key::Function,
|
|
||||||
|
|
||||||
// Whitespace / navigation
|
// Whitespace / navigation
|
||||||
"Space" => Key::Space,
|
"Space" => Key::Space,
|
||||||
|
|||||||
@@ -19,23 +19,19 @@
|
|||||||
//! regardless of the active layout — most Windows apps treat that as
|
//! regardless of the active layout — most Windows apps treat that as
|
||||||
//! Ctrl+V. AutoHotkey relies on the same behaviour.
|
//! Ctrl+V. AutoHotkey relies on the same behaviour.
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
use std::sync::atomic::{AtomicU16, Ordering};
|
use std::sync::atomic::{AtomicU16, Ordering};
|
||||||
|
|
||||||
/// `kVK_ANSI_V` — the keycode for the physical V key on a US QWERTY
|
/// `kVK_ANSI_V` — the keycode for the physical V key on a US QWERTY
|
||||||
/// layout. Used as the fallback whenever live resolution can't produce a
|
/// layout. Used as the fallback whenever live resolution can't produce a
|
||||||
/// better answer (no Unicode key layout data, lookup failure, non-macOS).
|
/// better answer (no Unicode key layout data, lookup failure, non-macOS).
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
const FALLBACK_V_KEYCODE: u16 = 9;
|
const FALLBACK_V_KEYCODE: u16 = 9;
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
static V_KEYCODE: AtomicU16 = AtomicU16::new(FALLBACK_V_KEYCODE);
|
static V_KEYCODE: AtomicU16 = AtomicU16::new(FALLBACK_V_KEYCODE);
|
||||||
|
|
||||||
/// Returns the keycode whose current-layout translation is `'v'`. Falls
|
/// Returns the keycode whose current-layout translation is `'v'`. Falls
|
||||||
/// back to `kVK_ANSI_V` when resolution hasn't run, the active input
|
/// back to `kVK_ANSI_V` when resolution hasn't run, the active input
|
||||||
/// source carries no Unicode key layout data, or no keycode in the layout
|
/// source carries no Unicode key layout data, or no keycode in the layout
|
||||||
/// produces `v`.
|
/// produces `v`.
|
||||||
#[cfg(target_os = "macos")]
|
|
||||||
pub fn paste_keycode_v() -> u16 {
|
pub fn paste_keycode_v() -> u16 {
|
||||||
V_KEYCODE.load(Ordering::Relaxed)
|
V_KEYCODE.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,10 +112,6 @@ pub fn show_dictate_window(app: &tauri::AppHandle) {
|
|||||||
let _ = window.set_position(PhysicalPosition::new(x, y));
|
let _ = window.set_position(PhysicalPosition::new(x, y));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Skip on Linux: tao's CursorIgnoreEvents handler unwraps the GdkWindow,
|
|
||||||
// which is None until the window is first shown, aborting the process.
|
|
||||||
// The click-through toggle is a macOS workaround and is never set on Linux.
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
|
||||||
let _ = window.set_ignore_cursor_events(false);
|
let _ = window.set_ignore_cursor_events(false);
|
||||||
let _ = window.show();
|
let _ = window.show();
|
||||||
}
|
}
|
||||||
@@ -1425,9 +1421,6 @@ pub fn run() {
|
|||||||
let handle_for_hide = app.handle().clone();
|
let handle_for_hide = app.handle().clone();
|
||||||
app.handle().listen("dictate:hide", move |_event| {
|
app.handle().listen("dictate:hide", move |_event| {
|
||||||
if let Some(window) = handle_for_hide.get_webview_window(DICTATE_WINDOW_LABEL) {
|
if let Some(window) = handle_for_hide.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||||
// Skip on Linux: aborts if the window was never realized
|
|
||||||
// (see show_dictate_window).
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
|
||||||
let _ = window.set_ignore_cursor_events(true);
|
let _ = window.set_ignore_cursor_events(true);
|
||||||
let _ = window.set_position(PhysicalPosition::new(-10_000, -10_000));
|
let _ = window.set_position(PhysicalPosition::new(-10_000, -10_000));
|
||||||
let _ = window.hide();
|
let _ = window.hide();
|
||||||
|
|||||||
@@ -4,14 +4,10 @@
|
|||||||
//! pipeline so the focused app performs its native paste action against
|
//! pipeline so the focused app performs its native paste action against
|
||||||
//! whatever the clipboard module has just staged.
|
//! whatever the clipboard module has just staged.
|
||||||
//!
|
//!
|
||||||
//! - **macOS** — Cmd down with Cmd flag, V down with Cmd flag, V up with
|
//! - **macOS** — Cmd down, V down with Cmd flag, V up with Cmd flag, Cmd
|
||||||
//! Cmd flag, Cmd up via `CGEventPost` at `kCGHIDEventTap`. The Cmd-down
|
//! up via `CGEventPost` at `kCGHIDEventTap`. Accessibility permission is
|
||||||
//! event carries the Command flag so its `flagsChanged` representation
|
//! load-bearing: without it the system swallows the events silently, so
|
||||||
//! matches hardware — Electron/Chromium tracks modifier state from that
|
//! callers must gate on [`crate::accessibility::is_trusted`].
|
||||||
//! flag and drops the paste otherwise (see the note on the event table).
|
|
||||||
//! Accessibility permission is load-bearing: without it the system
|
|
||||||
//! swallows the events silently, so callers must gate on
|
|
||||||
//! [`crate::accessibility::is_trusted`].
|
|
||||||
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
|
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
|
||||||
//! permission gate, but UAC/UIPI blocks delivery into elevated target
|
//! permission gate, but UAC/UIPI blocks delivery into elevated target
|
||||||
//! windows when we run non-elevated — nothing we can do short of also
|
//! windows when we run non-elevated — nothing we can do short of also
|
||||||
@@ -105,18 +101,7 @@ pub fn send_paste() -> Result<(), String> {
|
|||||||
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
||||||
|
|
||||||
let events = [
|
let events = [
|
||||||
// The Cmd-down event must carry the Command flag itself. On real
|
(KEYCODE_LEFT_CMD, true, 0),
|
||||||
// hardware the Cmd keyDown is a flagsChanged event whose flags
|
|
||||||
// already include Command; Chromium/Electron builds its tracked
|
|
||||||
// modifier state from that flag. Posting Cmd-down with flags = 0
|
|
||||||
// leaves that tracker showing "Command up", so the following V —
|
|
||||||
// even though its own flags carry Command — matches neither the
|
|
||||||
// Cmd+V accelerator (tracker says no modifier) nor plain-text
|
|
||||||
// insertion (event flags say Command), and Electron drops it
|
|
||||||
// silently. AppKit reads the V event's own flags and pastes
|
|
||||||
// regardless, which is why native apps worked but Electron
|
|
||||||
// targets (Slack, VS Code) silently no-op'd.
|
|
||||||
(KEYCODE_LEFT_CMD, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
|
||||||
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||||
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
|
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||||
(KEYCODE_LEFT_CMD, false, 0),
|
(KEYCODE_LEFT_CMD, false, 0),
|
||||||
|
|||||||
Reference in New Issue
Block a user