feat: add Qwen CustomVoice preset engine

This commit is contained in:
James Pine
2026-03-19 19:48:50 -07:00
parent d70b878b71
commit 4e0c731db8
14 changed files with 399 additions and 21 deletions
@@ -19,6 +19,8 @@ import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm';
const ENGINE_OPTIONS = [ const ENGINE_OPTIONS = [
{ value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' }, { value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' },
{ value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' }, { value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' },
{ value: 'qwen_custom_voice:1.7B', label: 'Qwen CustomVoice 1.7B', engine: 'qwen_custom_voice' },
{ value: 'qwen_custom_voice:0.6B', label: 'Qwen CustomVoice 0.6B', engine: 'qwen_custom_voice' },
{ value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' }, { value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' },
{ value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' }, { value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' },
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' }, { value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' },
@@ -29,6 +31,7 @@ const ENGINE_OPTIONS = [
const ENGINE_DESCRIPTIONS: Record<string, string> = { const ENGINE_DESCRIPTIONS: Record<string, string> = {
qwen: 'Multi-language, two sizes', qwen: 'Multi-language, two sizes',
qwen_custom_voice: '9 preset voices, instruct control',
luxtts: 'Fast, English-focused', luxtts: 'Fast, English-focused',
chatterbox: '23 languages, incl. Hebrew', chatterbox: '23 languages, incl. Hebrew',
chatterbox_turbo: 'English, [laugh] [cough] tags', chatterbox_turbo: 'English, [laugh] [cough] tags',
@@ -49,12 +52,22 @@ function getAvailableOptions(selectedProfile?: VoiceProfileResponse | null) {
function getSelectValue(engine: string, modelSize?: string): string { function getSelectValue(engine: string, modelSize?: string): string {
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`; if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
if (engine === 'qwen_custom_voice') return `qwen_custom_voice:${modelSize || '1.7B'}`;
if (engine === 'tada') return `tada:${modelSize || '1B'}`; if (engine === 'tada') return `tada:${modelSize || '1B'}`;
return engine; return engine;
} }
function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) { function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) {
if (value.startsWith('qwen:')) { if (value.startsWith('qwen_custom_voice:')) {
const [, modelSize] = value.split(':');
form.setValue('engine', 'qwen_custom_voice');
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
const currentLang = form.getValues('language');
const available = getLanguageOptionsForEngine('qwen_custom_voice');
if (!available.some((l) => l.value === currentLang)) {
form.setValue('language', available[0]?.value ?? 'en');
}
} else if (value.startsWith('qwen:')) {
const [, modelSize] = value.split(':'); const [, modelSize] = value.split(':');
form.setValue('engine', 'qwen'); form.setValue('engine', 'qwen');
form.setValue('modelSize', modelSize as '1.7B' | '0.6B'); form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
@@ -91,7 +91,7 @@ export function GenerationForm() {
)} )}
/> />
{form.watch('engine') === 'qwen' && ( {(form.watch('engine') === 'qwen' || form.watch('engine') === 'qwen_custom_voice') && (
<FormField <FormField
control={form.control} control={form.control}
name="instruct" name="instruct"
@@ -68,6 +68,10 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.', 'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.',
kokoro: kokoro:
'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.', 'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.',
'qwen-custom-voice-1.7B':
'Qwen3-TTS CustomVoice 1.7B by Alibaba. 9 premium preset voices with instruct-based style control for tone, emotion, and prosody. Supports 10 languages.',
'qwen-custom-voice-0.6B':
'Qwen3-TTS CustomVoice 0.6B by Alibaba. Lightweight version with the same 9 preset voices and instruct control. Faster inference for lower-end hardware.',
'whisper-base': 'whisper-base':
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.', 'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
'whisper-small': 'whisper-small':
@@ -396,6 +400,7 @@ export function ModelManagement() {
modelStatus?.models.filter( modelStatus?.models.filter(
(m) => (m) =>
m.model_name.startsWith('qwen-tts') || m.model_name.startsWith('qwen-tts') ||
m.model_name.startsWith('qwen-custom-voice') ||
m.model_name.startsWith('luxtts') || m.model_name.startsWith('luxtts') ||
m.model_name.startsWith('chatterbox') || m.model_name.startsWith('chatterbox') ||
m.model_name.startsWith('tada') || m.model_name.startsWith('tada') ||
@@ -17,6 +17,12 @@ import { useDeleteProfile, useExportProfile } from '@/lib/hooks/useProfiles';
import { cn } from '@/lib/utils/cn'; import { cn } from '@/lib/utils/cn';
import { useUIStore } from '@/stores/uiStore'; import { useUIStore } from '@/stores/uiStore';
/** Human-readable display names for preset engine badges. */
const ENGINE_DISPLAY_NAMES: Record<string, string> = {
kokoro: 'Kokoro',
qwen_custom_voice: 'CustomVoice',
};
interface ProfileCardProps { interface ProfileCardProps {
profile: VoiceProfileResponse; profile: VoiceProfileResponse;
} }
@@ -99,7 +105,7 @@ export function ProfileCard({ profile }: ProfileCardProps) {
</Badge> </Badge>
{profile.voice_type === 'preset' && ( {profile.voice_type === 'preset' && (
<Badge variant="secondary" className="text-xs h-5 px-1.5"> <Badge variant="secondary" className="text-xs h-5 px-1.5">
{profile.preset_engine} {ENGINE_DISPLAY_NAMES[profile.preset_engine ?? ''] ?? profile.preset_engine}
</Badge> </Badge>
)} )}
{profile.voice_type === 'designed' && ( {profile.voice_type === 'designed' && (
@@ -60,9 +60,10 @@ import { AudioSampleUpload } from './AudioSampleUpload';
import { SampleList } from './SampleList'; import { SampleList } from './SampleList';
const MAX_AUDIO_DURATION_SECONDS = 30; const MAX_AUDIO_DURATION_SECONDS = 30;
const PRESET_ONLY_ENGINES = new Set(['kokoro']); const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
const DEFAULT_ENGINE_OPTIONS = [ const DEFAULT_ENGINE_OPTIONS = [
{ value: 'qwen', label: 'Qwen3-TTS' }, { value: 'qwen', label: 'Qwen3-TTS' },
{ value: 'qwen_custom_voice', label: 'Qwen CustomVoice' },
{ value: 'luxtts', label: 'LuxTTS' }, { value: 'luxtts', label: 'LuxTTS' },
{ value: 'chatterbox', label: 'Chatterbox' }, { value: 'chatterbox', label: 'Chatterbox' },
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' }, { value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
@@ -849,6 +850,7 @@ export function ProfileForm() {
</FormControl> </FormControl>
<SelectContent> <SelectContent>
<SelectItem value="kokoro">Kokoro 82M</SelectItem> <SelectItem value="kokoro">Kokoro 82M</SelectItem>
<SelectItem value="qwen_custom_voice">Qwen CustomVoice</SelectItem>
</SelectContent> </SelectContent>
</Select> </Select>
</FormItem> </FormItem>
@@ -7,11 +7,12 @@ import { ProfileCard } from './ProfileCard';
import { ProfileForm } from './ProfileForm'; import { ProfileForm } from './ProfileForm';
/** Engines that use preset (built-in) voices instead of cloned profiles. */ /** Engines that use preset (built-in) voices instead of cloned profiles. */
const PRESET_ENGINES = new Set(['kokoro']); const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
/** Human-readable engine names for empty state messages. */ /** Human-readable engine names for empty state messages. */
const ENGINE_NAMES: Record<string, string> = { const ENGINE_NAMES: Record<string, string> = {
kokoro: 'Kokoro', kokoro: 'Kokoro',
qwen_custom_voice: 'Qwen CustomVoice',
}; };
export function ProfileList() { export function ProfileList() {
+8 -1
View File
@@ -62,7 +62,14 @@ export interface GenerationRequest {
language: LanguageCode; language: LanguageCode;
seed?: number; seed?: number;
model_size?: '1.7B' | '0.6B' | '1B' | '3B'; model_size?: '1.7B' | '0.6B' | '1B' | '3B';
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro'; engine?:
| 'qwen'
| 'qwen_custom_voice'
| 'luxtts'
| 'chatterbox'
| 'chatterbox_turbo'
| 'tada'
| 'kokoro';
instruct?: string; instruct?: string;
max_chunk_chars?: number; max_chunk_chars?: number;
crossfade_ms?: number; crossfade_ms?: number;
+1
View File
@@ -69,6 +69,7 @@ export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
chatterbox_turbo: ['en'], chatterbox_turbo: ['en'],
tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'], tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'],
kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'], kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'],
qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'],
} as const; } as const;
/** Helper: get language options for a given engine. */ /** Helper: get language options for a given engine. */
+25 -7
View File
@@ -17,7 +17,17 @@ const generationSchema = z.object({
seed: z.number().int().optional(), seed: z.number().int().optional(),
modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(), modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(),
instruct: z.string().max(500).optional(), instruct: z.string().max(500).optional(),
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada', 'kokoro']).optional(), engine: z
.enum([
'qwen',
'qwen_custom_voice',
'luxtts',
'chatterbox',
'chatterbox_turbo',
'tada',
'kokoro',
])
.optional(),
}); });
export type GenerationFormValues = z.infer<typeof generationSchema>; export type GenerationFormValues = z.infer<typeof generationSchema>;
@@ -85,7 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
: 'tada-1b' : 'tada-1b'
: engine === 'kokoro' : engine === 'kokoro'
? 'kokoro' ? 'kokoro'
: `qwen-tts-${data.modelSize}`; : engine === 'qwen_custom_voice'
? `qwen-custom-voice-${data.modelSize}`
: `qwen-tts-${data.modelSize}`;
const displayName = const displayName =
engine === 'luxtts' engine === 'luxtts'
? 'LuxTTS' ? 'LuxTTS'
@@ -99,9 +111,13 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
: 'TADA 1B' : 'TADA 1B'
: engine === 'kokoro' : engine === 'kokoro'
? 'Kokoro 82M' ? 'Kokoro 82M'
: data.modelSize === '1.7B' : engine === 'qwen_custom_voice'
? 'Qwen TTS 1.7B' ? data.modelSize === '1.7B'
: 'Qwen TTS 0.6B'; ? 'Qwen CustomVoice 1.7B'
: 'Qwen CustomVoice 0.6B'
: data.modelSize === '1.7B'
? 'Qwen TTS 1.7B'
: 'Qwen TTS 0.6B';
// Check if model needs downloading // Check if model needs downloading
try { try {
@@ -116,7 +132,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
console.error('Failed to check model status:', error); console.error('Failed to check model status:', error);
} }
const hasModelSizes = engine === 'qwen' || engine === 'tada'; const hasModelSizes =
engine === 'qwen' || engine === 'qwen_custom_voice' || engine === 'tada';
const supportsInstruct = engine === 'qwen' || engine === 'qwen_custom_voice';
const effectsChain = options.getEffectsChain?.(); const effectsChain = options.getEffectsChain?.();
// This now returns immediately with status="generating" // This now returns immediately with status="generating"
const result = await generation.mutateAsync({ const result = await generation.mutateAsync({
@@ -126,7 +144,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
seed: data.seed, seed: data.seed,
model_size: hasModelSizes ? data.modelSize : undefined, model_size: hasModelSizes ? data.modelSize : undefined,
engine, engine,
instruct: engine === 'qwen' ? data.instruct || undefined : undefined, instruct: supportsInstruct ? data.instruct || undefined : undefined,
max_chunk_chars: maxChunkChars, max_chunk_chars: maxChunkChars,
crossfade_ms: crossfadeMs, crossfade_ms: crossfadeMs,
normalize: normalizeAudio, normalize: normalizeAudio,
+51 -4
View File
@@ -163,6 +163,7 @@ _stt_backend: Optional[STTBackend] = None
# The factory function uses this for the if/elif chain; the model configs live on the backend classes. # The factory function uses this for the if/elif chain; the model configs live on the backend classes.
TTS_ENGINES = { TTS_ENGINES = {
"qwen": "Qwen TTS", "qwen": "Qwen TTS",
"qwen_custom_voice": "Qwen CustomVoice",
"luxtts": "LuxTTS", "luxtts": "LuxTTS",
"chatterbox": "Chatterbox TTS", "chatterbox": "Chatterbox TTS",
"chatterbox_turbo": "Chatterbox Turbo", "chatterbox_turbo": "Chatterbox Turbo",
@@ -205,6 +206,32 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
] ]
def _get_qwen_custom_voice_configs() -> list[ModelConfig]:
"""Return Qwen CustomVoice model configs."""
return [
ModelConfig(
model_name="qwen-custom-voice-1.7B",
display_name="Qwen CustomVoice 1.7B",
engine="qwen_custom_voice",
hf_repo_id="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
model_size="1.7B",
size_mb=3500,
supports_instruct=True,
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
),
ModelConfig(
model_name="qwen-custom-voice-0.6B",
display_name="Qwen CustomVoice 0.6B",
engine="qwen_custom_voice",
hf_repo_id="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
model_size="0.6B",
size_mb=1200,
supports_instruct=True,
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
),
]
def _get_non_qwen_tts_configs() -> list[ModelConfig]: def _get_non_qwen_tts_configs() -> list[ModelConfig]:
"""Return model configs for non-Qwen TTS engines. """Return model configs for non-Qwen TTS engines.
@@ -333,12 +360,12 @@ def _get_whisper_configs() -> list[ModelConfig]:
def get_all_model_configs() -> list[ModelConfig]: def get_all_model_configs() -> list[ModelConfig]:
"""Return the full list of model configs (TTS + STT).""" """Return the full list of model configs (TTS + STT)."""
return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs() return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
def get_tts_model_configs() -> list[ModelConfig]: def get_tts_model_configs() -> list[ModelConfig]:
"""Return only TTS model configs.""" """Return only TTS model configs."""
return _get_qwen_model_configs() + _get_non_qwen_tts_configs() return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs()
# Lookup helpers — these replace the if/elif chains in main.py # Lookup helpers — these replace the if/elif chains in main.py
@@ -369,7 +396,7 @@ def engine_has_model_sizes(engine: str) -> bool:
async def load_engine_model(engine: str, model_size: str = "default") -> None: async def load_engine_model(engine: str, model_size: str = "default") -> None:
"""Load a model for the given engine, handling engines with multiple model sizes.""" """Load a model for the given engine, handling engines with multiple model sizes."""
backend = get_tts_backend_for_engine(engine) backend = get_tts_backend_for_engine(engine)
if engine == "qwen": if engine in ("qwen", "qwen_custom_voice"):
await backend.load_model_async(model_size) await backend.load_model_async(model_size)
elif engine == "tada": elif engine == "tada":
await backend.load_model(model_size) await backend.load_model(model_size)
@@ -388,7 +415,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default")
cfg = c cfg = c
break break
if engine in ("qwen", "tada"): if engine in ("qwen", "qwen_custom_voice", "tada"):
if not backend._is_model_cached(model_size): if not backend._is_model_cached(model_size):
raise HTTPException( raise HTTPException(
status_code=400, status_code=400,
@@ -423,6 +450,14 @@ def unload_model_by_config(config: ModelConfig) -> bool:
return True return True
return False return False
if config.engine == "qwen_custom_voice":
backend = get_tts_backend_for_engine(config.engine)
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
if backend.is_loaded() and loaded_size == config.model_size:
backend.unload_model()
return True
return False
# All other TTS engines # All other TTS engines
backend = get_tts_backend_for_engine(config.engine) backend = get_tts_backend_for_engine(config.engine)
if backend.is_loaded(): if backend.is_loaded():
@@ -446,6 +481,11 @@ def check_model_loaded(config: ModelConfig) -> bool:
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None) loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
return tts_model.is_loaded() and loaded_size == config.model_size return tts_model.is_loaded() and loaded_size == config.model_size
if config.engine == "qwen_custom_voice":
backend = get_tts_backend_for_engine(config.engine)
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
return backend.is_loaded() and loaded_size == config.model_size
backend = get_tts_backend_for_engine(config.engine) backend = get_tts_backend_for_engine(config.engine)
return backend.is_loaded() return backend.is_loaded()
except Exception: except Exception:
@@ -463,6 +503,9 @@ def get_model_load_func(config: ModelConfig):
if config.engine == "qwen": if config.engine == "qwen":
return lambda: tts.get_tts_model().load_model(config.model_size) return lambda: tts.get_tts_model().load_model(config.model_size)
if config.engine == "qwen_custom_voice":
return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size)
return lambda: get_tts_backend_for_engine(config.engine).load_model() return lambda: get_tts_backend_for_engine(config.engine).load_model()
@@ -528,6 +571,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
from .kokoro_backend import KokoroTTSBackend from .kokoro_backend import KokoroTTSBackend
backend = KokoroTTSBackend() backend = KokoroTTSBackend()
elif engine == "qwen_custom_voice":
from .qwen_custom_voice_backend import QwenCustomVoiceBackend
backend = QwenCustomVoiceBackend()
else: else:
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}") raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
@@ -0,0 +1,210 @@
"""
Qwen3-TTS CustomVoice backend implementation.
Wraps the Qwen3-TTS-12Hz CustomVoice model for preset-speaker TTS with
instruction-based style control. Uses the same qwen_tts library as the
Base model (pytorch_backend.py) but loads a different checkpoint and
calls generate_custom_voice() instead of generate_voice_clone().
Key differences from the Base engine:
- Uses preset speakers (9 built-in voices) instead of zero-shot cloning
- Supports instruct parameter for tone/emotion/prosody control
- Two model sizes: 1.7B and 0.6B
Languages supported: zh, en, ja, ko, de, fr, ru, pt, es, it
"""
import asyncio
import logging
from typing import Optional
import numpy as np
import torch
from . import TTSBackend, LANGUAGE_CODE_TO_NAME
from .base import (
is_model_cached,
get_torch_device,
combine_voice_prompts as _combine_voice_prompts,
model_load_progress,
)
logger = logging.getLogger(__name__)
# ── Preset speakers ──────────────────────────────────────────────────
# (speaker_id, display_name, gender, native_language_code, description)
QWEN_CUSTOM_VOICES = [
("Vivian", "Vivian", "female", "zh", "Bright, slightly edgy young female voice"),
("Serena", "Serena", "female", "zh", "Warm, gentle young female voice"),
("Uncle_Fu", "Uncle Fu", "male", "zh", "Seasoned male voice with a low, mellow timbre"),
("Dylan", "Dylan", "male", "zh", "Youthful Beijing male voice with a clear, natural timbre"),
("Eric", "Eric", "male", "zh", "Lively Chengdu male voice with a slightly husky brightness"),
("Ryan", "Ryan", "male", "en", "Dynamic male voice with strong rhythmic drive"),
("Aiden", "Aiden", "male", "en", "Sunny American male voice with a clear midrange"),
("Ono_Anna", "Ono Anna", "female", "ja", "Playful Japanese female voice with a light, nimble timbre"),
("Sohee", "Sohee", "female", "ko", "Warm Korean female voice with rich emotion"),
]
QWEN_CV_DEFAULT_SPEAKER = "Ryan"
# HuggingFace repo IDs per model size
QWEN_CV_HF_REPOS = {
"1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
"0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
}
class QwenCustomVoiceBackend:
"""Qwen3-TTS CustomVoice backend — preset speakers with instruct control."""
def __init__(self, model_size: str = "1.7B"):
self.model = None
self.model_size = model_size
self.device = self._get_device()
self._current_model_size: Optional[str] = None
def _get_device(self) -> str:
return get_torch_device(allow_xpu=True, allow_directml=True)
def is_loaded(self) -> bool:
return self.model is not None
def _get_model_path(self, model_size: str) -> str:
if model_size not in QWEN_CV_HF_REPOS:
raise ValueError(f"Unknown model size: {model_size}")
return QWEN_CV_HF_REPOS[model_size]
def _is_model_cached(self, model_size: Optional[str] = None) -> bool:
size = model_size or self.model_size
return is_model_cached(self._get_model_path(size))
async def load_model_async(self, model_size: Optional[str] = None) -> None:
if model_size is None:
model_size = self.model_size
if self.model is not None and self._current_model_size == model_size:
return
if self.model is not None and self._current_model_size != model_size:
self.unload_model()
await asyncio.to_thread(self._load_model_sync, model_size)
# Alias for compatibility with the TTSBackend protocol
load_model = load_model_async
def _load_model_sync(self, model_size: str) -> None:
model_name = f"qwen-custom-voice-{model_size}"
is_cached = self._is_model_cached(model_size)
with model_load_progress(model_name, is_cached):
from qwen_tts import Qwen3TTSModel
model_path = self._get_model_path(model_size)
logger.info("Loading Qwen CustomVoice %s on %s...", model_size, self.device)
if self.device == "cpu":
self.model = Qwen3TTSModel.from_pretrained(
model_path,
torch_dtype=torch.float32,
low_cpu_mem_usage=False,
)
else:
self.model = Qwen3TTSModel.from_pretrained(
model_path,
device_map=self.device,
torch_dtype=torch.bfloat16,
)
self._current_model_size = model_size
self.model_size = model_size
logger.info("Qwen CustomVoice %s loaded successfully", model_size)
def unload_model(self) -> None:
if self.model is not None:
del self.model
self.model = None
self._current_model_size = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
logger.info("Qwen CustomVoice unloaded")
async def create_voice_prompt(
self,
audio_path: str,
reference_text: str,
use_cache: bool = True,
) -> tuple[dict, bool]:
"""
Create voice prompt for CustomVoice.
CustomVoice doesn't use reference audio — it uses preset speakers.
When called for a cloned profile (fallback), uses the default speaker.
For preset profiles, the voice_prompt dict is built by the profile
service and bypasses this method entirely.
"""
return {
"voice_type": "preset",
"preset_engine": "qwen_custom_voice",
"preset_voice_id": QWEN_CV_DEFAULT_SPEAKER,
}, False
async def combine_voice_prompts(
self,
audio_paths: list[str],
reference_texts: list[str],
) -> tuple[np.ndarray, str]:
return await _combine_voice_prompts(audio_paths, reference_texts)
async def generate(
self,
text: str,
voice_prompt: dict,
language: str = "en",
seed: Optional[int] = None,
instruct: Optional[str] = None,
) -> tuple[np.ndarray, int]:
"""
Generate audio using Qwen CustomVoice.
Args:
text: Text to synthesize
voice_prompt: Dict with preset_voice_id (speaker name)
language: Language code (zh, en, ja, ko, etc.)
seed: Random seed for reproducibility
instruct: Natural language instruction for style control
(e.g. "Speak in an angry tone", "Very happy")
Returns:
Tuple of (audio_array, sample_rate)
"""
await self.load_model_async(None)
speaker = voice_prompt.get("preset_voice_id") or QWEN_CV_DEFAULT_SPEAKER
def _generate_sync():
if seed is not None:
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
lang_name = LANGUAGE_CODE_TO_NAME.get(language, "auto")
kwargs = {
"text": text,
"language": lang_name.capitalize() if lang_name != "auto" else "Auto",
"speaker": speaker,
}
# Only pass instruct if non-empty
if instruct:
kwargs["instruct"] = instruct
wavs, sample_rate = self.model.generate_custom_voice(**kwargs)
return wavs[0], sample_rate
audio, sample_rate = await asyncio.to_thread(_generate_sync)
return audio, sample_rate
+2
View File
@@ -86,6 +86,8 @@ def build_server(cuda=False):
"--hidden-import", "--hidden-import",
"backend.backends.pytorch_backend", "backend.backends.pytorch_backend",
"--hidden-import", "--hidden-import",
"backend.backends.qwen_custom_voice_backend",
"--hidden-import",
"backend.utils.audio", "backend.utils.audio",
"--hidden-import", "--hidden-import",
"backend.utils.cache", "backend.utils.cache",
+1 -1
View File
@@ -78,7 +78,7 @@ class GenerationRequest(BaseModel):
seed: Optional[int] = Field(None, ge=0) seed: Optional[int] = Field(None, ge=0)
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$")
instruct: Optional[str] = Field(None, max_length=500) instruct: Optional[str] = Field(None, max_length=500)
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$") engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$")
max_chunk_chars: int = Field( max_chunk_chars: int = Field(
default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting" default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting"
) )
+69 -3
View File
@@ -90,6 +90,21 @@ async def list_preset_voices(engine: str):
for vid, name, gender, lang in KOKORO_VOICES for vid, name, gender, lang in KOKORO_VOICES
], ],
} }
if engine == "qwen_custom_voice":
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
return {
"engine": engine,
"voices": [
{
"voice_id": speaker_id,
"name": display_name,
"gender": gender,
"language": lang,
}
for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES
],
}
return {"engine": engine, "voices": []} return {"engine": engine, "voices": []}
@@ -103,9 +118,15 @@ async def seed_preset_profiles_route(
Creates profiles for all available preset voices that don't already exist. Creates profiles for all available preset voices that don't already exist.
Returns the count of newly created profiles. Returns the count of newly created profiles.
""" """
if engine != "kokoro": if engine == "kokoro":
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}") return _seed_kokoro_presets(db)
if engine == "qwen_custom_voice":
return _seed_qwen_custom_voice_presets(db)
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}")
def _seed_kokoro_presets(db: Session):
"""Seed Kokoro preset profiles."""
try: try:
from ..backends.kokoro_backend import KOKORO_VOICES from ..backends.kokoro_backend import KOKORO_VOICES
@@ -154,12 +175,57 @@ async def seed_preset_profiles_route(
db.commit() db.commit()
logger.info(f"Seeded {created} Kokoro preset profiles") logger.info(f"Seeded {created} Kokoro preset profiles")
return {"engine": engine, "created": created, "total_available": len(KOKORO_VOICES)} return {"engine": "kokoro", "created": created, "total_available": len(KOKORO_VOICES)}
except Exception as e: except Exception as e:
logger.exception(f"Failed to seed Kokoro profiles: {e}") logger.exception(f"Failed to seed Kokoro profiles: {e}")
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(e))
def _seed_qwen_custom_voice_presets(db: Session):
"""Seed Qwen CustomVoice preset profiles."""
try:
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
created = 0
for speaker_id, display_name, gender, lang, description in QWEN_CUSTOM_VOICES:
# Skip if preset already exists
existing = (
db.query(DBVoiceProfile)
.filter_by(preset_engine="qwen_custom_voice", preset_voice_id=speaker_id)
.first()
)
if existing:
continue
# Skip name collisions
if db.query(DBVoiceProfile).filter_by(name=display_name).first():
continue
profile = DBVoiceProfile(
id=str(uuid.uuid4()),
name=display_name,
description=f"Qwen CustomVoice — {description}",
language=lang,
voice_type="preset",
preset_engine="qwen_custom_voice",
preset_voice_id=speaker_id,
default_engine="qwen_custom_voice",
created_at=datetime.utcnow(),
updated_at=datetime.utcnow(),
)
db.add(profile)
created += 1
if created > 0:
db.commit()
logger.info(f"Seeded {created} Qwen CustomVoice preset profiles")
return {"engine": "qwen_custom_voice", "created": created, "total_available": len(QWEN_CUSTOM_VOICES)}
except Exception as e:
logger.exception(f"Failed to seed Qwen CustomVoice profiles: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse) @router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
async def get_profile( async def get_profile(
profile_id: str, profile_id: str,