From 4e0c731db85b1de355cef9fa45d386f0e69509bd Mon Sep 17 00:00:00 2001 From: James Pine Date: Thu, 19 Mar 2026 18:31:30 -0700 Subject: [PATCH] feat: add Qwen CustomVoice preset engine --- .../Generation/EngineModelSelector.tsx | 15 +- .../components/Generation/GenerationForm.tsx | 2 +- .../ServerSettings/ModelManagement.tsx | 5 + .../components/VoiceProfiles/ProfileCard.tsx | 8 +- .../components/VoiceProfiles/ProfileForm.tsx | 4 +- .../components/VoiceProfiles/ProfileList.tsx | 3 +- app/src/lib/api/types.ts | 9 +- app/src/lib/constants/languages.ts | 1 + app/src/lib/hooks/useGenerationForm.ts | 32 ++- backend/backends/__init__.py | 55 ++++- backend/backends/qwen_custom_voice_backend.py | 210 ++++++++++++++++++ backend/build_binary.py | 2 + backend/models.py | 2 +- backend/routes/profiles.py | 72 +++++- 14 files changed, 399 insertions(+), 21 deletions(-) create mode 100644 backend/backends/qwen_custom_voice_backend.py diff --git a/app/src/components/Generation/EngineModelSelector.tsx b/app/src/components/Generation/EngineModelSelector.tsx index 80fd82af..7195a287 100644 --- a/app/src/components/Generation/EngineModelSelector.tsx +++ b/app/src/components/Generation/EngineModelSelector.tsx @@ -19,6 +19,8 @@ import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm'; const ENGINE_OPTIONS = [ { value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' }, { value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' }, + { value: 'qwen_custom_voice:1.7B', label: 'Qwen CustomVoice 1.7B', engine: 'qwen_custom_voice' }, + { value: 'qwen_custom_voice:0.6B', label: 'Qwen CustomVoice 0.6B', engine: 'qwen_custom_voice' }, { value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' }, { value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' }, { value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' }, @@ -29,6 +31,7 @@ const ENGINE_OPTIONS = [ const ENGINE_DESCRIPTIONS: Record = { qwen: 'Multi-language, two sizes', + qwen_custom_voice: '9 preset voices, instruct control', luxtts: 'Fast, English-focused', chatterbox: '23 languages, incl. Hebrew', chatterbox_turbo: 'English, [laugh] [cough] tags', @@ -49,12 +52,22 @@ function getAvailableOptions(selectedProfile?: VoiceProfileResponse | null) { function getSelectValue(engine: string, modelSize?: string): string { if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`; + if (engine === 'qwen_custom_voice') return `qwen_custom_voice:${modelSize || '1.7B'}`; if (engine === 'tada') return `tada:${modelSize || '1B'}`; return engine; } function handleEngineChange(form: UseFormReturn, value: string) { - if (value.startsWith('qwen:')) { + if (value.startsWith('qwen_custom_voice:')) { + const [, modelSize] = value.split(':'); + form.setValue('engine', 'qwen_custom_voice'); + form.setValue('modelSize', modelSize as '1.7B' | '0.6B'); + const currentLang = form.getValues('language'); + const available = getLanguageOptionsForEngine('qwen_custom_voice'); + if (!available.some((l) => l.value === currentLang)) { + form.setValue('language', available[0]?.value ?? 'en'); + } + } else if (value.startsWith('qwen:')) { const [, modelSize] = value.split(':'); form.setValue('engine', 'qwen'); form.setValue('modelSize', modelSize as '1.7B' | '0.6B'); diff --git a/app/src/components/Generation/GenerationForm.tsx b/app/src/components/Generation/GenerationForm.tsx index 9f7a7cd7..1195e8b1 100644 --- a/app/src/components/Generation/GenerationForm.tsx +++ b/app/src/components/Generation/GenerationForm.tsx @@ -91,7 +91,7 @@ export function GenerationForm() { )} /> - {form.watch('engine') === 'qwen' && ( + {(form.watch('engine') === 'qwen' || form.watch('engine') === 'qwen_custom_voice') && ( = { 'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.', kokoro: 'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.', + 'qwen-custom-voice-1.7B': + 'Qwen3-TTS CustomVoice 1.7B by Alibaba. 9 premium preset voices with instruct-based style control for tone, emotion, and prosody. Supports 10 languages.', + 'qwen-custom-voice-0.6B': + 'Qwen3-TTS CustomVoice 0.6B by Alibaba. Lightweight version with the same 9 preset voices and instruct control. Faster inference for lower-end hardware.', 'whisper-base': 'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.', 'whisper-small': @@ -396,6 +400,7 @@ export function ModelManagement() { modelStatus?.models.filter( (m) => m.model_name.startsWith('qwen-tts') || + m.model_name.startsWith('qwen-custom-voice') || m.model_name.startsWith('luxtts') || m.model_name.startsWith('chatterbox') || m.model_name.startsWith('tada') || diff --git a/app/src/components/VoiceProfiles/ProfileCard.tsx b/app/src/components/VoiceProfiles/ProfileCard.tsx index e2a9d4d4..e634c38c 100644 --- a/app/src/components/VoiceProfiles/ProfileCard.tsx +++ b/app/src/components/VoiceProfiles/ProfileCard.tsx @@ -17,6 +17,12 @@ import { useDeleteProfile, useExportProfile } from '@/lib/hooks/useProfiles'; import { cn } from '@/lib/utils/cn'; import { useUIStore } from '@/stores/uiStore'; +/** Human-readable display names for preset engine badges. */ +const ENGINE_DISPLAY_NAMES: Record = { + kokoro: 'Kokoro', + qwen_custom_voice: 'CustomVoice', +}; + interface ProfileCardProps { profile: VoiceProfileResponse; } @@ -99,7 +105,7 @@ export function ProfileCard({ profile }: ProfileCardProps) { {profile.voice_type === 'preset' && ( - {profile.preset_engine} + {ENGINE_DISPLAY_NAMES[profile.preset_engine ?? ''] ?? profile.preset_engine} )} {profile.voice_type === 'designed' && ( diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index 0c4987b5..a148bfb5 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -60,9 +60,10 @@ import { AudioSampleUpload } from './AudioSampleUpload'; import { SampleList } from './SampleList'; const MAX_AUDIO_DURATION_SECONDS = 30; -const PRESET_ONLY_ENGINES = new Set(['kokoro']); +const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice']); const DEFAULT_ENGINE_OPTIONS = [ { value: 'qwen', label: 'Qwen3-TTS' }, + { value: 'qwen_custom_voice', label: 'Qwen CustomVoice' }, { value: 'luxtts', label: 'LuxTTS' }, { value: 'chatterbox', label: 'Chatterbox' }, { value: 'chatterbox_turbo', label: 'Chatterbox Turbo' }, @@ -849,6 +850,7 @@ export function ProfileForm() { Kokoro 82M + Qwen CustomVoice diff --git a/app/src/components/VoiceProfiles/ProfileList.tsx b/app/src/components/VoiceProfiles/ProfileList.tsx index 606f4a2e..be7332a0 100644 --- a/app/src/components/VoiceProfiles/ProfileList.tsx +++ b/app/src/components/VoiceProfiles/ProfileList.tsx @@ -7,11 +7,12 @@ import { ProfileCard } from './ProfileCard'; import { ProfileForm } from './ProfileForm'; /** Engines that use preset (built-in) voices instead of cloned profiles. */ -const PRESET_ENGINES = new Set(['kokoro']); +const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice']); /** Human-readable engine names for empty state messages. */ const ENGINE_NAMES: Record = { kokoro: 'Kokoro', + qwen_custom_voice: 'Qwen CustomVoice', }; export function ProfileList() { diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 34e8038c..86e3012f 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -62,7 +62,14 @@ export interface GenerationRequest { language: LanguageCode; seed?: number; model_size?: '1.7B' | '0.6B' | '1B' | '3B'; - engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro'; + engine?: + | 'qwen' + | 'qwen_custom_voice' + | 'luxtts' + | 'chatterbox' + | 'chatterbox_turbo' + | 'tada' + | 'kokoro'; instruct?: string; max_chunk_chars?: number; crossfade_ms?: number; diff --git a/app/src/lib/constants/languages.ts b/app/src/lib/constants/languages.ts index 1a5c5f26..e28c519b 100644 --- a/app/src/lib/constants/languages.ts +++ b/app/src/lib/constants/languages.ts @@ -69,6 +69,7 @@ export const ENGINE_LANGUAGES: Record = { chatterbox_turbo: ['en'], tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'], kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'], + qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'], } as const; /** Helper: get language options for a given engine. */ diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts index 894d9333..0acdabbf 100644 --- a/app/src/lib/hooks/useGenerationForm.ts +++ b/app/src/lib/hooks/useGenerationForm.ts @@ -17,7 +17,17 @@ const generationSchema = z.object({ seed: z.number().int().optional(), modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(), instruct: z.string().max(500).optional(), - engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada', 'kokoro']).optional(), + engine: z + .enum([ + 'qwen', + 'qwen_custom_voice', + 'luxtts', + 'chatterbox', + 'chatterbox_turbo', + 'tada', + 'kokoro', + ]) + .optional(), }); export type GenerationFormValues = z.infer; @@ -85,7 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { : 'tada-1b' : engine === 'kokoro' ? 'kokoro' - : `qwen-tts-${data.modelSize}`; + : engine === 'qwen_custom_voice' + ? `qwen-custom-voice-${data.modelSize}` + : `qwen-tts-${data.modelSize}`; const displayName = engine === 'luxtts' ? 'LuxTTS' @@ -99,9 +111,13 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { : 'TADA 1B' : engine === 'kokoro' ? 'Kokoro 82M' - : data.modelSize === '1.7B' - ? 'Qwen TTS 1.7B' - : 'Qwen TTS 0.6B'; + : engine === 'qwen_custom_voice' + ? data.modelSize === '1.7B' + ? 'Qwen CustomVoice 1.7B' + : 'Qwen CustomVoice 0.6B' + : data.modelSize === '1.7B' + ? 'Qwen TTS 1.7B' + : 'Qwen TTS 0.6B'; // Check if model needs downloading try { @@ -116,7 +132,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { console.error('Failed to check model status:', error); } - const hasModelSizes = engine === 'qwen' || engine === 'tada'; + const hasModelSizes = + engine === 'qwen' || engine === 'qwen_custom_voice' || engine === 'tada'; + const supportsInstruct = engine === 'qwen' || engine === 'qwen_custom_voice'; const effectsChain = options.getEffectsChain?.(); // This now returns immediately with status="generating" const result = await generation.mutateAsync({ @@ -126,7 +144,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) { seed: data.seed, model_size: hasModelSizes ? data.modelSize : undefined, engine, - instruct: engine === 'qwen' ? data.instruct || undefined : undefined, + instruct: supportsInstruct ? data.instruct || undefined : undefined, max_chunk_chars: maxChunkChars, crossfade_ms: crossfadeMs, normalize: normalizeAudio, diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index 33ae57d8..db19b140 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -163,6 +163,7 @@ _stt_backend: Optional[STTBackend] = None # The factory function uses this for the if/elif chain; the model configs live on the backend classes. TTS_ENGINES = { "qwen": "Qwen TTS", + "qwen_custom_voice": "Qwen CustomVoice", "luxtts": "LuxTTS", "chatterbox": "Chatterbox TTS", "chatterbox_turbo": "Chatterbox Turbo", @@ -205,6 +206,32 @@ def _get_qwen_model_configs() -> list[ModelConfig]: ] +def _get_qwen_custom_voice_configs() -> list[ModelConfig]: + """Return Qwen CustomVoice model configs.""" + return [ + ModelConfig( + model_name="qwen-custom-voice-1.7B", + display_name="Qwen CustomVoice 1.7B", + engine="qwen_custom_voice", + hf_repo_id="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + model_size="1.7B", + size_mb=3500, + supports_instruct=True, + languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"], + ), + ModelConfig( + model_name="qwen-custom-voice-0.6B", + display_name="Qwen CustomVoice 0.6B", + engine="qwen_custom_voice", + hf_repo_id="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", + model_size="0.6B", + size_mb=1200, + supports_instruct=True, + languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"], + ), + ] + + def _get_non_qwen_tts_configs() -> list[ModelConfig]: """Return model configs for non-Qwen TTS engines. @@ -333,12 +360,12 @@ def _get_whisper_configs() -> list[ModelConfig]: def get_all_model_configs() -> list[ModelConfig]: """Return the full list of model configs (TTS + STT).""" - return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs() + return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs() def get_tts_model_configs() -> list[ModelConfig]: """Return only TTS model configs.""" - return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() # Lookup helpers — these replace the if/elif chains in main.py @@ -369,7 +396,7 @@ def engine_has_model_sizes(engine: str) -> bool: async def load_engine_model(engine: str, model_size: str = "default") -> None: """Load a model for the given engine, handling engines with multiple model sizes.""" backend = get_tts_backend_for_engine(engine) - if engine == "qwen": + if engine in ("qwen", "qwen_custom_voice"): await backend.load_model_async(model_size) elif engine == "tada": await backend.load_model(model_size) @@ -388,7 +415,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default") cfg = c break - if engine in ("qwen", "tada"): + if engine in ("qwen", "qwen_custom_voice", "tada"): if not backend._is_model_cached(model_size): raise HTTPException( status_code=400, @@ -423,6 +450,14 @@ def unload_model_by_config(config: ModelConfig) -> bool: return True return False + if config.engine == "qwen_custom_voice": + backend = get_tts_backend_for_engine(config.engine) + loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None) + if backend.is_loaded() and loaded_size == config.model_size: + backend.unload_model() + return True + return False + # All other TTS engines backend = get_tts_backend_for_engine(config.engine) if backend.is_loaded(): @@ -446,6 +481,11 @@ def check_model_loaded(config: ModelConfig) -> bool: loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None) return tts_model.is_loaded() and loaded_size == config.model_size + if config.engine == "qwen_custom_voice": + backend = get_tts_backend_for_engine(config.engine) + loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None) + return backend.is_loaded() and loaded_size == config.model_size + backend = get_tts_backend_for_engine(config.engine) return backend.is_loaded() except Exception: @@ -463,6 +503,9 @@ def get_model_load_func(config: ModelConfig): if config.engine == "qwen": return lambda: tts.get_tts_model().load_model(config.model_size) + if config.engine == "qwen_custom_voice": + return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size) + return lambda: get_tts_backend_for_engine(config.engine).load_model() @@ -528,6 +571,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: from .kokoro_backend import KokoroTTSBackend backend = KokoroTTSBackend() + elif engine == "qwen_custom_voice": + from .qwen_custom_voice_backend import QwenCustomVoiceBackend + + backend = QwenCustomVoiceBackend() else: raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}") diff --git a/backend/backends/qwen_custom_voice_backend.py b/backend/backends/qwen_custom_voice_backend.py new file mode 100644 index 00000000..fbbf9f30 --- /dev/null +++ b/backend/backends/qwen_custom_voice_backend.py @@ -0,0 +1,210 @@ +""" +Qwen3-TTS CustomVoice backend implementation. + +Wraps the Qwen3-TTS-12Hz CustomVoice model for preset-speaker TTS with +instruction-based style control. Uses the same qwen_tts library as the +Base model (pytorch_backend.py) but loads a different checkpoint and +calls generate_custom_voice() instead of generate_voice_clone(). + +Key differences from the Base engine: + - Uses preset speakers (9 built-in voices) instead of zero-shot cloning + - Supports instruct parameter for tone/emotion/prosody control + - Two model sizes: 1.7B and 0.6B + +Languages supported: zh, en, ja, ko, de, fr, ru, pt, es, it +""" + +import asyncio +import logging +from typing import Optional + +import numpy as np +import torch + +from . import TTSBackend, LANGUAGE_CODE_TO_NAME +from .base import ( + is_model_cached, + get_torch_device, + combine_voice_prompts as _combine_voice_prompts, + model_load_progress, +) + +logger = logging.getLogger(__name__) + +# ── Preset speakers ────────────────────────────────────────────────── + +# (speaker_id, display_name, gender, native_language_code, description) +QWEN_CUSTOM_VOICES = [ + ("Vivian", "Vivian", "female", "zh", "Bright, slightly edgy young female voice"), + ("Serena", "Serena", "female", "zh", "Warm, gentle young female voice"), + ("Uncle_Fu", "Uncle Fu", "male", "zh", "Seasoned male voice with a low, mellow timbre"), + ("Dylan", "Dylan", "male", "zh", "Youthful Beijing male voice with a clear, natural timbre"), + ("Eric", "Eric", "male", "zh", "Lively Chengdu male voice with a slightly husky brightness"), + ("Ryan", "Ryan", "male", "en", "Dynamic male voice with strong rhythmic drive"), + ("Aiden", "Aiden", "male", "en", "Sunny American male voice with a clear midrange"), + ("Ono_Anna", "Ono Anna", "female", "ja", "Playful Japanese female voice with a light, nimble timbre"), + ("Sohee", "Sohee", "female", "ko", "Warm Korean female voice with rich emotion"), +] + +QWEN_CV_DEFAULT_SPEAKER = "Ryan" + +# HuggingFace repo IDs per model size +QWEN_CV_HF_REPOS = { + "1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", +} + + +class QwenCustomVoiceBackend: + """Qwen3-TTS CustomVoice backend — preset speakers with instruct control.""" + + def __init__(self, model_size: str = "1.7B"): + self.model = None + self.model_size = model_size + self.device = self._get_device() + self._current_model_size: Optional[str] = None + + def _get_device(self) -> str: + return get_torch_device(allow_xpu=True, allow_directml=True) + + def is_loaded(self) -> bool: + return self.model is not None + + def _get_model_path(self, model_size: str) -> str: + if model_size not in QWEN_CV_HF_REPOS: + raise ValueError(f"Unknown model size: {model_size}") + return QWEN_CV_HF_REPOS[model_size] + + def _is_model_cached(self, model_size: Optional[str] = None) -> bool: + size = model_size or self.model_size + return is_model_cached(self._get_model_path(size)) + + async def load_model_async(self, model_size: Optional[str] = None) -> None: + if model_size is None: + model_size = self.model_size + + if self.model is not None and self._current_model_size == model_size: + return + + if self.model is not None and self._current_model_size != model_size: + self.unload_model() + + await asyncio.to_thread(self._load_model_sync, model_size) + + # Alias for compatibility with the TTSBackend protocol + load_model = load_model_async + + def _load_model_sync(self, model_size: str) -> None: + model_name = f"qwen-custom-voice-{model_size}" + is_cached = self._is_model_cached(model_size) + + with model_load_progress(model_name, is_cached): + from qwen_tts import Qwen3TTSModel + + model_path = self._get_model_path(model_size) + logger.info("Loading Qwen CustomVoice %s on %s...", model_size, self.device) + + if self.device == "cpu": + self.model = Qwen3TTSModel.from_pretrained( + model_path, + torch_dtype=torch.float32, + low_cpu_mem_usage=False, + ) + else: + self.model = Qwen3TTSModel.from_pretrained( + model_path, + device_map=self.device, + torch_dtype=torch.bfloat16, + ) + + self._current_model_size = model_size + self.model_size = model_size + logger.info("Qwen CustomVoice %s loaded successfully", model_size) + + def unload_model(self) -> None: + if self.model is not None: + del self.model + self.model = None + self._current_model_size = None + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + logger.info("Qwen CustomVoice unloaded") + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + """ + Create voice prompt for CustomVoice. + + CustomVoice doesn't use reference audio — it uses preset speakers. + When called for a cloned profile (fallback), uses the default speaker. + For preset profiles, the voice_prompt dict is built by the profile + service and bypasses this method entirely. + """ + return { + "voice_type": "preset", + "preset_engine": "qwen_custom_voice", + "preset_voice_id": QWEN_CV_DEFAULT_SPEAKER, + }, False + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + return await _combine_voice_prompts(audio_paths, reference_texts) + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "en", + seed: Optional[int] = None, + instruct: Optional[str] = None, + ) -> tuple[np.ndarray, int]: + """ + Generate audio using Qwen CustomVoice. + + Args: + text: Text to synthesize + voice_prompt: Dict with preset_voice_id (speaker name) + language: Language code (zh, en, ja, ko, etc.) + seed: Random seed for reproducibility + instruct: Natural language instruction for style control + (e.g. "Speak in an angry tone", "Very happy") + + Returns: + Tuple of (audio_array, sample_rate) + """ + await self.load_model_async(None) + + speaker = voice_prompt.get("preset_voice_id") or QWEN_CV_DEFAULT_SPEAKER + + def _generate_sync(): + if seed is not None: + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + + lang_name = LANGUAGE_CODE_TO_NAME.get(language, "auto") + + kwargs = { + "text": text, + "language": lang_name.capitalize() if lang_name != "auto" else "Auto", + "speaker": speaker, + } + + # Only pass instruct if non-empty + if instruct: + kwargs["instruct"] = instruct + + wavs, sample_rate = self.model.generate_custom_voice(**kwargs) + return wavs[0], sample_rate + + audio, sample_rate = await asyncio.to_thread(_generate_sync) + return audio, sample_rate diff --git a/backend/build_binary.py b/backend/build_binary.py index ca7b21ac..43ad0719 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -86,6 +86,8 @@ def build_server(cuda=False): "--hidden-import", "backend.backends.pytorch_backend", "--hidden-import", + "backend.backends.qwen_custom_voice_backend", + "--hidden-import", "backend.utils.audio", "--hidden-import", "backend.utils.cache", diff --git a/backend/models.py b/backend/models.py index f568e0fa..f2f43d4b 100644 --- a/backend/models.py +++ b/backend/models.py @@ -78,7 +78,7 @@ class GenerationRequest(BaseModel): seed: Optional[int] = Field(None, ge=0) model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$") instruct: Optional[str] = Field(None, max_length=500) - engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$") + engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$") max_chunk_chars: int = Field( default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting" ) diff --git a/backend/routes/profiles.py b/backend/routes/profiles.py index a13b02fc..665f6b24 100644 --- a/backend/routes/profiles.py +++ b/backend/routes/profiles.py @@ -90,6 +90,21 @@ async def list_preset_voices(engine: str): for vid, name, gender, lang in KOKORO_VOICES ], } + if engine == "qwen_custom_voice": + from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES + + return { + "engine": engine, + "voices": [ + { + "voice_id": speaker_id, + "name": display_name, + "gender": gender, + "language": lang, + } + for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES + ], + } return {"engine": engine, "voices": []} @@ -103,9 +118,15 @@ async def seed_preset_profiles_route( Creates profiles for all available preset voices that don't already exist. Returns the count of newly created profiles. """ - if engine != "kokoro": - raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}") + if engine == "kokoro": + return _seed_kokoro_presets(db) + if engine == "qwen_custom_voice": + return _seed_qwen_custom_voice_presets(db) + raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}") + +def _seed_kokoro_presets(db: Session): + """Seed Kokoro preset profiles.""" try: from ..backends.kokoro_backend import KOKORO_VOICES @@ -154,12 +175,57 @@ async def seed_preset_profiles_route( db.commit() logger.info(f"Seeded {created} Kokoro preset profiles") - return {"engine": engine, "created": created, "total_available": len(KOKORO_VOICES)} + return {"engine": "kokoro", "created": created, "total_available": len(KOKORO_VOICES)} except Exception as e: logger.exception(f"Failed to seed Kokoro profiles: {e}") raise HTTPException(status_code=500, detail=str(e)) +def _seed_qwen_custom_voice_presets(db: Session): + """Seed Qwen CustomVoice preset profiles.""" + try: + from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES + + created = 0 + for speaker_id, display_name, gender, lang, description in QWEN_CUSTOM_VOICES: + # Skip if preset already exists + existing = ( + db.query(DBVoiceProfile) + .filter_by(preset_engine="qwen_custom_voice", preset_voice_id=speaker_id) + .first() + ) + if existing: + continue + + # Skip name collisions + if db.query(DBVoiceProfile).filter_by(name=display_name).first(): + continue + + profile = DBVoiceProfile( + id=str(uuid.uuid4()), + name=display_name, + description=f"Qwen CustomVoice — {description}", + language=lang, + voice_type="preset", + preset_engine="qwen_custom_voice", + preset_voice_id=speaker_id, + default_engine="qwen_custom_voice", + created_at=datetime.utcnow(), + updated_at=datetime.utcnow(), + ) + db.add(profile) + created += 1 + + if created > 0: + db.commit() + logger.info(f"Seeded {created} Qwen CustomVoice preset profiles") + + return {"engine": "qwen_custom_voice", "created": created, "total_available": len(QWEN_CUSTOM_VOICES)} + except Exception as e: + logger.exception(f"Failed to seed Qwen CustomVoice profiles: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse) async def get_profile( profile_id: str,