mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-12 19:30:38 -07:00
feat: add Qwen CustomVoice preset engine
This commit is contained in:
@@ -19,6 +19,8 @@ import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm';
|
||||
const ENGINE_OPTIONS = [
|
||||
{ value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' },
|
||||
{ value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' },
|
||||
{ value: 'qwen_custom_voice:1.7B', label: 'Qwen CustomVoice 1.7B', engine: 'qwen_custom_voice' },
|
||||
{ value: 'qwen_custom_voice:0.6B', label: 'Qwen CustomVoice 0.6B', engine: 'qwen_custom_voice' },
|
||||
{ value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' },
|
||||
{ value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' },
|
||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' },
|
||||
@@ -29,6 +31,7 @@ const ENGINE_OPTIONS = [
|
||||
|
||||
const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
||||
qwen: 'Multi-language, two sizes',
|
||||
qwen_custom_voice: '9 preset voices, instruct control',
|
||||
luxtts: 'Fast, English-focused',
|
||||
chatterbox: '23 languages, incl. Hebrew',
|
||||
chatterbox_turbo: 'English, [laugh] [cough] tags',
|
||||
@@ -49,12 +52,22 @@ function getAvailableOptions(selectedProfile?: VoiceProfileResponse | null) {
|
||||
|
||||
function getSelectValue(engine: string, modelSize?: string): string {
|
||||
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
|
||||
if (engine === 'qwen_custom_voice') return `qwen_custom_voice:${modelSize || '1.7B'}`;
|
||||
if (engine === 'tada') return `tada:${modelSize || '1B'}`;
|
||||
return engine;
|
||||
}
|
||||
|
||||
function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) {
|
||||
if (value.startsWith('qwen:')) {
|
||||
if (value.startsWith('qwen_custom_voice:')) {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen_custom_voice');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
const currentLang = form.getValues('language');
|
||||
const available = getLanguageOptionsForEngine('qwen_custom_voice');
|
||||
if (!available.some((l) => l.value === currentLang)) {
|
||||
form.setValue('language', available[0]?.value ?? 'en');
|
||||
}
|
||||
} else if (value.startsWith('qwen:')) {
|
||||
const [, modelSize] = value.split(':');
|
||||
form.setValue('engine', 'qwen');
|
||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||
|
||||
@@ -91,7 +91,7 @@ export function GenerationForm() {
|
||||
)}
|
||||
/>
|
||||
|
||||
{form.watch('engine') === 'qwen' && (
|
||||
{(form.watch('engine') === 'qwen' || form.watch('engine') === 'qwen_custom_voice') && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="instruct"
|
||||
|
||||
@@ -68,6 +68,10 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
|
||||
'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.',
|
||||
kokoro:
|
||||
'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.',
|
||||
'qwen-custom-voice-1.7B':
|
||||
'Qwen3-TTS CustomVoice 1.7B by Alibaba. 9 premium preset voices with instruct-based style control for tone, emotion, and prosody. Supports 10 languages.',
|
||||
'qwen-custom-voice-0.6B':
|
||||
'Qwen3-TTS CustomVoice 0.6B by Alibaba. Lightweight version with the same 9 preset voices and instruct control. Faster inference for lower-end hardware.',
|
||||
'whisper-base':
|
||||
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
|
||||
'whisper-small':
|
||||
@@ -396,6 +400,7 @@ export function ModelManagement() {
|
||||
modelStatus?.models.filter(
|
||||
(m) =>
|
||||
m.model_name.startsWith('qwen-tts') ||
|
||||
m.model_name.startsWith('qwen-custom-voice') ||
|
||||
m.model_name.startsWith('luxtts') ||
|
||||
m.model_name.startsWith('chatterbox') ||
|
||||
m.model_name.startsWith('tada') ||
|
||||
|
||||
@@ -17,6 +17,12 @@ import { useDeleteProfile, useExportProfile } from '@/lib/hooks/useProfiles';
|
||||
import { cn } from '@/lib/utils/cn';
|
||||
import { useUIStore } from '@/stores/uiStore';
|
||||
|
||||
/** Human-readable display names for preset engine badges. */
|
||||
const ENGINE_DISPLAY_NAMES: Record<string, string> = {
|
||||
kokoro: 'Kokoro',
|
||||
qwen_custom_voice: 'CustomVoice',
|
||||
};
|
||||
|
||||
interface ProfileCardProps {
|
||||
profile: VoiceProfileResponse;
|
||||
}
|
||||
@@ -99,7 +105,7 @@ export function ProfileCard({ profile }: ProfileCardProps) {
|
||||
</Badge>
|
||||
{profile.voice_type === 'preset' && (
|
||||
<Badge variant="secondary" className="text-xs h-5 px-1.5">
|
||||
{profile.preset_engine}
|
||||
{ENGINE_DISPLAY_NAMES[profile.preset_engine ?? ''] ?? profile.preset_engine}
|
||||
</Badge>
|
||||
)}
|
||||
{profile.voice_type === 'designed' && (
|
||||
|
||||
@@ -60,9 +60,10 @@ import { AudioSampleUpload } from './AudioSampleUpload';
|
||||
import { SampleList } from './SampleList';
|
||||
|
||||
const MAX_AUDIO_DURATION_SECONDS = 30;
|
||||
const PRESET_ONLY_ENGINES = new Set(['kokoro']);
|
||||
const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
|
||||
const DEFAULT_ENGINE_OPTIONS = [
|
||||
{ value: 'qwen', label: 'Qwen3-TTS' },
|
||||
{ value: 'qwen_custom_voice', label: 'Qwen CustomVoice' },
|
||||
{ value: 'luxtts', label: 'LuxTTS' },
|
||||
{ value: 'chatterbox', label: 'Chatterbox' },
|
||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
|
||||
@@ -849,6 +850,7 @@ export function ProfileForm() {
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="kokoro">Kokoro 82M</SelectItem>
|
||||
<SelectItem value="qwen_custom_voice">Qwen CustomVoice</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
@@ -7,11 +7,12 @@ import { ProfileCard } from './ProfileCard';
|
||||
import { ProfileForm } from './ProfileForm';
|
||||
|
||||
/** Engines that use preset (built-in) voices instead of cloned profiles. */
|
||||
const PRESET_ENGINES = new Set(['kokoro']);
|
||||
const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
|
||||
|
||||
/** Human-readable engine names for empty state messages. */
|
||||
const ENGINE_NAMES: Record<string, string> = {
|
||||
kokoro: 'Kokoro',
|
||||
qwen_custom_voice: 'Qwen CustomVoice',
|
||||
};
|
||||
|
||||
export function ProfileList() {
|
||||
|
||||
@@ -62,7 +62,14 @@ export interface GenerationRequest {
|
||||
language: LanguageCode;
|
||||
seed?: number;
|
||||
model_size?: '1.7B' | '0.6B' | '1B' | '3B';
|
||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro';
|
||||
engine?:
|
||||
| 'qwen'
|
||||
| 'qwen_custom_voice'
|
||||
| 'luxtts'
|
||||
| 'chatterbox'
|
||||
| 'chatterbox_turbo'
|
||||
| 'tada'
|
||||
| 'kokoro';
|
||||
instruct?: string;
|
||||
max_chunk_chars?: number;
|
||||
crossfade_ms?: number;
|
||||
|
||||
@@ -69,6 +69,7 @@ export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
|
||||
chatterbox_turbo: ['en'],
|
||||
tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'],
|
||||
kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'],
|
||||
qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'],
|
||||
} as const;
|
||||
|
||||
/** Helper: get language options for a given engine. */
|
||||
|
||||
@@ -17,7 +17,17 @@ const generationSchema = z.object({
|
||||
seed: z.number().int().optional(),
|
||||
modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(),
|
||||
instruct: z.string().max(500).optional(),
|
||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada', 'kokoro']).optional(),
|
||||
engine: z
|
||||
.enum([
|
||||
'qwen',
|
||||
'qwen_custom_voice',
|
||||
'luxtts',
|
||||
'chatterbox',
|
||||
'chatterbox_turbo',
|
||||
'tada',
|
||||
'kokoro',
|
||||
])
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||
@@ -85,7 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
: 'tada-1b'
|
||||
: engine === 'kokoro'
|
||||
? 'kokoro'
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
: engine === 'qwen_custom_voice'
|
||||
? `qwen-custom-voice-${data.modelSize}`
|
||||
: `qwen-tts-${data.modelSize}`;
|
||||
const displayName =
|
||||
engine === 'luxtts'
|
||||
? 'LuxTTS'
|
||||
@@ -99,9 +111,13 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
: 'TADA 1B'
|
||||
: engine === 'kokoro'
|
||||
? 'Kokoro 82M'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
: engine === 'qwen_custom_voice'
|
||||
? data.modelSize === '1.7B'
|
||||
? 'Qwen CustomVoice 1.7B'
|
||||
: 'Qwen CustomVoice 0.6B'
|
||||
: data.modelSize === '1.7B'
|
||||
? 'Qwen TTS 1.7B'
|
||||
: 'Qwen TTS 0.6B';
|
||||
|
||||
// Check if model needs downloading
|
||||
try {
|
||||
@@ -116,7 +132,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
console.error('Failed to check model status:', error);
|
||||
}
|
||||
|
||||
const hasModelSizes = engine === 'qwen' || engine === 'tada';
|
||||
const hasModelSizes =
|
||||
engine === 'qwen' || engine === 'qwen_custom_voice' || engine === 'tada';
|
||||
const supportsInstruct = engine === 'qwen' || engine === 'qwen_custom_voice';
|
||||
const effectsChain = options.getEffectsChain?.();
|
||||
// This now returns immediately with status="generating"
|
||||
const result = await generation.mutateAsync({
|
||||
@@ -126,7 +144,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
||||
seed: data.seed,
|
||||
model_size: hasModelSizes ? data.modelSize : undefined,
|
||||
engine,
|
||||
instruct: engine === 'qwen' ? data.instruct || undefined : undefined,
|
||||
instruct: supportsInstruct ? data.instruct || undefined : undefined,
|
||||
max_chunk_chars: maxChunkChars,
|
||||
crossfade_ms: crossfadeMs,
|
||||
normalize: normalizeAudio,
|
||||
|
||||
@@ -163,6 +163,7 @@ _stt_backend: Optional[STTBackend] = None
|
||||
# The factory function uses this for the if/elif chain; the model configs live on the backend classes.
|
||||
TTS_ENGINES = {
|
||||
"qwen": "Qwen TTS",
|
||||
"qwen_custom_voice": "Qwen CustomVoice",
|
||||
"luxtts": "LuxTTS",
|
||||
"chatterbox": "Chatterbox TTS",
|
||||
"chatterbox_turbo": "Chatterbox Turbo",
|
||||
@@ -205,6 +206,32 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
||||
]
|
||||
|
||||
|
||||
def _get_qwen_custom_voice_configs() -> list[ModelConfig]:
|
||||
"""Return Qwen CustomVoice model configs."""
|
||||
return [
|
||||
ModelConfig(
|
||||
model_name="qwen-custom-voice-1.7B",
|
||||
display_name="Qwen CustomVoice 1.7B",
|
||||
engine="qwen_custom_voice",
|
||||
hf_repo_id="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
model_size="1.7B",
|
||||
size_mb=3500,
|
||||
supports_instruct=True,
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
ModelConfig(
|
||||
model_name="qwen-custom-voice-0.6B",
|
||||
display_name="Qwen CustomVoice 0.6B",
|
||||
engine="qwen_custom_voice",
|
||||
hf_repo_id="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
|
||||
model_size="0.6B",
|
||||
size_mb=1200,
|
||||
supports_instruct=True,
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _get_non_qwen_tts_configs() -> list[ModelConfig]:
|
||||
"""Return model configs for non-Qwen TTS engines.
|
||||
|
||||
@@ -333,12 +360,12 @@ def _get_whisper_configs() -> list[ModelConfig]:
|
||||
|
||||
def get_all_model_configs() -> list[ModelConfig]:
|
||||
"""Return the full list of model configs (TTS + STT)."""
|
||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
|
||||
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
|
||||
|
||||
|
||||
def get_tts_model_configs() -> list[ModelConfig]:
|
||||
"""Return only TTS model configs."""
|
||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs()
|
||||
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs()
|
||||
|
||||
|
||||
# Lookup helpers — these replace the if/elif chains in main.py
|
||||
@@ -369,7 +396,7 @@ def engine_has_model_sizes(engine: str) -> bool:
|
||||
async def load_engine_model(engine: str, model_size: str = "default") -> None:
|
||||
"""Load a model for the given engine, handling engines with multiple model sizes."""
|
||||
backend = get_tts_backend_for_engine(engine)
|
||||
if engine == "qwen":
|
||||
if engine in ("qwen", "qwen_custom_voice"):
|
||||
await backend.load_model_async(model_size)
|
||||
elif engine == "tada":
|
||||
await backend.load_model(model_size)
|
||||
@@ -388,7 +415,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default")
|
||||
cfg = c
|
||||
break
|
||||
|
||||
if engine in ("qwen", "tada"):
|
||||
if engine in ("qwen", "qwen_custom_voice", "tada"):
|
||||
if not backend._is_model_cached(model_size):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
@@ -423,6 +450,14 @@ def unload_model_by_config(config: ModelConfig) -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
if config.engine == "qwen_custom_voice":
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
|
||||
if backend.is_loaded() and loaded_size == config.model_size:
|
||||
backend.unload_model()
|
||||
return True
|
||||
return False
|
||||
|
||||
# All other TTS engines
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
if backend.is_loaded():
|
||||
@@ -446,6 +481,11 @@ def check_model_loaded(config: ModelConfig) -> bool:
|
||||
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
|
||||
return tts_model.is_loaded() and loaded_size == config.model_size
|
||||
|
||||
if config.engine == "qwen_custom_voice":
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
|
||||
return backend.is_loaded() and loaded_size == config.model_size
|
||||
|
||||
backend = get_tts_backend_for_engine(config.engine)
|
||||
return backend.is_loaded()
|
||||
except Exception:
|
||||
@@ -463,6 +503,9 @@ def get_model_load_func(config: ModelConfig):
|
||||
if config.engine == "qwen":
|
||||
return lambda: tts.get_tts_model().load_model(config.model_size)
|
||||
|
||||
if config.engine == "qwen_custom_voice":
|
||||
return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size)
|
||||
|
||||
return lambda: get_tts_backend_for_engine(config.engine).load_model()
|
||||
|
||||
|
||||
@@ -528,6 +571,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
||||
from .kokoro_backend import KokoroTTSBackend
|
||||
|
||||
backend = KokoroTTSBackend()
|
||||
elif engine == "qwen_custom_voice":
|
||||
from .qwen_custom_voice_backend import QwenCustomVoiceBackend
|
||||
|
||||
backend = QwenCustomVoiceBackend()
|
||||
else:
|
||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Qwen3-TTS CustomVoice backend implementation.
|
||||
|
||||
Wraps the Qwen3-TTS-12Hz CustomVoice model for preset-speaker TTS with
|
||||
instruction-based style control. Uses the same qwen_tts library as the
|
||||
Base model (pytorch_backend.py) but loads a different checkpoint and
|
||||
calls generate_custom_voice() instead of generate_voice_clone().
|
||||
|
||||
Key differences from the Base engine:
|
||||
- Uses preset speakers (9 built-in voices) instead of zero-shot cloning
|
||||
- Supports instruct parameter for tone/emotion/prosody control
|
||||
- Two model sizes: 1.7B and 0.6B
|
||||
|
||||
Languages supported: zh, en, ja, ko, de, fr, ru, pt, es, it
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from . import TTSBackend, LANGUAGE_CODE_TO_NAME
|
||||
from .base import (
|
||||
is_model_cached,
|
||||
get_torch_device,
|
||||
combine_voice_prompts as _combine_voice_prompts,
|
||||
model_load_progress,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Preset speakers ──────────────────────────────────────────────────
|
||||
|
||||
# (speaker_id, display_name, gender, native_language_code, description)
|
||||
QWEN_CUSTOM_VOICES = [
|
||||
("Vivian", "Vivian", "female", "zh", "Bright, slightly edgy young female voice"),
|
||||
("Serena", "Serena", "female", "zh", "Warm, gentle young female voice"),
|
||||
("Uncle_Fu", "Uncle Fu", "male", "zh", "Seasoned male voice with a low, mellow timbre"),
|
||||
("Dylan", "Dylan", "male", "zh", "Youthful Beijing male voice with a clear, natural timbre"),
|
||||
("Eric", "Eric", "male", "zh", "Lively Chengdu male voice with a slightly husky brightness"),
|
||||
("Ryan", "Ryan", "male", "en", "Dynamic male voice with strong rhythmic drive"),
|
||||
("Aiden", "Aiden", "male", "en", "Sunny American male voice with a clear midrange"),
|
||||
("Ono_Anna", "Ono Anna", "female", "ja", "Playful Japanese female voice with a light, nimble timbre"),
|
||||
("Sohee", "Sohee", "female", "ko", "Warm Korean female voice with rich emotion"),
|
||||
]
|
||||
|
||||
QWEN_CV_DEFAULT_SPEAKER = "Ryan"
|
||||
|
||||
# HuggingFace repo IDs per model size
|
||||
QWEN_CV_HF_REPOS = {
|
||||
"1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
|
||||
}
|
||||
|
||||
|
||||
class QwenCustomVoiceBackend:
|
||||
"""Qwen3-TTS CustomVoice backend — preset speakers with instruct control."""
|
||||
|
||||
def __init__(self, model_size: str = "1.7B"):
|
||||
self.model = None
|
||||
self.model_size = model_size
|
||||
self.device = self._get_device()
|
||||
self._current_model_size: Optional[str] = None
|
||||
|
||||
def _get_device(self) -> str:
|
||||
return get_torch_device(allow_xpu=True, allow_directml=True)
|
||||
|
||||
def is_loaded(self) -> bool:
|
||||
return self.model is not None
|
||||
|
||||
def _get_model_path(self, model_size: str) -> str:
|
||||
if model_size not in QWEN_CV_HF_REPOS:
|
||||
raise ValueError(f"Unknown model size: {model_size}")
|
||||
return QWEN_CV_HF_REPOS[model_size]
|
||||
|
||||
def _is_model_cached(self, model_size: Optional[str] = None) -> bool:
|
||||
size = model_size or self.model_size
|
||||
return is_model_cached(self._get_model_path(size))
|
||||
|
||||
async def load_model_async(self, model_size: Optional[str] = None) -> None:
|
||||
if model_size is None:
|
||||
model_size = self.model_size
|
||||
|
||||
if self.model is not None and self._current_model_size == model_size:
|
||||
return
|
||||
|
||||
if self.model is not None and self._current_model_size != model_size:
|
||||
self.unload_model()
|
||||
|
||||
await asyncio.to_thread(self._load_model_sync, model_size)
|
||||
|
||||
# Alias for compatibility with the TTSBackend protocol
|
||||
load_model = load_model_async
|
||||
|
||||
def _load_model_sync(self, model_size: str) -> None:
|
||||
model_name = f"qwen-custom-voice-{model_size}"
|
||||
is_cached = self._is_model_cached(model_size)
|
||||
|
||||
with model_load_progress(model_name, is_cached):
|
||||
from qwen_tts import Qwen3TTSModel
|
||||
|
||||
model_path = self._get_model_path(model_size)
|
||||
logger.info("Loading Qwen CustomVoice %s on %s...", model_size, self.device)
|
||||
|
||||
if self.device == "cpu":
|
||||
self.model = Qwen3TTSModel.from_pretrained(
|
||||
model_path,
|
||||
torch_dtype=torch.float32,
|
||||
low_cpu_mem_usage=False,
|
||||
)
|
||||
else:
|
||||
self.model = Qwen3TTSModel.from_pretrained(
|
||||
model_path,
|
||||
device_map=self.device,
|
||||
torch_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
self._current_model_size = model_size
|
||||
self.model_size = model_size
|
||||
logger.info("Qwen CustomVoice %s loaded successfully", model_size)
|
||||
|
||||
def unload_model(self) -> None:
|
||||
if self.model is not None:
|
||||
del self.model
|
||||
self.model = None
|
||||
self._current_model_size = None
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
logger.info("Qwen CustomVoice unloaded")
|
||||
|
||||
async def create_voice_prompt(
|
||||
self,
|
||||
audio_path: str,
|
||||
reference_text: str,
|
||||
use_cache: bool = True,
|
||||
) -> tuple[dict, bool]:
|
||||
"""
|
||||
Create voice prompt for CustomVoice.
|
||||
|
||||
CustomVoice doesn't use reference audio — it uses preset speakers.
|
||||
When called for a cloned profile (fallback), uses the default speaker.
|
||||
For preset profiles, the voice_prompt dict is built by the profile
|
||||
service and bypasses this method entirely.
|
||||
"""
|
||||
return {
|
||||
"voice_type": "preset",
|
||||
"preset_engine": "qwen_custom_voice",
|
||||
"preset_voice_id": QWEN_CV_DEFAULT_SPEAKER,
|
||||
}, False
|
||||
|
||||
async def combine_voice_prompts(
|
||||
self,
|
||||
audio_paths: list[str],
|
||||
reference_texts: list[str],
|
||||
) -> tuple[np.ndarray, str]:
|
||||
return await _combine_voice_prompts(audio_paths, reference_texts)
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
text: str,
|
||||
voice_prompt: dict,
|
||||
language: str = "en",
|
||||
seed: Optional[int] = None,
|
||||
instruct: Optional[str] = None,
|
||||
) -> tuple[np.ndarray, int]:
|
||||
"""
|
||||
Generate audio using Qwen CustomVoice.
|
||||
|
||||
Args:
|
||||
text: Text to synthesize
|
||||
voice_prompt: Dict with preset_voice_id (speaker name)
|
||||
language: Language code (zh, en, ja, ko, etc.)
|
||||
seed: Random seed for reproducibility
|
||||
instruct: Natural language instruction for style control
|
||||
(e.g. "Speak in an angry tone", "Very happy")
|
||||
|
||||
Returns:
|
||||
Tuple of (audio_array, sample_rate)
|
||||
"""
|
||||
await self.load_model_async(None)
|
||||
|
||||
speaker = voice_prompt.get("preset_voice_id") or QWEN_CV_DEFAULT_SPEAKER
|
||||
|
||||
def _generate_sync():
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
lang_name = LANGUAGE_CODE_TO_NAME.get(language, "auto")
|
||||
|
||||
kwargs = {
|
||||
"text": text,
|
||||
"language": lang_name.capitalize() if lang_name != "auto" else "Auto",
|
||||
"speaker": speaker,
|
||||
}
|
||||
|
||||
# Only pass instruct if non-empty
|
||||
if instruct:
|
||||
kwargs["instruct"] = instruct
|
||||
|
||||
wavs, sample_rate = self.model.generate_custom_voice(**kwargs)
|
||||
return wavs[0], sample_rate
|
||||
|
||||
audio, sample_rate = await asyncio.to_thread(_generate_sync)
|
||||
return audio, sample_rate
|
||||
@@ -86,6 +86,8 @@ def build_server(cuda=False):
|
||||
"--hidden-import",
|
||||
"backend.backends.pytorch_backend",
|
||||
"--hidden-import",
|
||||
"backend.backends.qwen_custom_voice_backend",
|
||||
"--hidden-import",
|
||||
"backend.utils.audio",
|
||||
"--hidden-import",
|
||||
"backend.utils.cache",
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ class GenerationRequest(BaseModel):
|
||||
seed: Optional[int] = Field(None, ge=0)
|
||||
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$")
|
||||
instruct: Optional[str] = Field(None, max_length=500)
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$")
|
||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$")
|
||||
max_chunk_chars: int = Field(
|
||||
default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting"
|
||||
)
|
||||
|
||||
@@ -90,6 +90,21 @@ async def list_preset_voices(engine: str):
|
||||
for vid, name, gender, lang in KOKORO_VOICES
|
||||
],
|
||||
}
|
||||
if engine == "qwen_custom_voice":
|
||||
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
|
||||
|
||||
return {
|
||||
"engine": engine,
|
||||
"voices": [
|
||||
{
|
||||
"voice_id": speaker_id,
|
||||
"name": display_name,
|
||||
"gender": gender,
|
||||
"language": lang,
|
||||
}
|
||||
for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES
|
||||
],
|
||||
}
|
||||
return {"engine": engine, "voices": []}
|
||||
|
||||
|
||||
@@ -103,9 +118,15 @@ async def seed_preset_profiles_route(
|
||||
Creates profiles for all available preset voices that don't already exist.
|
||||
Returns the count of newly created profiles.
|
||||
"""
|
||||
if engine != "kokoro":
|
||||
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}")
|
||||
if engine == "kokoro":
|
||||
return _seed_kokoro_presets(db)
|
||||
if engine == "qwen_custom_voice":
|
||||
return _seed_qwen_custom_voice_presets(db)
|
||||
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}")
|
||||
|
||||
|
||||
def _seed_kokoro_presets(db: Session):
|
||||
"""Seed Kokoro preset profiles."""
|
||||
try:
|
||||
from ..backends.kokoro_backend import KOKORO_VOICES
|
||||
|
||||
@@ -154,12 +175,57 @@ async def seed_preset_profiles_route(
|
||||
db.commit()
|
||||
logger.info(f"Seeded {created} Kokoro preset profiles")
|
||||
|
||||
return {"engine": engine, "created": created, "total_available": len(KOKORO_VOICES)}
|
||||
return {"engine": "kokoro", "created": created, "total_available": len(KOKORO_VOICES)}
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to seed Kokoro profiles: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
def _seed_qwen_custom_voice_presets(db: Session):
|
||||
"""Seed Qwen CustomVoice preset profiles."""
|
||||
try:
|
||||
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
|
||||
|
||||
created = 0
|
||||
for speaker_id, display_name, gender, lang, description in QWEN_CUSTOM_VOICES:
|
||||
# Skip if preset already exists
|
||||
existing = (
|
||||
db.query(DBVoiceProfile)
|
||||
.filter_by(preset_engine="qwen_custom_voice", preset_voice_id=speaker_id)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
continue
|
||||
|
||||
# Skip name collisions
|
||||
if db.query(DBVoiceProfile).filter_by(name=display_name).first():
|
||||
continue
|
||||
|
||||
profile = DBVoiceProfile(
|
||||
id=str(uuid.uuid4()),
|
||||
name=display_name,
|
||||
description=f"Qwen CustomVoice — {description}",
|
||||
language=lang,
|
||||
voice_type="preset",
|
||||
preset_engine="qwen_custom_voice",
|
||||
preset_voice_id=speaker_id,
|
||||
default_engine="qwen_custom_voice",
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
db.add(profile)
|
||||
created += 1
|
||||
|
||||
if created > 0:
|
||||
db.commit()
|
||||
logger.info(f"Seeded {created} Qwen CustomVoice preset profiles")
|
||||
|
||||
return {"engine": "qwen_custom_voice", "created": created, "total_available": len(QWEN_CUSTOM_VOICES)}
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to seed Qwen CustomVoice profiles: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
|
||||
async def get_profile(
|
||||
profile_id: str,
|
||||
|
||||
Reference in New Issue
Block a user