mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-19 23:00:45 -07:00
feat: add Qwen CustomVoice preset engine
This commit is contained in:
@@ -19,6 +19,8 @@ import type { GenerationFormValues } from '@/lib/hooks/useGenerationForm';
|
|||||||
const ENGINE_OPTIONS = [
|
const ENGINE_OPTIONS = [
|
||||||
{ value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' },
|
{ value: 'qwen:1.7B', label: 'Qwen3-TTS 1.7B', engine: 'qwen' },
|
||||||
{ value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' },
|
{ value: 'qwen:0.6B', label: 'Qwen3-TTS 0.6B', engine: 'qwen' },
|
||||||
|
{ value: 'qwen_custom_voice:1.7B', label: 'Qwen CustomVoice 1.7B', engine: 'qwen_custom_voice' },
|
||||||
|
{ value: 'qwen_custom_voice:0.6B', label: 'Qwen CustomVoice 0.6B', engine: 'qwen_custom_voice' },
|
||||||
{ value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' },
|
{ value: 'luxtts', label: 'LuxTTS', engine: 'luxtts' },
|
||||||
{ value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' },
|
{ value: 'chatterbox', label: 'Chatterbox', engine: 'chatterbox' },
|
||||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' },
|
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo', engine: 'chatterbox_turbo' },
|
||||||
@@ -29,6 +31,7 @@ const ENGINE_OPTIONS = [
|
|||||||
|
|
||||||
const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
const ENGINE_DESCRIPTIONS: Record<string, string> = {
|
||||||
qwen: 'Multi-language, two sizes',
|
qwen: 'Multi-language, two sizes',
|
||||||
|
qwen_custom_voice: '9 preset voices, instruct control',
|
||||||
luxtts: 'Fast, English-focused',
|
luxtts: 'Fast, English-focused',
|
||||||
chatterbox: '23 languages, incl. Hebrew',
|
chatterbox: '23 languages, incl. Hebrew',
|
||||||
chatterbox_turbo: 'English, [laugh] [cough] tags',
|
chatterbox_turbo: 'English, [laugh] [cough] tags',
|
||||||
@@ -49,12 +52,22 @@ function getAvailableOptions(selectedProfile?: VoiceProfileResponse | null) {
|
|||||||
|
|
||||||
function getSelectValue(engine: string, modelSize?: string): string {
|
function getSelectValue(engine: string, modelSize?: string): string {
|
||||||
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
|
if (engine === 'qwen') return `qwen:${modelSize || '1.7B'}`;
|
||||||
|
if (engine === 'qwen_custom_voice') return `qwen_custom_voice:${modelSize || '1.7B'}`;
|
||||||
if (engine === 'tada') return `tada:${modelSize || '1B'}`;
|
if (engine === 'tada') return `tada:${modelSize || '1B'}`;
|
||||||
return engine;
|
return engine;
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) {
|
function handleEngineChange(form: UseFormReturn<GenerationFormValues>, value: string) {
|
||||||
if (value.startsWith('qwen:')) {
|
if (value.startsWith('qwen_custom_voice:')) {
|
||||||
|
const [, modelSize] = value.split(':');
|
||||||
|
form.setValue('engine', 'qwen_custom_voice');
|
||||||
|
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||||
|
const currentLang = form.getValues('language');
|
||||||
|
const available = getLanguageOptionsForEngine('qwen_custom_voice');
|
||||||
|
if (!available.some((l) => l.value === currentLang)) {
|
||||||
|
form.setValue('language', available[0]?.value ?? 'en');
|
||||||
|
}
|
||||||
|
} else if (value.startsWith('qwen:')) {
|
||||||
const [, modelSize] = value.split(':');
|
const [, modelSize] = value.split(':');
|
||||||
form.setValue('engine', 'qwen');
|
form.setValue('engine', 'qwen');
|
||||||
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
form.setValue('modelSize', modelSize as '1.7B' | '0.6B');
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ export function GenerationForm() {
|
|||||||
)}
|
)}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
{form.watch('engine') === 'qwen' && (
|
{(form.watch('engine') === 'qwen' || form.watch('engine') === 'qwen_custom_voice') && (
|
||||||
<FormField
|
<FormField
|
||||||
control={form.control}
|
control={form.control}
|
||||||
name="instruct"
|
name="instruct"
|
||||||
|
|||||||
@@ -68,6 +68,10 @@ const MODEL_DESCRIPTIONS: Record<string, string> = {
|
|||||||
'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.',
|
'HumeAI TADA 3B Multilingual — built on Llama 3.2 3B. Supports 10 languages with high-fidelity voice cloning via text-acoustic dual alignment.',
|
||||||
kokoro:
|
kokoro:
|
||||||
'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.',
|
'Kokoro 82M by hexgrad. Tiny 82M-parameter TTS that runs at CPU realtime. Supports 8 languages with pre-built voice styles. Apache 2.0 licensed.',
|
||||||
|
'qwen-custom-voice-1.7B':
|
||||||
|
'Qwen3-TTS CustomVoice 1.7B by Alibaba. 9 premium preset voices with instruct-based style control for tone, emotion, and prosody. Supports 10 languages.',
|
||||||
|
'qwen-custom-voice-0.6B':
|
||||||
|
'Qwen3-TTS CustomVoice 0.6B by Alibaba. Lightweight version with the same 9 preset voices and instruct control. Faster inference for lower-end hardware.',
|
||||||
'whisper-base':
|
'whisper-base':
|
||||||
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
|
'Smallest Whisper model (74M parameters). Fast transcription with moderate accuracy.',
|
||||||
'whisper-small':
|
'whisper-small':
|
||||||
@@ -396,6 +400,7 @@ export function ModelManagement() {
|
|||||||
modelStatus?.models.filter(
|
modelStatus?.models.filter(
|
||||||
(m) =>
|
(m) =>
|
||||||
m.model_name.startsWith('qwen-tts') ||
|
m.model_name.startsWith('qwen-tts') ||
|
||||||
|
m.model_name.startsWith('qwen-custom-voice') ||
|
||||||
m.model_name.startsWith('luxtts') ||
|
m.model_name.startsWith('luxtts') ||
|
||||||
m.model_name.startsWith('chatterbox') ||
|
m.model_name.startsWith('chatterbox') ||
|
||||||
m.model_name.startsWith('tada') ||
|
m.model_name.startsWith('tada') ||
|
||||||
|
|||||||
@@ -17,6 +17,12 @@ import { useDeleteProfile, useExportProfile } from '@/lib/hooks/useProfiles';
|
|||||||
import { cn } from '@/lib/utils/cn';
|
import { cn } from '@/lib/utils/cn';
|
||||||
import { useUIStore } from '@/stores/uiStore';
|
import { useUIStore } from '@/stores/uiStore';
|
||||||
|
|
||||||
|
/** Human-readable display names for preset engine badges. */
|
||||||
|
const ENGINE_DISPLAY_NAMES: Record<string, string> = {
|
||||||
|
kokoro: 'Kokoro',
|
||||||
|
qwen_custom_voice: 'CustomVoice',
|
||||||
|
};
|
||||||
|
|
||||||
interface ProfileCardProps {
|
interface ProfileCardProps {
|
||||||
profile: VoiceProfileResponse;
|
profile: VoiceProfileResponse;
|
||||||
}
|
}
|
||||||
@@ -99,7 +105,7 @@ export function ProfileCard({ profile }: ProfileCardProps) {
|
|||||||
</Badge>
|
</Badge>
|
||||||
{profile.voice_type === 'preset' && (
|
{profile.voice_type === 'preset' && (
|
||||||
<Badge variant="secondary" className="text-xs h-5 px-1.5">
|
<Badge variant="secondary" className="text-xs h-5 px-1.5">
|
||||||
{profile.preset_engine}
|
{ENGINE_DISPLAY_NAMES[profile.preset_engine ?? ''] ?? profile.preset_engine}
|
||||||
</Badge>
|
</Badge>
|
||||||
)}
|
)}
|
||||||
{profile.voice_type === 'designed' && (
|
{profile.voice_type === 'designed' && (
|
||||||
|
|||||||
@@ -60,9 +60,10 @@ import { AudioSampleUpload } from './AudioSampleUpload';
|
|||||||
import { SampleList } from './SampleList';
|
import { SampleList } from './SampleList';
|
||||||
|
|
||||||
const MAX_AUDIO_DURATION_SECONDS = 30;
|
const MAX_AUDIO_DURATION_SECONDS = 30;
|
||||||
const PRESET_ONLY_ENGINES = new Set(['kokoro']);
|
const PRESET_ONLY_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
|
||||||
const DEFAULT_ENGINE_OPTIONS = [
|
const DEFAULT_ENGINE_OPTIONS = [
|
||||||
{ value: 'qwen', label: 'Qwen3-TTS' },
|
{ value: 'qwen', label: 'Qwen3-TTS' },
|
||||||
|
{ value: 'qwen_custom_voice', label: 'Qwen CustomVoice' },
|
||||||
{ value: 'luxtts', label: 'LuxTTS' },
|
{ value: 'luxtts', label: 'LuxTTS' },
|
||||||
{ value: 'chatterbox', label: 'Chatterbox' },
|
{ value: 'chatterbox', label: 'Chatterbox' },
|
||||||
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
|
{ value: 'chatterbox_turbo', label: 'Chatterbox Turbo' },
|
||||||
@@ -849,6 +850,7 @@ export function ProfileForm() {
|
|||||||
</FormControl>
|
</FormControl>
|
||||||
<SelectContent>
|
<SelectContent>
|
||||||
<SelectItem value="kokoro">Kokoro 82M</SelectItem>
|
<SelectItem value="kokoro">Kokoro 82M</SelectItem>
|
||||||
|
<SelectItem value="qwen_custom_voice">Qwen CustomVoice</SelectItem>
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
</FormItem>
|
</FormItem>
|
||||||
|
|||||||
@@ -7,11 +7,12 @@ import { ProfileCard } from './ProfileCard';
|
|||||||
import { ProfileForm } from './ProfileForm';
|
import { ProfileForm } from './ProfileForm';
|
||||||
|
|
||||||
/** Engines that use preset (built-in) voices instead of cloned profiles. */
|
/** Engines that use preset (built-in) voices instead of cloned profiles. */
|
||||||
const PRESET_ENGINES = new Set(['kokoro']);
|
const PRESET_ENGINES = new Set(['kokoro', 'qwen_custom_voice']);
|
||||||
|
|
||||||
/** Human-readable engine names for empty state messages. */
|
/** Human-readable engine names for empty state messages. */
|
||||||
const ENGINE_NAMES: Record<string, string> = {
|
const ENGINE_NAMES: Record<string, string> = {
|
||||||
kokoro: 'Kokoro',
|
kokoro: 'Kokoro',
|
||||||
|
qwen_custom_voice: 'Qwen CustomVoice',
|
||||||
};
|
};
|
||||||
|
|
||||||
export function ProfileList() {
|
export function ProfileList() {
|
||||||
|
|||||||
@@ -62,7 +62,14 @@ export interface GenerationRequest {
|
|||||||
language: LanguageCode;
|
language: LanguageCode;
|
||||||
seed?: number;
|
seed?: number;
|
||||||
model_size?: '1.7B' | '0.6B' | '1B' | '3B';
|
model_size?: '1.7B' | '0.6B' | '1B' | '3B';
|
||||||
engine?: 'qwen' | 'luxtts' | 'chatterbox' | 'chatterbox_turbo' | 'tada' | 'kokoro';
|
engine?:
|
||||||
|
| 'qwen'
|
||||||
|
| 'qwen_custom_voice'
|
||||||
|
| 'luxtts'
|
||||||
|
| 'chatterbox'
|
||||||
|
| 'chatterbox_turbo'
|
||||||
|
| 'tada'
|
||||||
|
| 'kokoro';
|
||||||
instruct?: string;
|
instruct?: string;
|
||||||
max_chunk_chars?: number;
|
max_chunk_chars?: number;
|
||||||
crossfade_ms?: number;
|
crossfade_ms?: number;
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ export const ENGINE_LANGUAGES: Record<string, readonly LanguageCode[]> = {
|
|||||||
chatterbox_turbo: ['en'],
|
chatterbox_turbo: ['en'],
|
||||||
tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'],
|
tada: ['en', 'ar', 'zh', 'de', 'es', 'fr', 'it', 'ja', 'pl', 'pt'],
|
||||||
kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'],
|
kokoro: ['en', 'es', 'fr', 'hi', 'it', 'pt', 'ja', 'zh'],
|
||||||
|
qwen_custom_voice: ['zh', 'en', 'ja', 'ko', 'de', 'fr', 'ru', 'pt', 'es', 'it'],
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/** Helper: get language options for a given engine. */
|
/** Helper: get language options for a given engine. */
|
||||||
|
|||||||
@@ -17,7 +17,17 @@ const generationSchema = z.object({
|
|||||||
seed: z.number().int().optional(),
|
seed: z.number().int().optional(),
|
||||||
modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(),
|
modelSize: z.enum(['1.7B', '0.6B', '1B', '3B']).optional(),
|
||||||
instruct: z.string().max(500).optional(),
|
instruct: z.string().max(500).optional(),
|
||||||
engine: z.enum(['qwen', 'luxtts', 'chatterbox', 'chatterbox_turbo', 'tada', 'kokoro']).optional(),
|
engine: z
|
||||||
|
.enum([
|
||||||
|
'qwen',
|
||||||
|
'qwen_custom_voice',
|
||||||
|
'luxtts',
|
||||||
|
'chatterbox',
|
||||||
|
'chatterbox_turbo',
|
||||||
|
'tada',
|
||||||
|
'kokoro',
|
||||||
|
])
|
||||||
|
.optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
export type GenerationFormValues = z.infer<typeof generationSchema>;
|
||||||
@@ -85,7 +95,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
|||||||
: 'tada-1b'
|
: 'tada-1b'
|
||||||
: engine === 'kokoro'
|
: engine === 'kokoro'
|
||||||
? 'kokoro'
|
? 'kokoro'
|
||||||
: `qwen-tts-${data.modelSize}`;
|
: engine === 'qwen_custom_voice'
|
||||||
|
? `qwen-custom-voice-${data.modelSize}`
|
||||||
|
: `qwen-tts-${data.modelSize}`;
|
||||||
const displayName =
|
const displayName =
|
||||||
engine === 'luxtts'
|
engine === 'luxtts'
|
||||||
? 'LuxTTS'
|
? 'LuxTTS'
|
||||||
@@ -99,9 +111,13 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
|||||||
: 'TADA 1B'
|
: 'TADA 1B'
|
||||||
: engine === 'kokoro'
|
: engine === 'kokoro'
|
||||||
? 'Kokoro 82M'
|
? 'Kokoro 82M'
|
||||||
: data.modelSize === '1.7B'
|
: engine === 'qwen_custom_voice'
|
||||||
? 'Qwen TTS 1.7B'
|
? data.modelSize === '1.7B'
|
||||||
: 'Qwen TTS 0.6B';
|
? 'Qwen CustomVoice 1.7B'
|
||||||
|
: 'Qwen CustomVoice 0.6B'
|
||||||
|
: data.modelSize === '1.7B'
|
||||||
|
? 'Qwen TTS 1.7B'
|
||||||
|
: 'Qwen TTS 0.6B';
|
||||||
|
|
||||||
// Check if model needs downloading
|
// Check if model needs downloading
|
||||||
try {
|
try {
|
||||||
@@ -116,7 +132,9 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
|||||||
console.error('Failed to check model status:', error);
|
console.error('Failed to check model status:', error);
|
||||||
}
|
}
|
||||||
|
|
||||||
const hasModelSizes = engine === 'qwen' || engine === 'tada';
|
const hasModelSizes =
|
||||||
|
engine === 'qwen' || engine === 'qwen_custom_voice' || engine === 'tada';
|
||||||
|
const supportsInstruct = engine === 'qwen' || engine === 'qwen_custom_voice';
|
||||||
const effectsChain = options.getEffectsChain?.();
|
const effectsChain = options.getEffectsChain?.();
|
||||||
// This now returns immediately with status="generating"
|
// This now returns immediately with status="generating"
|
||||||
const result = await generation.mutateAsync({
|
const result = await generation.mutateAsync({
|
||||||
@@ -126,7 +144,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
|
|||||||
seed: data.seed,
|
seed: data.seed,
|
||||||
model_size: hasModelSizes ? data.modelSize : undefined,
|
model_size: hasModelSizes ? data.modelSize : undefined,
|
||||||
engine,
|
engine,
|
||||||
instruct: engine === 'qwen' ? data.instruct || undefined : undefined,
|
instruct: supportsInstruct ? data.instruct || undefined : undefined,
|
||||||
max_chunk_chars: maxChunkChars,
|
max_chunk_chars: maxChunkChars,
|
||||||
crossfade_ms: crossfadeMs,
|
crossfade_ms: crossfadeMs,
|
||||||
normalize: normalizeAudio,
|
normalize: normalizeAudio,
|
||||||
|
|||||||
@@ -163,6 +163,7 @@ _stt_backend: Optional[STTBackend] = None
|
|||||||
# The factory function uses this for the if/elif chain; the model configs live on the backend classes.
|
# The factory function uses this for the if/elif chain; the model configs live on the backend classes.
|
||||||
TTS_ENGINES = {
|
TTS_ENGINES = {
|
||||||
"qwen": "Qwen TTS",
|
"qwen": "Qwen TTS",
|
||||||
|
"qwen_custom_voice": "Qwen CustomVoice",
|
||||||
"luxtts": "LuxTTS",
|
"luxtts": "LuxTTS",
|
||||||
"chatterbox": "Chatterbox TTS",
|
"chatterbox": "Chatterbox TTS",
|
||||||
"chatterbox_turbo": "Chatterbox Turbo",
|
"chatterbox_turbo": "Chatterbox Turbo",
|
||||||
@@ -205,6 +206,32 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_qwen_custom_voice_configs() -> list[ModelConfig]:
|
||||||
|
"""Return Qwen CustomVoice model configs."""
|
||||||
|
return [
|
||||||
|
ModelConfig(
|
||||||
|
model_name="qwen-custom-voice-1.7B",
|
||||||
|
display_name="Qwen CustomVoice 1.7B",
|
||||||
|
engine="qwen_custom_voice",
|
||||||
|
hf_repo_id="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||||
|
model_size="1.7B",
|
||||||
|
size_mb=3500,
|
||||||
|
supports_instruct=True,
|
||||||
|
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||||
|
),
|
||||||
|
ModelConfig(
|
||||||
|
model_name="qwen-custom-voice-0.6B",
|
||||||
|
display_name="Qwen CustomVoice 0.6B",
|
||||||
|
engine="qwen_custom_voice",
|
||||||
|
hf_repo_id="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
|
||||||
|
model_size="0.6B",
|
||||||
|
size_mb=1200,
|
||||||
|
supports_instruct=True,
|
||||||
|
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _get_non_qwen_tts_configs() -> list[ModelConfig]:
|
def _get_non_qwen_tts_configs() -> list[ModelConfig]:
|
||||||
"""Return model configs for non-Qwen TTS engines.
|
"""Return model configs for non-Qwen TTS engines.
|
||||||
|
|
||||||
@@ -333,12 +360,12 @@ def _get_whisper_configs() -> list[ModelConfig]:
|
|||||||
|
|
||||||
def get_all_model_configs() -> list[ModelConfig]:
|
def get_all_model_configs() -> list[ModelConfig]:
|
||||||
"""Return the full list of model configs (TTS + STT)."""
|
"""Return the full list of model configs (TTS + STT)."""
|
||||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
|
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs() + _get_whisper_configs()
|
||||||
|
|
||||||
|
|
||||||
def get_tts_model_configs() -> list[ModelConfig]:
|
def get_tts_model_configs() -> list[ModelConfig]:
|
||||||
"""Return only TTS model configs."""
|
"""Return only TTS model configs."""
|
||||||
return _get_qwen_model_configs() + _get_non_qwen_tts_configs()
|
return _get_qwen_model_configs() + _get_qwen_custom_voice_configs() + _get_non_qwen_tts_configs()
|
||||||
|
|
||||||
|
|
||||||
# Lookup helpers — these replace the if/elif chains in main.py
|
# Lookup helpers — these replace the if/elif chains in main.py
|
||||||
@@ -369,7 +396,7 @@ def engine_has_model_sizes(engine: str) -> bool:
|
|||||||
async def load_engine_model(engine: str, model_size: str = "default") -> None:
|
async def load_engine_model(engine: str, model_size: str = "default") -> None:
|
||||||
"""Load a model for the given engine, handling engines with multiple model sizes."""
|
"""Load a model for the given engine, handling engines with multiple model sizes."""
|
||||||
backend = get_tts_backend_for_engine(engine)
|
backend = get_tts_backend_for_engine(engine)
|
||||||
if engine == "qwen":
|
if engine in ("qwen", "qwen_custom_voice"):
|
||||||
await backend.load_model_async(model_size)
|
await backend.load_model_async(model_size)
|
||||||
elif engine == "tada":
|
elif engine == "tada":
|
||||||
await backend.load_model(model_size)
|
await backend.load_model(model_size)
|
||||||
@@ -388,7 +415,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default")
|
|||||||
cfg = c
|
cfg = c
|
||||||
break
|
break
|
||||||
|
|
||||||
if engine in ("qwen", "tada"):
|
if engine in ("qwen", "qwen_custom_voice", "tada"):
|
||||||
if not backend._is_model_cached(model_size):
|
if not backend._is_model_cached(model_size):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
@@ -423,6 +450,14 @@ def unload_model_by_config(config: ModelConfig) -> bool:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if config.engine == "qwen_custom_voice":
|
||||||
|
backend = get_tts_backend_for_engine(config.engine)
|
||||||
|
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
|
||||||
|
if backend.is_loaded() and loaded_size == config.model_size:
|
||||||
|
backend.unload_model()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
# All other TTS engines
|
# All other TTS engines
|
||||||
backend = get_tts_backend_for_engine(config.engine)
|
backend = get_tts_backend_for_engine(config.engine)
|
||||||
if backend.is_loaded():
|
if backend.is_loaded():
|
||||||
@@ -446,6 +481,11 @@ def check_model_loaded(config: ModelConfig) -> bool:
|
|||||||
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
|
loaded_size = getattr(tts_model, "_current_model_size", None) or getattr(tts_model, "model_size", None)
|
||||||
return tts_model.is_loaded() and loaded_size == config.model_size
|
return tts_model.is_loaded() and loaded_size == config.model_size
|
||||||
|
|
||||||
|
if config.engine == "qwen_custom_voice":
|
||||||
|
backend = get_tts_backend_for_engine(config.engine)
|
||||||
|
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
|
||||||
|
return backend.is_loaded() and loaded_size == config.model_size
|
||||||
|
|
||||||
backend = get_tts_backend_for_engine(config.engine)
|
backend = get_tts_backend_for_engine(config.engine)
|
||||||
return backend.is_loaded()
|
return backend.is_loaded()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -463,6 +503,9 @@ def get_model_load_func(config: ModelConfig):
|
|||||||
if config.engine == "qwen":
|
if config.engine == "qwen":
|
||||||
return lambda: tts.get_tts_model().load_model(config.model_size)
|
return lambda: tts.get_tts_model().load_model(config.model_size)
|
||||||
|
|
||||||
|
if config.engine == "qwen_custom_voice":
|
||||||
|
return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size)
|
||||||
|
|
||||||
return lambda: get_tts_backend_for_engine(config.engine).load_model()
|
return lambda: get_tts_backend_for_engine(config.engine).load_model()
|
||||||
|
|
||||||
|
|
||||||
@@ -528,6 +571,10 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend:
|
|||||||
from .kokoro_backend import KokoroTTSBackend
|
from .kokoro_backend import KokoroTTSBackend
|
||||||
|
|
||||||
backend = KokoroTTSBackend()
|
backend = KokoroTTSBackend()
|
||||||
|
elif engine == "qwen_custom_voice":
|
||||||
|
from .qwen_custom_voice_backend import QwenCustomVoiceBackend
|
||||||
|
|
||||||
|
backend = QwenCustomVoiceBackend()
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
raise ValueError(f"Unknown TTS engine: {engine}. Supported: {list(TTS_ENGINES.keys())}")
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""
|
||||||
|
Qwen3-TTS CustomVoice backend implementation.
|
||||||
|
|
||||||
|
Wraps the Qwen3-TTS-12Hz CustomVoice model for preset-speaker TTS with
|
||||||
|
instruction-based style control. Uses the same qwen_tts library as the
|
||||||
|
Base model (pytorch_backend.py) but loads a different checkpoint and
|
||||||
|
calls generate_custom_voice() instead of generate_voice_clone().
|
||||||
|
|
||||||
|
Key differences from the Base engine:
|
||||||
|
- Uses preset speakers (9 built-in voices) instead of zero-shot cloning
|
||||||
|
- Supports instruct parameter for tone/emotion/prosody control
|
||||||
|
- Two model sizes: 1.7B and 0.6B
|
||||||
|
|
||||||
|
Languages supported: zh, en, ja, ko, de, fr, ru, pt, es, it
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from . import TTSBackend, LANGUAGE_CODE_TO_NAME
|
||||||
|
from .base import (
|
||||||
|
is_model_cached,
|
||||||
|
get_torch_device,
|
||||||
|
combine_voice_prompts as _combine_voice_prompts,
|
||||||
|
model_load_progress,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ── Preset speakers ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# (speaker_id, display_name, gender, native_language_code, description)
|
||||||
|
QWEN_CUSTOM_VOICES = [
|
||||||
|
("Vivian", "Vivian", "female", "zh", "Bright, slightly edgy young female voice"),
|
||||||
|
("Serena", "Serena", "female", "zh", "Warm, gentle young female voice"),
|
||||||
|
("Uncle_Fu", "Uncle Fu", "male", "zh", "Seasoned male voice with a low, mellow timbre"),
|
||||||
|
("Dylan", "Dylan", "male", "zh", "Youthful Beijing male voice with a clear, natural timbre"),
|
||||||
|
("Eric", "Eric", "male", "zh", "Lively Chengdu male voice with a slightly husky brightness"),
|
||||||
|
("Ryan", "Ryan", "male", "en", "Dynamic male voice with strong rhythmic drive"),
|
||||||
|
("Aiden", "Aiden", "male", "en", "Sunny American male voice with a clear midrange"),
|
||||||
|
("Ono_Anna", "Ono Anna", "female", "ja", "Playful Japanese female voice with a light, nimble timbre"),
|
||||||
|
("Sohee", "Sohee", "female", "ko", "Warm Korean female voice with rich emotion"),
|
||||||
|
]
|
||||||
|
|
||||||
|
QWEN_CV_DEFAULT_SPEAKER = "Ryan"
|
||||||
|
|
||||||
|
# HuggingFace repo IDs per model size
|
||||||
|
QWEN_CV_HF_REPOS = {
|
||||||
|
"1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||||
|
"0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class QwenCustomVoiceBackend:
|
||||||
|
"""Qwen3-TTS CustomVoice backend — preset speakers with instruct control."""
|
||||||
|
|
||||||
|
def __init__(self, model_size: str = "1.7B"):
|
||||||
|
self.model = None
|
||||||
|
self.model_size = model_size
|
||||||
|
self.device = self._get_device()
|
||||||
|
self._current_model_size: Optional[str] = None
|
||||||
|
|
||||||
|
def _get_device(self) -> str:
|
||||||
|
return get_torch_device(allow_xpu=True, allow_directml=True)
|
||||||
|
|
||||||
|
def is_loaded(self) -> bool:
|
||||||
|
return self.model is not None
|
||||||
|
|
||||||
|
def _get_model_path(self, model_size: str) -> str:
|
||||||
|
if model_size not in QWEN_CV_HF_REPOS:
|
||||||
|
raise ValueError(f"Unknown model size: {model_size}")
|
||||||
|
return QWEN_CV_HF_REPOS[model_size]
|
||||||
|
|
||||||
|
def _is_model_cached(self, model_size: Optional[str] = None) -> bool:
|
||||||
|
size = model_size or self.model_size
|
||||||
|
return is_model_cached(self._get_model_path(size))
|
||||||
|
|
||||||
|
async def load_model_async(self, model_size: Optional[str] = None) -> None:
|
||||||
|
if model_size is None:
|
||||||
|
model_size = self.model_size
|
||||||
|
|
||||||
|
if self.model is not None and self._current_model_size == model_size:
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.model is not None and self._current_model_size != model_size:
|
||||||
|
self.unload_model()
|
||||||
|
|
||||||
|
await asyncio.to_thread(self._load_model_sync, model_size)
|
||||||
|
|
||||||
|
# Alias for compatibility with the TTSBackend protocol
|
||||||
|
load_model = load_model_async
|
||||||
|
|
||||||
|
def _load_model_sync(self, model_size: str) -> None:
|
||||||
|
model_name = f"qwen-custom-voice-{model_size}"
|
||||||
|
is_cached = self._is_model_cached(model_size)
|
||||||
|
|
||||||
|
with model_load_progress(model_name, is_cached):
|
||||||
|
from qwen_tts import Qwen3TTSModel
|
||||||
|
|
||||||
|
model_path = self._get_model_path(model_size)
|
||||||
|
logger.info("Loading Qwen CustomVoice %s on %s...", model_size, self.device)
|
||||||
|
|
||||||
|
if self.device == "cpu":
|
||||||
|
self.model = Qwen3TTSModel.from_pretrained(
|
||||||
|
model_path,
|
||||||
|
torch_dtype=torch.float32,
|
||||||
|
low_cpu_mem_usage=False,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.model = Qwen3TTSModel.from_pretrained(
|
||||||
|
model_path,
|
||||||
|
device_map=self.device,
|
||||||
|
torch_dtype=torch.bfloat16,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._current_model_size = model_size
|
||||||
|
self.model_size = model_size
|
||||||
|
logger.info("Qwen CustomVoice %s loaded successfully", model_size)
|
||||||
|
|
||||||
|
def unload_model(self) -> None:
|
||||||
|
if self.model is not None:
|
||||||
|
del self.model
|
||||||
|
self.model = None
|
||||||
|
self._current_model_size = None
|
||||||
|
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
torch.cuda.empty_cache()
|
||||||
|
|
||||||
|
logger.info("Qwen CustomVoice unloaded")
|
||||||
|
|
||||||
|
async def create_voice_prompt(
|
||||||
|
self,
|
||||||
|
audio_path: str,
|
||||||
|
reference_text: str,
|
||||||
|
use_cache: bool = True,
|
||||||
|
) -> tuple[dict, bool]:
|
||||||
|
"""
|
||||||
|
Create voice prompt for CustomVoice.
|
||||||
|
|
||||||
|
CustomVoice doesn't use reference audio — it uses preset speakers.
|
||||||
|
When called for a cloned profile (fallback), uses the default speaker.
|
||||||
|
For preset profiles, the voice_prompt dict is built by the profile
|
||||||
|
service and bypasses this method entirely.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"voice_type": "preset",
|
||||||
|
"preset_engine": "qwen_custom_voice",
|
||||||
|
"preset_voice_id": QWEN_CV_DEFAULT_SPEAKER,
|
||||||
|
}, False
|
||||||
|
|
||||||
|
async def combine_voice_prompts(
|
||||||
|
self,
|
||||||
|
audio_paths: list[str],
|
||||||
|
reference_texts: list[str],
|
||||||
|
) -> tuple[np.ndarray, str]:
|
||||||
|
return await _combine_voice_prompts(audio_paths, reference_texts)
|
||||||
|
|
||||||
|
async def generate(
|
||||||
|
self,
|
||||||
|
text: str,
|
||||||
|
voice_prompt: dict,
|
||||||
|
language: str = "en",
|
||||||
|
seed: Optional[int] = None,
|
||||||
|
instruct: Optional[str] = None,
|
||||||
|
) -> tuple[np.ndarray, int]:
|
||||||
|
"""
|
||||||
|
Generate audio using Qwen CustomVoice.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text to synthesize
|
||||||
|
voice_prompt: Dict with preset_voice_id (speaker name)
|
||||||
|
language: Language code (zh, en, ja, ko, etc.)
|
||||||
|
seed: Random seed for reproducibility
|
||||||
|
instruct: Natural language instruction for style control
|
||||||
|
(e.g. "Speak in an angry tone", "Very happy")
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (audio_array, sample_rate)
|
||||||
|
"""
|
||||||
|
await self.load_model_async(None)
|
||||||
|
|
||||||
|
speaker = voice_prompt.get("preset_voice_id") or QWEN_CV_DEFAULT_SPEAKER
|
||||||
|
|
||||||
|
def _generate_sync():
|
||||||
|
if seed is not None:
|
||||||
|
torch.manual_seed(seed)
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
torch.cuda.manual_seed(seed)
|
||||||
|
|
||||||
|
lang_name = LANGUAGE_CODE_TO_NAME.get(language, "auto")
|
||||||
|
|
||||||
|
kwargs = {
|
||||||
|
"text": text,
|
||||||
|
"language": lang_name.capitalize() if lang_name != "auto" else "Auto",
|
||||||
|
"speaker": speaker,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Only pass instruct if non-empty
|
||||||
|
if instruct:
|
||||||
|
kwargs["instruct"] = instruct
|
||||||
|
|
||||||
|
wavs, sample_rate = self.model.generate_custom_voice(**kwargs)
|
||||||
|
return wavs[0], sample_rate
|
||||||
|
|
||||||
|
audio, sample_rate = await asyncio.to_thread(_generate_sync)
|
||||||
|
return audio, sample_rate
|
||||||
@@ -86,6 +86,8 @@ def build_server(cuda=False):
|
|||||||
"--hidden-import",
|
"--hidden-import",
|
||||||
"backend.backends.pytorch_backend",
|
"backend.backends.pytorch_backend",
|
||||||
"--hidden-import",
|
"--hidden-import",
|
||||||
|
"backend.backends.qwen_custom_voice_backend",
|
||||||
|
"--hidden-import",
|
||||||
"backend.utils.audio",
|
"backend.utils.audio",
|
||||||
"--hidden-import",
|
"--hidden-import",
|
||||||
"backend.utils.cache",
|
"backend.utils.cache",
|
||||||
|
|||||||
+1
-1
@@ -78,7 +78,7 @@ class GenerationRequest(BaseModel):
|
|||||||
seed: Optional[int] = Field(None, ge=0)
|
seed: Optional[int] = Field(None, ge=0)
|
||||||
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$")
|
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B|1B|3B)$")
|
||||||
instruct: Optional[str] = Field(None, max_length=500)
|
instruct: Optional[str] = Field(None, max_length=500)
|
||||||
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$")
|
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|qwen_custom_voice|luxtts|chatterbox|chatterbox_turbo|tada|kokoro)$")
|
||||||
max_chunk_chars: int = Field(
|
max_chunk_chars: int = Field(
|
||||||
default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting"
|
default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -90,6 +90,21 @@ async def list_preset_voices(engine: str):
|
|||||||
for vid, name, gender, lang in KOKORO_VOICES
|
for vid, name, gender, lang in KOKORO_VOICES
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
if engine == "qwen_custom_voice":
|
||||||
|
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
|
||||||
|
|
||||||
|
return {
|
||||||
|
"engine": engine,
|
||||||
|
"voices": [
|
||||||
|
{
|
||||||
|
"voice_id": speaker_id,
|
||||||
|
"name": display_name,
|
||||||
|
"gender": gender,
|
||||||
|
"language": lang,
|
||||||
|
}
|
||||||
|
for speaker_id, display_name, gender, lang, _desc in QWEN_CUSTOM_VOICES
|
||||||
|
],
|
||||||
|
}
|
||||||
return {"engine": engine, "voices": []}
|
return {"engine": engine, "voices": []}
|
||||||
|
|
||||||
|
|
||||||
@@ -103,9 +118,15 @@ async def seed_preset_profiles_route(
|
|||||||
Creates profiles for all available preset voices that don't already exist.
|
Creates profiles for all available preset voices that don't already exist.
|
||||||
Returns the count of newly created profiles.
|
Returns the count of newly created profiles.
|
||||||
"""
|
"""
|
||||||
if engine != "kokoro":
|
if engine == "kokoro":
|
||||||
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}")
|
return _seed_kokoro_presets(db)
|
||||||
|
if engine == "qwen_custom_voice":
|
||||||
|
return _seed_qwen_custom_voice_presets(db)
|
||||||
|
raise HTTPException(status_code=400, detail=f"No presets available for engine: {engine}")
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_kokoro_presets(db: Session):
|
||||||
|
"""Seed Kokoro preset profiles."""
|
||||||
try:
|
try:
|
||||||
from ..backends.kokoro_backend import KOKORO_VOICES
|
from ..backends.kokoro_backend import KOKORO_VOICES
|
||||||
|
|
||||||
@@ -154,12 +175,57 @@ async def seed_preset_profiles_route(
|
|||||||
db.commit()
|
db.commit()
|
||||||
logger.info(f"Seeded {created} Kokoro preset profiles")
|
logger.info(f"Seeded {created} Kokoro preset profiles")
|
||||||
|
|
||||||
return {"engine": engine, "created": created, "total_available": len(KOKORO_VOICES)}
|
return {"engine": "kokoro", "created": created, "total_available": len(KOKORO_VOICES)}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Failed to seed Kokoro profiles: {e}")
|
logger.exception(f"Failed to seed Kokoro profiles: {e}")
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_qwen_custom_voice_presets(db: Session):
|
||||||
|
"""Seed Qwen CustomVoice preset profiles."""
|
||||||
|
try:
|
||||||
|
from ..backends.qwen_custom_voice_backend import QWEN_CUSTOM_VOICES
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
for speaker_id, display_name, gender, lang, description in QWEN_CUSTOM_VOICES:
|
||||||
|
# Skip if preset already exists
|
||||||
|
existing = (
|
||||||
|
db.query(DBVoiceProfile)
|
||||||
|
.filter_by(preset_engine="qwen_custom_voice", preset_voice_id=speaker_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Skip name collisions
|
||||||
|
if db.query(DBVoiceProfile).filter_by(name=display_name).first():
|
||||||
|
continue
|
||||||
|
|
||||||
|
profile = DBVoiceProfile(
|
||||||
|
id=str(uuid.uuid4()),
|
||||||
|
name=display_name,
|
||||||
|
description=f"Qwen CustomVoice — {description}",
|
||||||
|
language=lang,
|
||||||
|
voice_type="preset",
|
||||||
|
preset_engine="qwen_custom_voice",
|
||||||
|
preset_voice_id=speaker_id,
|
||||||
|
default_engine="qwen_custom_voice",
|
||||||
|
created_at=datetime.utcnow(),
|
||||||
|
updated_at=datetime.utcnow(),
|
||||||
|
)
|
||||||
|
db.add(profile)
|
||||||
|
created += 1
|
||||||
|
|
||||||
|
if created > 0:
|
||||||
|
db.commit()
|
||||||
|
logger.info(f"Seeded {created} Qwen CustomVoice preset profiles")
|
||||||
|
|
||||||
|
return {"engine": "qwen_custom_voice", "created": created, "total_available": len(QWEN_CUSTOM_VOICES)}
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"Failed to seed Qwen CustomVoice profiles: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
|
@router.get("/profiles/{profile_id}", response_model=models.VoiceProfileResponse)
|
||||||
async def get_profile(
|
async def get_profile(
|
||||||
profile_id: str,
|
profile_id: str,
|
||||||
|
|||||||
Reference in New Issue
Block a user