mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 14:20:42 -07:00
add model loading status, effects preset dropdown, clean up UI
Backend: - Generation service reports 'loading_model' status only when model is not yet in memory, then 'generating' once inference starts - Migrate hf_offline_patch.py from print() to logging module - Update ADDING_TTS_ENGINES.md for post-refactor file paths Frontend: - HistoryTable shows 'Loading model...' vs 'Generating...' based on step - FloatingGenerateBox: replace instruct toggle + inline effects editor with an effects preset dropdown (third dropdown after language and engine) - Instruct UI removed for now (form field preserved for future models) - Remove focus ring from Select component globally
This commit is contained in:
@@ -1,8 +1,8 @@
|
|||||||
|
import { useQuery } from '@tanstack/react-query';
|
||||||
import { useMatchRoute } from '@tanstack/react-router';
|
import { useMatchRoute } from '@tanstack/react-router';
|
||||||
import { AnimatePresence, motion } from 'framer-motion';
|
import { AnimatePresence, motion } from 'framer-motion';
|
||||||
import { Loader2, SlidersHorizontal, Sparkles } from 'lucide-react';
|
import { Loader2, Sparkles } from 'lucide-react';
|
||||||
import { useEffect, useRef, useState } from 'react';
|
import { useEffect, useRef, useState } from 'react';
|
||||||
import { EffectsChainEditor } from '@/components/Effects/EffectsChainEditor';
|
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { Form, FormControl, FormField, FormItem, FormMessage } from '@/components/ui/form';
|
import { Form, FormControl, FormField, FormItem, FormMessage } from '@/components/ui/form';
|
||||||
import {
|
import {
|
||||||
@@ -13,7 +13,7 @@ import {
|
|||||||
SelectValue,
|
SelectValue,
|
||||||
} from '@/components/ui/select';
|
} from '@/components/ui/select';
|
||||||
import { Textarea } from '@/components/ui/textarea';
|
import { Textarea } from '@/components/ui/textarea';
|
||||||
import type { EffectConfig } from '@/lib/api/types';
|
import { apiClient } from '@/lib/api/client';
|
||||||
import { getLanguageOptionsForEngine, type LanguageCode } from '@/lib/constants/languages';
|
import { getLanguageOptionsForEngine, type LanguageCode } from '@/lib/constants/languages';
|
||||||
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
|
||||||
import { useProfile, useProfiles } from '@/lib/hooks/useProfiles';
|
import { useProfile, useProfiles } from '@/lib/hooks/useProfiles';
|
||||||
@@ -39,8 +39,7 @@ export function FloatingGenerateBox({
|
|||||||
const { data: selectedProfile } = useProfile(selectedProfileId || '');
|
const { data: selectedProfile } = useProfile(selectedProfileId || '');
|
||||||
const { data: profiles } = useProfiles();
|
const { data: profiles } = useProfiles();
|
||||||
const [isExpanded, setIsExpanded] = useState(false);
|
const [isExpanded, setIsExpanded] = useState(false);
|
||||||
const [isInstructMode, setIsInstructMode] = useState(false);
|
const [selectedPresetId, setSelectedPresetId] = useState<string | null>(null);
|
||||||
const [effectsChain, setEffectsChain] = useState<EffectConfig[]>([]);
|
|
||||||
const containerRef = useRef<HTMLDivElement>(null);
|
const containerRef = useRef<HTMLDivElement>(null);
|
||||||
const textareaRef = useRef<HTMLTextAreaElement | null>(null);
|
const textareaRef = useRef<HTMLTextAreaElement | null>(null);
|
||||||
const matchRoute = useMatchRoute();
|
const matchRoute = useMatchRoute();
|
||||||
@@ -50,18 +49,28 @@ export function FloatingGenerateBox({
|
|||||||
const { data: currentStory } = useStory(selectedStoryId);
|
const { data: currentStory } = useStory(selectedStoryId);
|
||||||
const addPendingStoryAdd = useGenerationStore((s) => s.addPendingStoryAdd);
|
const addPendingStoryAdd = useGenerationStore((s) => s.addPendingStoryAdd);
|
||||||
|
|
||||||
|
// Fetch effect presets for the dropdown
|
||||||
|
const { data: effectPresets } = useQuery({
|
||||||
|
queryKey: ['effectPresets'],
|
||||||
|
queryFn: () => apiClient.listEffectPresets(),
|
||||||
|
});
|
||||||
|
|
||||||
// Calculate if track editor is visible (on stories route with items)
|
// Calculate if track editor is visible (on stories route with items)
|
||||||
const hasTrackEditor = isStoriesRoute && currentStory && currentStory.items.length > 0;
|
const hasTrackEditor = isStoriesRoute && currentStory && currentStory.items.length > 0;
|
||||||
|
|
||||||
const { form, handleSubmit, isPending } = useGenerationForm({
|
const { form, handleSubmit, isPending } = useGenerationForm({
|
||||||
onSuccess: async (generationId) => {
|
onSuccess: async (generationId) => {
|
||||||
setIsExpanded(false);
|
setIsExpanded(false);
|
||||||
// Defer the story add until TTS completes — useGenerationProgress handles it
|
// Defer the story add until TTS completes -- useGenerationProgress handles it
|
||||||
if (isStoriesRoute && selectedStoryId && generationId) {
|
if (isStoriesRoute && selectedStoryId && generationId) {
|
||||||
addPendingStoryAdd(generationId, selectedStoryId);
|
addPendingStoryAdd(generationId, selectedStoryId);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
getEffectsChain: () => (effectsChain.length > 0 ? effectsChain : undefined),
|
getEffectsChain: () => {
|
||||||
|
if (!selectedPresetId || !effectPresets) return undefined;
|
||||||
|
const preset = effectPresets.find((p) => p.id === selectedPresetId);
|
||||||
|
return preset?.effects_chain;
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Click away handler to collapse the box
|
// Click away handler to collapse the box
|
||||||
@@ -189,111 +198,57 @@ export function FloatingGenerateBox({
|
|||||||
<Form {...form}>
|
<Form {...form}>
|
||||||
<form onSubmit={form.handleSubmit(onSubmit)}>
|
<form onSubmit={form.handleSubmit(onSubmit)}>
|
||||||
<div className="flex gap-2">
|
<div className="flex gap-2">
|
||||||
<motion.div
|
<motion.div className="flex-1" transition={{ duration: 0.3, ease: 'easeOut' }}>
|
||||||
className={cn('flex-1', isExpanded && 'mr-12')}
|
<FormField
|
||||||
transition={{ duration: 0.3, ease: 'easeOut' }}
|
control={form.control}
|
||||||
>
|
name="text"
|
||||||
{/* Text field - hidden when in instruct mode */}
|
render={({ field }) => (
|
||||||
<div style={{ display: isInstructMode ? 'none' : 'block' }}>
|
<FormItem>
|
||||||
<FormField
|
<FormControl>
|
||||||
control={form.control}
|
<motion.div
|
||||||
name="text"
|
animate={{
|
||||||
render={({ field }) => (
|
height: isExpanded ? 'auto' : '32px',
|
||||||
<FormItem>
|
}}
|
||||||
<FormControl>
|
transition={{ duration: 0.15, ease: 'easeOut' }}
|
||||||
<motion.div
|
style={{ overflow: 'hidden' }}
|
||||||
animate={{
|
>
|
||||||
height: isExpanded ? 'auto' : '32px',
|
{form.watch('engine') === 'chatterbox_turbo' ? (
|
||||||
}}
|
<ParalinguisticInput
|
||||||
transition={{ duration: 0.15, ease: 'easeOut' }}
|
value={field.value}
|
||||||
style={{ overflow: 'hidden' }}
|
onChange={field.onChange}
|
||||||
>
|
placeholder={
|
||||||
{form.watch('engine') === 'chatterbox_turbo' ? (
|
isStoriesRoute && currentStory
|
||||||
<ParalinguisticInput
|
? `Generate speech for "${currentStory.name}"... (type / for effects)`
|
||||||
value={field.value}
|
: selectedProfile
|
||||||
onChange={field.onChange}
|
? `Type / for effects like [laugh], [sigh]...`
|
||||||
placeholder={
|
: 'Select a voice profile above...'
|
||||||
isStoriesRoute && currentStory
|
}
|
||||||
? `Generate speech for "${currentStory.name}"... (type / for effects)`
|
className="px-3 py-2 resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm w-full"
|
||||||
: selectedProfile
|
style={{
|
||||||
? `Type / for effects like [laugh], [sigh]...`
|
minHeight: isExpanded ? '100px' : '32px',
|
||||||
: 'Select a voice profile above...'
|
maxHeight: '300px',
|
||||||
}
|
overflowY: 'auto',
|
||||||
className="px-3 py-2 resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm w-full"
|
}}
|
||||||
style={{
|
disabled={!selectedProfileId}
|
||||||
minHeight: isExpanded ? '100px' : '32px',
|
onClick={() => setIsExpanded(true)}
|
||||||
maxHeight: '300px',
|
onFocus={() => setIsExpanded(true)}
|
||||||
overflowY: 'auto',
|
/>
|
||||||
}}
|
) : (
|
||||||
disabled={!selectedProfileId}
|
|
||||||
onClick={() => setIsExpanded(true)}
|
|
||||||
onFocus={() => setIsExpanded(true)}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<Textarea
|
|
||||||
{...field}
|
|
||||||
ref={(node: HTMLTextAreaElement | null) => {
|
|
||||||
// Store ref for auto-resize (only for active field)
|
|
||||||
if (!isInstructMode) {
|
|
||||||
textareaRef.current = node;
|
|
||||||
}
|
|
||||||
// Forward ref to react-hook-form
|
|
||||||
if (typeof field.ref === 'function') {
|
|
||||||
field.ref(node);
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
placeholder={
|
|
||||||
isStoriesRoute && currentStory
|
|
||||||
? `Generate speech for "${currentStory.name}"...`
|
|
||||||
: selectedProfile
|
|
||||||
? `Generate speech using ${selectedProfile.name}...`
|
|
||||||
: 'Select a voice profile above...'
|
|
||||||
}
|
|
||||||
className="resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm placeholder:text-muted-foreground/60 w-full"
|
|
||||||
style={{
|
|
||||||
minHeight: isExpanded ? '100px' : '32px',
|
|
||||||
maxHeight: '300px',
|
|
||||||
}}
|
|
||||||
disabled={!selectedProfileId}
|
|
||||||
onClick={() => setIsExpanded(true)}
|
|
||||||
onFocus={() => setIsExpanded(true)}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</motion.div>
|
|
||||||
</FormControl>
|
|
||||||
<FormMessage className="text-xs" />
|
|
||||||
</FormItem>
|
|
||||||
)}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
{/* Instruct field - hidden when in text mode */}
|
|
||||||
<div style={{ display: isInstructMode ? 'block' : 'none' }}>
|
|
||||||
<FormField
|
|
||||||
control={form.control}
|
|
||||||
name="instruct"
|
|
||||||
render={({ field }) => (
|
|
||||||
<FormItem>
|
|
||||||
<FormControl>
|
|
||||||
<motion.div
|
|
||||||
animate={{
|
|
||||||
height: isExpanded ? 'auto' : '32px',
|
|
||||||
}}
|
|
||||||
transition={{ duration: 0.15, ease: 'easeOut' }}
|
|
||||||
style={{ overflow: 'hidden' }}
|
|
||||||
>
|
|
||||||
<Textarea
|
<Textarea
|
||||||
{...field}
|
{...field}
|
||||||
ref={(node: HTMLTextAreaElement | null) => {
|
ref={(node: HTMLTextAreaElement | null) => {
|
||||||
// Store ref for auto-resize (only for active field)
|
textareaRef.current = node;
|
||||||
if (isInstructMode) {
|
|
||||||
textareaRef.current = node;
|
|
||||||
}
|
|
||||||
// Forward ref to react-hook-form
|
|
||||||
if (typeof field.ref === 'function') {
|
if (typeof field.ref === 'function') {
|
||||||
field.ref(node);
|
field.ref(node);
|
||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
placeholder="e.g. very happy and excited"
|
placeholder={
|
||||||
|
isStoriesRoute && currentStory
|
||||||
|
? `Generate speech for "${currentStory.name}"...`
|
||||||
|
: selectedProfile
|
||||||
|
? `Generate speech using ${selectedProfile.name}...`
|
||||||
|
: 'Select a voice profile above...'
|
||||||
|
}
|
||||||
className="resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm placeholder:text-muted-foreground/60 w-full"
|
className="resize-none bg-transparent border-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:outline-none focus:ring-0 outline-none ring-0 rounded-2xl text-sm placeholder:text-muted-foreground/60 w-full"
|
||||||
style={{
|
style={{
|
||||||
minHeight: isExpanded ? '100px' : '32px',
|
minHeight: isExpanded ? '100px' : '32px',
|
||||||
@@ -303,13 +258,13 @@ export function FloatingGenerateBox({
|
|||||||
onClick={() => setIsExpanded(true)}
|
onClick={() => setIsExpanded(true)}
|
||||||
onFocus={() => setIsExpanded(true)}
|
onFocus={() => setIsExpanded(true)}
|
||||||
/>
|
/>
|
||||||
</motion.div>
|
)}
|
||||||
</FormControl>
|
</motion.div>
|
||||||
<FormMessage className="text-xs" />
|
</FormControl>
|
||||||
</FormItem>
|
<FormMessage className="text-xs" />
|
||||||
)}
|
</FormItem>
|
||||||
/>
|
)}
|
||||||
</div>
|
/>
|
||||||
</motion.div>
|
</motion.div>
|
||||||
|
|
||||||
<div className="relative shrink-0">
|
<div className="relative shrink-0">
|
||||||
@@ -341,62 +296,9 @@ export function FloatingGenerateBox({
|
|||||||
: 'Generate speech'}
|
: 'Generate speech'}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<AnimatePresence>
|
|
||||||
{isExpanded && form.watch('engine') === 'qwen' && (
|
|
||||||
<motion.div
|
|
||||||
initial={{ opacity: 0, scale: 0.8 }}
|
|
||||||
animate={{ opacity: 1, scale: 1 }}
|
|
||||||
exit={{ opacity: 0, scale: 0.8 }}
|
|
||||||
transition={{ duration: 0.2 }}
|
|
||||||
className="absolute top-0 right-[calc(100%+0.5rem)]"
|
|
||||||
>
|
|
||||||
<div className="group relative">
|
|
||||||
<Button
|
|
||||||
type="button"
|
|
||||||
variant="ghost"
|
|
||||||
size="icon"
|
|
||||||
onClick={() => setIsInstructMode(!isInstructMode)}
|
|
||||||
className={cn(
|
|
||||||
'h-10 w-10 rounded-full transition-all duration-200',
|
|
||||||
isInstructMode
|
|
||||||
? 'bg-accent text-accent-foreground border border-accent hover:bg-accent/90'
|
|
||||||
: effectsChain.length > 0
|
|
||||||
? 'bg-accent/50 text-accent-foreground border border-accent/50 hover:bg-accent/70'
|
|
||||||
: 'bg-card border border-border hover:bg-background/50',
|
|
||||||
)}
|
|
||||||
aria-label={
|
|
||||||
isInstructMode ? 'Fine tune instructions, on' : 'Fine tune instructions'
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<SlidersHorizontal className="h-4 w-4" />
|
|
||||||
</Button>
|
|
||||||
<span className="pointer-events-none absolute bottom-full left-1/2 -translate-x-1/2 mb-2 whitespace-nowrap rounded-md bg-popover px-3 py-1.5 text-xs text-popover-foreground border border-border opacity-0 transition-opacity group-hover:opacity-100 z-[9999]">
|
|
||||||
Fine tune instructions & effects
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</motion.div>
|
|
||||||
)}
|
|
||||||
</AnimatePresence>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Effects chain editor panel - shown alongside instruct */}
|
|
||||||
<AnimatePresence>
|
|
||||||
{isExpanded && isInstructMode && (
|
|
||||||
<motion.div
|
|
||||||
initial={{ height: 0, opacity: 0 }}
|
|
||||||
animate={{ height: 'auto', opacity: 1 }}
|
|
||||||
exit={{ height: 0, opacity: 0 }}
|
|
||||||
transition={{ duration: 0.2 }}
|
|
||||||
className="overflow-hidden mt-2"
|
|
||||||
>
|
|
||||||
<div className="border-t border-border/50 pt-2 pb-1">
|
|
||||||
<EffectsChainEditor value={effectsChain} onChange={setEffectsChain} compact />
|
|
||||||
</div>
|
|
||||||
</motion.div>
|
|
||||||
)}
|
|
||||||
</AnimatePresence>
|
|
||||||
|
|
||||||
<AnimatePresence>
|
<AnimatePresence>
|
||||||
<motion.div
|
<motion.div
|
||||||
initial={{ height: 0, opacity: 0 }}
|
initial={{ height: 0, opacity: 0 }}
|
||||||
@@ -458,6 +360,29 @@ export function FloatingGenerateBox({
|
|||||||
<FormItem className="flex-1 space-y-0">
|
<FormItem className="flex-1 space-y-0">
|
||||||
<EngineModelSelector form={form} compact />
|
<EngineModelSelector form={form} compact />
|
||||||
</FormItem>
|
</FormItem>
|
||||||
|
|
||||||
|
<FormItem className="flex-1 space-y-0">
|
||||||
|
<Select
|
||||||
|
value={selectedPresetId || 'none'}
|
||||||
|
onValueChange={(value) =>
|
||||||
|
setSelectedPresetId(value === 'none' ? null : value)
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||||
|
<SelectValue placeholder="No effects" />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="none" className="text-xs">
|
||||||
|
No effects
|
||||||
|
</SelectItem>
|
||||||
|
{effectPresets?.map((preset) => (
|
||||||
|
<SelectItem key={preset.id} value={preset.id} className="text-xs">
|
||||||
|
{preset.name}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</FormItem>
|
||||||
</div>
|
</div>
|
||||||
</motion.div>
|
</motion.div>
|
||||||
</AnimatePresence>
|
</AnimatePresence>
|
||||||
|
|||||||
@@ -394,7 +394,8 @@ export function HistoryTable() {
|
|||||||
>
|
>
|
||||||
{history.map((gen) => {
|
{history.map((gen) => {
|
||||||
const isCurrentlyPlaying = currentAudioId === gen.id && isPlaying;
|
const isCurrentlyPlaying = currentAudioId === gen.id && isPlaying;
|
||||||
const isGenerating = gen.status === 'generating';
|
const isInProgress = gen.status === 'loading_model' || gen.status === 'generating';
|
||||||
|
const isGenerating = isInProgress;
|
||||||
const isFailed = gen.status === 'failed';
|
const isFailed = gen.status === 'failed';
|
||||||
const isPlayable = !isGenerating && !isFailed;
|
const isPlayable = !isGenerating && !isFailed;
|
||||||
const hasVersions = gen.versions && gen.versions.length > 1;
|
const hasVersions = gen.versions && gen.versions.length > 1;
|
||||||
@@ -472,8 +473,10 @@ export function HistoryTable() {
|
|||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
<div className="text-xs text-muted-foreground">
|
<div className="text-xs text-muted-foreground">
|
||||||
{isGenerating ? (
|
{isInProgress ? (
|
||||||
<span className="text-accent">Generating...</span>
|
<span className="text-accent">
|
||||||
|
{gen.status === 'loading_model' ? 'Loading model...' : 'Generating...'}
|
||||||
|
</span>
|
||||||
) : (
|
) : (
|
||||||
formatDate(gen.created_at)
|
formatDate(gen.created_at)
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ const SelectTrigger = React.forwardRef<
|
|||||||
<SelectPrimitive.Trigger
|
<SelectPrimitive.Trigger
|
||||||
ref={ref}
|
ref={ref}
|
||||||
className={cn(
|
className={cn(
|
||||||
'flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1',
|
'flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1',
|
||||||
className,
|
className,
|
||||||
)}
|
)}
|
||||||
{...props}
|
{...props}
|
||||||
|
|||||||
@@ -73,7 +73,7 @@ export interface GenerationResponse {
|
|||||||
instruct?: string;
|
instruct?: string;
|
||||||
engine?: string;
|
engine?: string;
|
||||||
model_size?: string;
|
model_size?: string;
|
||||||
status: 'generating' | 'completed' | 'failed';
|
status: 'loading_model' | 'generating' | 'completed' | 'failed';
|
||||||
error?: string;
|
error?: string;
|
||||||
is_favorited?: boolean;
|
is_favorited?: boolean;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { useServerStore } from '@/stores/serverStore';
|
|||||||
|
|
||||||
interface GenerationStatusEvent {
|
interface GenerationStatusEvent {
|
||||||
id: string;
|
id: string;
|
||||||
status: 'generating' | 'completed' | 'failed' | 'not_found';
|
status: 'loading_model' | 'generating' | 'completed' | 'failed' | 'not_found';
|
||||||
duration?: number;
|
duration?: number;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,12 +55,13 @@ async def run_generation(
|
|||||||
bg_db = next(get_db())
|
bg_db = next(get_db())
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# --- Load model --------------------------------------------------
|
|
||||||
await load_engine_model(engine, model_size)
|
|
||||||
|
|
||||||
tts_model = get_tts_backend_for_engine(engine)
|
tts_model = get_tts_backend_for_engine(engine)
|
||||||
|
|
||||||
# --- Build voice prompt ------------------------------------------
|
if not tts_model.is_loaded():
|
||||||
|
await history.update_generation_status(generation_id, "loading_model", bg_db)
|
||||||
|
|
||||||
|
await load_engine_model(engine, model_size)
|
||||||
|
|
||||||
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
voice_prompt = await profiles.create_voice_prompt_for_profile(
|
||||||
profile_id,
|
profile_id,
|
||||||
bg_db,
|
bg_db,
|
||||||
@@ -68,7 +69,7 @@ async def run_generation(
|
|||||||
engine=engine,
|
engine=engine,
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Inference ---------------------------------------------------
|
await history.update_generation_status(generation_id, "generating", bg_db)
|
||||||
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
||||||
|
|
||||||
gen_kwargs: dict = dict(
|
gen_kwargs: dict = dict(
|
||||||
|
|||||||
@@ -1,24 +1,24 @@
|
|||||||
"""
|
"""Monkey-patch huggingface_hub to force offline mode with cached models.
|
||||||
Monkey patch for huggingface_hub to force offline mode with cached models.
|
|
||||||
This prevents mlx_audio from making network requests when models are already downloaded.
|
Prevents mlx_audio from making network requests when models are already
|
||||||
|
downloaded. Must be imported BEFORE mlx_audio.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def patch_huggingface_hub_offline():
|
def patch_huggingface_hub_offline():
|
||||||
"""
|
"""Monkey-patch huggingface_hub to force offline mode."""
|
||||||
Monkey-patch huggingface_hub to force offline mode.
|
|
||||||
This must be called BEFORE importing mlx_audio.
|
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
import huggingface_hub
|
import huggingface_hub # noqa: F401 -- need the package loaded
|
||||||
from huggingface_hub import constants as hf_constants
|
from huggingface_hub import constants as hf_constants
|
||||||
from huggingface_hub.file_download import _try_to_load_from_cache
|
from huggingface_hub.file_download import _try_to_load_from_cache
|
||||||
|
|
||||||
# Store original function
|
|
||||||
original_try_load = _try_to_load_from_cache
|
original_try_load = _try_to_load_from_cache
|
||||||
|
|
||||||
def _patched_try_to_load_from_cache(
|
def _patched_try_to_load_from_cache(
|
||||||
@@ -28,11 +28,6 @@ def patch_huggingface_hub_offline():
|
|||||||
revision: Optional[str] = None,
|
revision: Optional[str] = None,
|
||||||
repo_type: Optional[str] = None,
|
repo_type: Optional[str] = None,
|
||||||
):
|
):
|
||||||
"""
|
|
||||||
Patched version that forces offline mode.
|
|
||||||
Returns None if not cached (instead of making network request).
|
|
||||||
"""
|
|
||||||
# Always use the original function, but we're already in HF_HUB_OFFLINE mode
|
|
||||||
result = original_try_load(
|
result = original_try_load(
|
||||||
repo_id=repo_id,
|
repo_id=repo_id,
|
||||||
filename=filename,
|
filename=filename,
|
||||||
@@ -42,59 +37,52 @@ def patch_huggingface_hub_offline():
|
|||||||
)
|
)
|
||||||
|
|
||||||
if result is None:
|
if result is None:
|
||||||
# File not in cache - log this for debugging
|
|
||||||
cache_path = Path(hf_constants.HF_HUB_CACHE) / f"models--{repo_id.replace('/', '--')}"
|
cache_path = Path(hf_constants.HF_HUB_CACHE) / f"models--{repo_id.replace('/', '--')}"
|
||||||
print(f"[HF_PATCH] File not cached: {repo_id}/{filename}")
|
logger.debug("file not cached: %s/%s (expected at %s)", repo_id, filename, cache_path)
|
||||||
print(f"[HF_PATCH] Expected at: {cache_path}")
|
|
||||||
else:
|
else:
|
||||||
print(f"[HF_PATCH] Cache hit: {repo_id}/{filename}")
|
logger.debug("cache hit: %s/%s", repo_id, filename)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Replace the function
|
|
||||||
import huggingface_hub.file_download as fd
|
import huggingface_hub.file_download as fd
|
||||||
fd._try_to_load_from_cache = _patched_try_to_load_from_cache
|
|
||||||
|
|
||||||
print("[HF_PATCH] huggingface_hub patched for offline mode")
|
fd._try_to_load_from_cache = _patched_try_to_load_from_cache
|
||||||
|
logger.debug("huggingface_hub patched for offline mode")
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print("[HF_PATCH] huggingface_hub not found, skipping patch")
|
logger.debug("huggingface_hub not available, skipping offline patch")
|
||||||
except Exception as e:
|
except Exception:
|
||||||
print(f"[HF_PATCH] Error patching huggingface_hub: {e}")
|
logger.exception("failed to patch huggingface_hub for offline mode")
|
||||||
|
|
||||||
|
|
||||||
def ensure_original_qwen_config_cached():
|
def ensure_original_qwen_config_cached():
|
||||||
"""
|
"""Symlink the original Qwen repo cache to the MLX community version.
|
||||||
The MLX community model is based on the original Qwen model.
|
|
||||||
mlx_audio may try to fetch config from the original repo.
|
mlx_audio may try to fetch config from the original Qwen repo. If only
|
||||||
We need to ensure that config is available in the cache.
|
the MLX community variant is cached, create a symlink so the cache lookup
|
||||||
"""
|
succeeds without a network request.
|
||||||
from huggingface_hub import constants as hf_constants
|
"""
|
||||||
|
try:
|
||||||
|
from huggingface_hub import constants as hf_constants
|
||||||
|
except ImportError:
|
||||||
|
return
|
||||||
|
|
||||||
# Original Qwen model that mlx_audio might reference
|
|
||||||
original_repo = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
original_repo = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||||
mlx_repo = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
mlx_repo = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
|
||||||
|
|
||||||
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
cache_dir = Path(hf_constants.HF_HUB_CACHE)
|
||||||
|
|
||||||
original_path = cache_dir / f"models--{original_repo.replace('/', '--')}"
|
original_path = cache_dir / f"models--{original_repo.replace('/', '--')}"
|
||||||
mlx_path = cache_dir / f"models--{mlx_repo.replace('/', '--')}"
|
mlx_path = cache_dir / f"models--{mlx_repo.replace('/', '--')}"
|
||||||
|
|
||||||
# If original repo cache doesn't exist but MLX does, create a symlink or copy config
|
|
||||||
if not original_path.exists() and mlx_path.exists():
|
if not original_path.exists() and mlx_path.exists():
|
||||||
print(f"[HF_PATCH] Original repo not cached, but MLX version is")
|
|
||||||
print(f"[HF_PATCH] Creating symlink from {original_repo} -> {mlx_repo}")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Create a symlink so the cache lookup succeeds
|
|
||||||
original_path.parent.mkdir(parents=True, exist_ok=True)
|
original_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
original_path.symlink_to(mlx_path, target_is_directory=True)
|
original_path.symlink_to(mlx_path, target_is_directory=True)
|
||||||
print(f"[HF_PATCH] Symlink created successfully")
|
logger.info("created cache symlink: %s -> %s", original_repo, mlx_repo)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
print(f"[HF_PATCH] Could not create symlink: {e}")
|
logger.warning("could not create cache symlink for %s", original_repo, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
# Auto-apply patch when module is imported
|
|
||||||
if os.environ.get("VOICEBOX_OFFLINE_PATCH", "1") != "0":
|
if os.environ.get("VOICEBOX_OFFLINE_PATCH", "1") != "0":
|
||||||
patch_huggingface_hub_offline()
|
patch_huggingface_hub_offline()
|
||||||
ensure_original_qwen_config_cached()
|
ensure_original_qwen_config_cached()
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ Guide for adding new TTS model backends. Based on the implementation of LuxTTS (
|
|||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
Adding an engine touches ~12 files across 4 layers (down from ~19 after the model config registry refactor). The backend protocol work is straightforward — the real time sink is dependency hell, upstream library bugs, and PyInstaller bundling.
|
Adding an engine touches ~10 files across 4 layers. The backend protocol work is straightforward — the real time sink is dependency hell, upstream library bugs, and PyInstaller bundling.
|
||||||
|
|
||||||
|
The backend is split into layers: `routes/` (thin HTTP handlers), `services/` (business logic), `backends/` (engine implementations), and `utils/` (shared utilities). New engines only need to touch `backends/` and `models.py` on the backend side — the route and service layers use a model config registry that handles dispatch automatically.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -119,26 +121,24 @@ In `backend/models.py`:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Phase 2: API Integration (`main.py`)
|
## Phase 2: Route and Service Integration
|
||||||
|
|
||||||
With the model config registry, `main.py` has **zero per-engine dispatch points**. All endpoints use registry helpers like `get_model_config()`, `load_engine_model()`, `engine_needs_trim()`, `check_model_loaded()`, etc.
|
With the model config registry, the route and service layers have **zero per-engine dispatch points**. All endpoints use registry helpers like `get_model_config()`, `load_engine_model()`, `engine_needs_trim()`, `check_model_loaded()`, etc.
|
||||||
|
|
||||||
**You don't need to touch `main.py` at all** unless your engine needs custom behavior in the generate endpoint (e.g. a new post-processing step beyond `trim_tts_output`).
|
**You don't need to touch any route or service files** unless your engine needs custom behavior in the generate pipeline (e.g. a new post-processing step beyond `trim_tts_output`).
|
||||||
|
|
||||||
### 2.1 What the registry handles automatically
|
### 2.1 What the registry handles automatically
|
||||||
|
|
||||||
| Endpoint | Registry function used |
|
| Route file | Registry function used |
|
||||||
|----------|----------------------|
|
|------------|----------------------|
|
||||||
| `POST /generate` | `load_engine_model(engine, size)` + `engine_needs_trim(engine)` |
|
| `routes/generations.py` | `load_engine_model(engine, size)` + `engine_needs_trim(engine)` |
|
||||||
| `POST /generate/stream` | `ensure_model_cached_or_raise(engine, size)` + `load_engine_model()` |
|
| `routes/models.py` | `get_all_model_configs()` + `check_model_loaded(config)` |
|
||||||
| `GET /models/status` | `get_all_model_configs()` + `check_model_loaded(config)` |
|
| `routes/models.py` | `get_model_config(name)` + `get_model_load_func(config)` |
|
||||||
| `POST /models/download` | `get_model_config(name)` + `get_model_load_func(config)` |
|
| `services/generation.py` | `get_tts_backend_for_engine()` + `ensure_model_cached_or_raise()` |
|
||||||
| `POST /models/{name}/unload` | `get_model_config(name)` + `unload_model_by_config(config)` |
|
|
||||||
| `DELETE /models/{name}` | `get_model_config(name)` + `unload_model_by_config(config)` |
|
|
||||||
|
|
||||||
### 2.2 Post-processing
|
### 2.2 Post-processing
|
||||||
|
|
||||||
If your model produces trailing silence or hallucinated audio, set `needs_trim=True` on your `ModelConfig`. The generate endpoint checks `engine_needs_trim(engine)` and applies `trim_tts_output()` automatically.
|
If your model produces trailing silence or hallucinated audio, set `needs_trim=True` on your `ModelConfig`. The generation service checks `engine_needs_trim(engine)` and applies `trim_tts_output()` automatically.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -321,7 +321,7 @@ Used by both Chatterbox backends. LuxTTS works fine on MPS.
|
|||||||
|
|
||||||
To get download progress bars in the UI, wrap model loading with `HFProgressTracker`:
|
To get download progress bars in the UI, wrap model loading with `HFProgressTracker`:
|
||||||
```python
|
```python
|
||||||
from backend.utils.hf_progress import HFProgressTracker
|
from ..utils.hf_progress import HFProgressTracker
|
||||||
tracker = HFProgressTracker(model_name, progress_manager)
|
tracker = HFProgressTracker(model_name, progress_manager)
|
||||||
with tracker.patch_download():
|
with tracker.patch_download():
|
||||||
model = ModelClass.from_pretrained(repo_id)
|
model = ModelClass.from_pretrained(repo_id)
|
||||||
@@ -339,7 +339,7 @@ The tracker monkey-patches tqdm to intercept HuggingFace's internal progress bar
|
|||||||
- [ ] `backend/requirements.txt` — dependencies added (check for `--no-deps` needs)
|
- [ ] `backend/requirements.txt` — dependencies added (check for `--no-deps` needs)
|
||||||
- [ ] `justfile` — `--no-deps` install step if needed
|
- [ ] `justfile` — `--no-deps` install step if needed
|
||||||
|
|
||||||
### API (`backend/main.py`)
|
### Routes and services
|
||||||
No changes needed — the model config registry handles all dispatch automatically.
|
No changes needed — the model config registry handles all dispatch automatically.
|
||||||
|
|
||||||
### Frontend
|
### Frontend
|
||||||
|
|||||||
Reference in New Issue
Block a user