mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 04:40:40 -07:00
feat: add Chatterbox TTS engine for multilingual voice cloning
- New ChatterboxTTSBackend wrapping ChatterboxMultilingualTTS (ResembleAI/chatterbox) - Supports 23 languages including Hebrew, forces CPU on macOS (MPS issue) - Monkey-patches torch.load for CPU loading, forces eager attention for compatibility - trim_tts_output utility cuts trailing silence/hallucination from Chatterbox output - Full engine integration: /generate, /generate/stream, model status/download/delete - Hebrew (he) added to supported languages in frontend and backend validation - Single flat model dropdown extended with Chatterbox option in both generation UIs - ModelManagement UI groups LuxTTS and Chatterbox under 'Other Voice Models' section
This commit is contained in:
@@ -80,6 +80,95 @@ def save_audio(
|
||||
sf.write(path, audio, sample_rate)
|
||||
|
||||
|
||||
def trim_tts_output(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = 24000,
|
||||
frame_ms: int = 20,
|
||||
silence_threshold_db: float = -40.0,
|
||||
min_silence_ms: int = 200,
|
||||
max_internal_silence_ms: int = 1000,
|
||||
fade_ms: int = 30,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Trim trailing silence and post-silence hallucination from TTS output.
|
||||
|
||||
Chatterbox sometimes produces ``[speech][silence][hallucinated noise]``.
|
||||
This detects internal silence gaps longer than *max_internal_silence_ms*
|
||||
and cuts the audio at that boundary, then trims trailing silence and
|
||||
applies a short cosine fade-out.
|
||||
|
||||
Args:
|
||||
audio: Input audio array (mono float32)
|
||||
sample_rate: Sample rate in Hz
|
||||
frame_ms: Frame size for RMS energy calculation
|
||||
silence_threshold_db: dB threshold below which a frame is silence
|
||||
min_silence_ms: Minimum trailing silence to keep
|
||||
max_internal_silence_ms: Cut after any silence gap longer than this
|
||||
fade_ms: Cosine fade-out duration in ms
|
||||
|
||||
Returns:
|
||||
Trimmed audio array
|
||||
"""
|
||||
frame_len = int(sample_rate * frame_ms / 1000)
|
||||
if frame_len == 0 or len(audio) < frame_len:
|
||||
return audio
|
||||
|
||||
n_frames = len(audio) // frame_len
|
||||
threshold_linear = 10 ** (silence_threshold_db / 20)
|
||||
|
||||
# Compute per-frame RMS
|
||||
rms = np.array(
|
||||
[
|
||||
np.sqrt(np.mean(audio[i * frame_len : (i + 1) * frame_len] ** 2))
|
||||
for i in range(n_frames)
|
||||
]
|
||||
)
|
||||
is_speech = rms >= threshold_linear
|
||||
|
||||
# Find first speech frame
|
||||
first_speech = 0
|
||||
for i, s in enumerate(is_speech):
|
||||
if s:
|
||||
first_speech = max(0, i - 1) # keep 1 frame padding
|
||||
break
|
||||
|
||||
# Walk forward from first speech; cut at long internal silence gaps
|
||||
max_silence_frames = int(max_internal_silence_ms / frame_ms)
|
||||
consecutive_silence = 0
|
||||
cut_frame = n_frames
|
||||
|
||||
for i in range(first_speech, n_frames):
|
||||
if is_speech[i]:
|
||||
consecutive_silence = 0
|
||||
else:
|
||||
consecutive_silence += 1
|
||||
if consecutive_silence >= max_silence_frames:
|
||||
cut_frame = i - consecutive_silence + 1
|
||||
break
|
||||
|
||||
# Trim trailing silence from the cut point
|
||||
min_silence_frames = int(min_silence_ms / frame_ms)
|
||||
end_frame = cut_frame
|
||||
while end_frame > first_speech and not is_speech[end_frame - 1]:
|
||||
end_frame -= 1
|
||||
# Keep a short tail
|
||||
end_frame = min(end_frame + min_silence_frames, cut_frame)
|
||||
|
||||
# Convert frames back to samples
|
||||
start_sample = first_speech * frame_len
|
||||
end_sample = min(end_frame * frame_len, len(audio))
|
||||
|
||||
trimmed = audio[start_sample:end_sample].copy()
|
||||
|
||||
# Cosine fade-out
|
||||
fade_samples = int(sample_rate * fade_ms / 1000)
|
||||
if fade_samples > 0 and len(trimmed) > fade_samples:
|
||||
fade = np.cos(np.linspace(0, np.pi / 2, fade_samples)) ** 2
|
||||
trimmed[-fade_samples:] *= fade
|
||||
|
||||
return trimmed
|
||||
|
||||
|
||||
def validate_reference_audio(
|
||||
audio_path: str,
|
||||
min_duration: float = 2.0,
|
||||
|
||||
Reference in New Issue
Block a user