mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 04:40:40 -07:00
fix: patch VoiceEncoder.forward to cast float64 mels to float32
The previous approach of patching librosa.load didn't work because melspectrogram itself performs float64 math (numpy dot, signal.lfilter) regardless of input dtype. The actual mismatch happens when pack() creates a float64 tensor from the mel arrays and passes it into the float32 LSTM weights in VoiceEncoder.forward(). Fix by monkey-patching VoiceEncoder.forward() to call mels.float() before the LSTM, ensuring the input always matches the model dtype.
This commit is contained in:
@@ -178,6 +178,21 @@ class ChatterboxTTSBackend:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
# Monkey-patch VoiceEncoder.forward to cast input to float32.
|
||||
# The upstream melspectrogram returns float64 numpy arrays when
|
||||
# hp.normalized_mels is False (the default). pack() preserves
|
||||
# the dtype, so double tensors hit float32 LSTM weights →
|
||||
# "expected m1 and m2 to have the same dtype: float != double".
|
||||
_ve = self.model.ve
|
||||
_orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward
|
||||
|
||||
import types
|
||||
|
||||
def _f32_forward(self_ve, mels):
|
||||
return _orig_ve_forward(self_ve, mels.float())
|
||||
|
||||
_ve.forward = types.MethodType(_f32_forward, _ve)
|
||||
|
||||
logger.info("Chatterbox Multilingual TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
|
||||
@@ -178,6 +178,21 @@ class ChatterboxTurboTTSBackend:
|
||||
progress_manager.mark_complete(model_name)
|
||||
task_manager.complete_download(model_name)
|
||||
|
||||
# Monkey-patch VoiceEncoder.forward to cast input to float32.
|
||||
# The upstream melspectrogram returns float64 numpy arrays when
|
||||
# hp.normalized_mels is False (the default). pack() preserves
|
||||
# the dtype, so double tensors hit float32 LSTM weights →
|
||||
# "expected m1 and m2 to have the same dtype: float != double".
|
||||
_ve = self.model.ve
|
||||
_orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward
|
||||
|
||||
import types
|
||||
|
||||
def _f32_forward(self_ve, mels):
|
||||
return _orig_ve_forward(self_ve, mels.float())
|
||||
|
||||
_ve.forward = types.MethodType(_f32_forward, _ve)
|
||||
|
||||
logger.info("Chatterbox Turbo TTS loaded successfully")
|
||||
|
||||
except ImportError as e:
|
||||
|
||||
Reference in New Issue
Block a user