From bfd7b815a596c95fabd57a8570be56db994a3d5f Mon Sep 17 00:00:00 2001 From: James Pine Date: Fri, 13 Mar 2026 04:55:32 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20patch=20S3Tokenizer.log=5Fmel=5Fspectrog?= =?UTF-8?q?ram=20for=20float64=E2=86=92float32=20cast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The actual dtype mismatch was in S3Tokenizer.log_mel_spectrogram, not VoiceEncoder.forward. librosa.load returns float64 numpy, which torch.from_numpy preserves as double. The STFT output (double) then hits _mel_filters (float32) in a matmul at s3tokenizer.py:163. Now patching both entry points after model load: 1. S3Tokenizer.log_mel_spectrogram — cast audio to float32 before STFT 2. VoiceEncoder.forward — cast mels to float32 before LSTM Remove debug traceback logging (no longer needed). --- backend/backends/chatterbox_backend.py | 32 +++++++++++------ backend/backends/chatterbox_turbo_backend.py | 38 ++++++++++++++------ 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/backend/backends/chatterbox_backend.py b/backend/backends/chatterbox_backend.py index 826d7c05..a440f998 100644 --- a/backend/backends/chatterbox_backend.py +++ b/backend/backends/chatterbox_backend.py @@ -178,20 +178,32 @@ class ChatterboxTTSBackend: progress_manager.mark_complete(model_name) task_manager.complete_download(model_name) - # Monkey-patch VoiceEncoder.forward to cast input to float32. - # The upstream melspectrogram returns float64 numpy arrays when - # hp.normalized_mels is False (the default). pack() preserves - # the dtype, so double tensors hit float32 LSTM weights → - # "expected m1 and m2 to have the same dtype: float != double". - _ve = self.model.ve - _orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward - + # Patch float64 → float32 dtype mismatches in upstream chatterbox. + # librosa.load returns float64 numpy; multiple upstream code paths + # convert it to a torch tensor via torch.from_numpy() without + # casting, then matmul it against float32 model weights. import types - def _f32_forward(self_ve, mels): + # Patch S3Tokenizer (used by s3gen.tokenizer) + _tokzr = self.model.s3gen.tokenizer + _orig_log_mel = _tokzr.log_mel_spectrogram.__func__ + + def _f32_log_mel(self_tokzr, audio, padding=0): + import torch as _torch + if _torch.is_tensor(audio): + audio = audio.float() + return _orig_log_mel(self_tokzr, audio, padding) + + _tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr) + + # Patch VoiceEncoder + _ve = self.model.ve + _orig_ve_forward = _ve.forward.__func__ + + def _f32_ve_forward(self_ve, mels): return _orig_ve_forward(self_ve, mels.float()) - _ve.forward = types.MethodType(_f32_forward, _ve) + _ve.forward = types.MethodType(_f32_ve_forward, _ve) logger.info("Chatterbox Multilingual TTS loaded successfully") diff --git a/backend/backends/chatterbox_turbo_backend.py b/backend/backends/chatterbox_turbo_backend.py index 6c5bf842..3ce54f38 100644 --- a/backend/backends/chatterbox_turbo_backend.py +++ b/backend/backends/chatterbox_turbo_backend.py @@ -178,20 +178,38 @@ class ChatterboxTurboTTSBackend: progress_manager.mark_complete(model_name) task_manager.complete_download(model_name) - # Monkey-patch VoiceEncoder.forward to cast input to float32. - # The upstream melspectrogram returns float64 numpy arrays when - # hp.normalized_mels is False (the default). pack() preserves - # the dtype, so double tensors hit float32 LSTM weights → - # "expected m1 and m2 to have the same dtype: float != double". - _ve = self.model.ve - _orig_ve_forward = _ve.forward.__func__ if hasattr(_ve.forward, '__func__') else _ve.forward - + # Patch float64 → float32 dtype mismatches in upstream chatterbox. + # librosa.load returns float64 numpy; multiple upstream code paths + # convert it to a torch tensor via torch.from_numpy() without + # casting, then matmul it against float32 model weights. + # We patch the two known entry points: + # + # 1. S3Tokenizer.log_mel_spectrogram — the audio tensor from + # librosa hits _mel_filters (float32) in a matmul. + # 2. VoiceEncoder.forward — float64 mel spectrograms hit the + # float32 LSTM weights. import types - def _f32_forward(self_ve, mels): + # Patch S3Tokenizer (used by s3gen.tokenizer) + _tokzr = self.model.s3gen.tokenizer + _orig_log_mel = _tokzr.log_mel_spectrogram.__func__ + + def _f32_log_mel(self_tokzr, audio, padding=0): + import torch as _torch + if _torch.is_tensor(audio): + audio = audio.float() + return _orig_log_mel(self_tokzr, audio, padding) + + _tokzr.log_mel_spectrogram = types.MethodType(_f32_log_mel, _tokzr) + + # Patch VoiceEncoder + _ve = self.model.ve + _orig_ve_forward = _ve.forward.__func__ + + def _f32_ve_forward(self_ve, mels): return _orig_ve_forward(self_ve, mels.float()) - _ve.forward = types.MethodType(_f32_forward, _ve) + _ve.forward = types.MethodType(_f32_ve_forward, _ve) logger.info("Chatterbox Turbo TTS loaded successfully")