From 115de231d08dc8c2eb92cbad2ca16498b2fef871 Mon Sep 17 00:00:00 2001 From: Jamie Pine <32987599+jamiepine@users.noreply.github.com> Date: Sun, 19 Apr 2026 19:22:18 -0700 Subject: [PATCH] fix(audio): preprocess reference samples instead of rejecting them (#502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(audio): preprocess reference samples instead of rejecting them Uploaded/recorded voice samples were rejected outright whenever the peak exceeded 0.99 ("Audio is clipping (reduce input gain)"). That wasn't actionable: a recording in the app has no pre-gain control, and an already-captured file can't be re-taken by the user. The Settings "Normalize audio" toggle only affects generated TTS output, so users who enabled it expecting it to help with sample uploads were still blocked. Replace the hard reject with a small, always-on preprocess step that runs right after load: - DC-offset removal - Conservative edge-silence trim (top_db=30) with 100 ms padding kept - Peak cap at 0.95 if the input peak exceeds that Duration and RMS checks now run on the preprocessed waveform, so samples that were previously rejected for being "hot" are accepted and stored with safe headroom. True in-waveform clipping artifacts still can't be repaired — peak scaling only prevents downstream re-clipping during multi-sample combination and TTS inference. Adds a unit-test file (previously none existed for audio.py). Fixes #456. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(audio): raise trim threshold, cap pad at net-neutral Review feedback on the preprocessor: 1. ``trim_top_db=30`` was labelled "conservative" in the docstring but is actually *more* aggressive than librosa's default of 60. Normal speech dynamic range sits around 30 dB, so 30 dB would eat quiet trailing syllables and soft consonants. Raise the default to 40 dB — below normal speech dynamic range but still catching obvious edge silence — and fix the docstring. 2. Unconditional 100 ms edge padding ran even when ``librosa.effects.trim`` removed nothing. For a well-recorded 29.9 s upload that path would push the waveform past the 30 s ceiling and trigger a spurious "too long" rejection. Only pad when trimming actually shortened the audio, and cap the pad so the output never exceeds the input length. Adds a regression test for the net-neutral length behaviour. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- backend/tests/test_audio_preprocess.py | 112 +++++++++++++++++++++++++ backend/utils/audio.py | 80 ++++++++++++++++-- 2 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_audio_preprocess.py diff --git a/backend/tests/test_audio_preprocess.py b/backend/tests/test_audio_preprocess.py new file mode 100644 index 00000000..e52c44af --- /dev/null +++ b/backend/tests/test_audio_preprocess.py @@ -0,0 +1,112 @@ +""" +Unit tests for reference-audio preprocessing. + +Covers :func:`backend.utils.audio.preprocess_reference_audio` and +:func:`backend.utils.audio.validate_and_load_reference_audio`. +""" + +import sys +from pathlib import Path + +import numpy as np +import pytest +import soundfile as sf + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.audio import ( # noqa: E402 + preprocess_reference_audio, + validate_and_load_reference_audio, +) + + +SR = 24000 + + +def _tone(duration_s: float, amp: float = 0.3, freq: float = 220.0) -> np.ndarray: + n = int(duration_s * SR) + t = np.arange(n, dtype=np.float32) / SR + return (amp * np.sin(2 * np.pi * freq * t)).astype(np.float32) + + +def test_peak_cap_scales_hot_input(): + audio = _tone(3.0, amp=0.99) + out = preprocess_reference_audio(audio, SR) + assert np.abs(out).max() <= 0.951 + + +def test_peak_cap_leaves_moderate_input_untouched(): + audio = _tone(3.0, amp=0.5) + out = preprocess_reference_audio(audio, SR) + assert np.isclose(np.abs(out).max(), 0.5, atol=1e-3) + + +def test_dc_offset_removed(): + audio = _tone(3.0, amp=0.3) + 0.1 + out = preprocess_reference_audio(audio, SR) + assert abs(float(np.mean(out))) < 1e-3 + + +def test_silence_is_trimmed_with_padding_kept(): + silence = np.zeros(int(SR * 1.0), dtype=np.float32) + speech = _tone(3.0, amp=0.3) + audio = np.concatenate([silence, speech, silence]) + out = preprocess_reference_audio(audio, SR) + # Most of the 2s of leading/trailing silence should be gone, but the + # 3s of speech plus ~200ms of padding should remain. + assert len(audio) - len(out) >= SR, "expected >=1s of silence trimmed" + assert len(out) >= int(3.0 * SR), "speech body should be preserved" + + +def test_clean_audio_is_not_padded_past_original_length(): + # Well-recorded audio with no edge silence shouldn't get longer after + # preprocessing — otherwise a 29.9 s upload could be pushed past the + # 30 s max_duration ceiling downstream. + audio = _tone(3.0, amp=0.3) + out = preprocess_reference_audio(audio, SR) + assert len(out) <= len(audio) + + +def test_empty_input_returns_empty(): + out = preprocess_reference_audio(np.zeros(0, dtype=np.float32), SR) + assert out.size == 0 + + +def test_validate_accepts_previously_rejected_hot_file(tmp_path): + audio = _tone(3.0, amp=0.995) + path = tmp_path / "hot.wav" + sf.write(str(path), audio, SR) + + ok, err, out_audio, out_sr = validate_and_load_reference_audio(str(path)) + + assert ok, f"expected pass, got error: {err}" + assert out_audio is not None + assert out_sr == SR + assert np.abs(out_audio).max() <= 0.951 + + +def test_validate_still_rejects_silent_input(tmp_path): + audio = np.zeros(int(SR * 3.0), dtype=np.float32) + path = tmp_path / "silent.wav" + sf.write(str(path), audio, SR) + + ok, err, _, _ = validate_and_load_reference_audio(str(path)) + + assert not ok + assert err is not None + assert "too short" in err.lower() or "quiet" in err.lower() + + +def test_validate_rejects_too_short(tmp_path): + audio = _tone(0.5, amp=0.3) + path = tmp_path / "short.wav" + sf.write(str(path), audio, SR) + + ok, err, _, _ = validate_and_load_reference_audio(str(path)) + + assert not ok + assert "too short" in (err or "").lower() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/backend/utils/audio.py b/backend/utils/audio.py index 2016992c..7e0fd6fd 100644 --- a/backend/utils/audio.py +++ b/backend/utils/audio.py @@ -199,6 +199,66 @@ def trim_tts_output( return trimmed +def preprocess_reference_audio( + audio: np.ndarray, + sample_rate: int, + peak_target: float = 0.95, + trim_top_db: float = 40.0, + edge_padding_ms: int = 100, +) -> np.ndarray: + """ + Clean up a reference-audio sample before validation/storage. + + Removes DC offset, trims leading/trailing silence, and caps the peak so a + slightly-hot recording doesn't get rejected downstream as "clipping". The + goal is to accept reasonable real-world recordings — not to repair badly + distorted ones. True clipping artifacts inside the waveform can't be + recovered by peak scaling and will still sound bad. + + Args: + audio: Mono audio array. + sample_rate: Sample rate of ``audio`` in Hz. + peak_target: Peak amplitude cap in [0, 1]. Applied only if the input + peak exceeds this value. + trim_top_db: Silence threshold for edge trimming, in dB below peak. + 40 dB sits below normal speech dynamic range (≈30 dB) so soft + trailing syllables are preserved, while still catching obvious + leading/trailing silence. Lower values are more aggressive; + librosa's own default is 60. + edge_padding_ms: Milliseconds of padding to add back at each edge + *only if* trimming shortened the waveform, so TTS engines have a + brief silence to anchor on without ever making the output longer + than the input. + + Returns: + Preprocessed audio array (float32). + """ + audio = audio.astype(np.float32, copy=False) + + if audio.size == 0: + return audio + + audio = audio - float(np.mean(audio)) + + trimmed, _ = librosa.effects.trim(audio, top_db=trim_top_db) + if 0 < trimmed.size < audio.size: + pad_each = int(sample_rate * edge_padding_ms / 1000) + # Never pad past the original length — for near-max-duration uploads + # an unconditional pad would push them over the 30 s ceiling and + # trigger a spurious "too long" rejection. + headroom = (audio.size - trimmed.size) // 2 + pad = min(pad_each, max(headroom, 0)) + if pad > 0: + trimmed = np.pad(trimmed, (pad, pad), mode="constant") + audio = trimmed + + peak = float(np.abs(audio).max()) + if peak > peak_target and peak > 0: + audio = audio * (peak_target / peak) + + return audio + + def validate_reference_audio( audio_path: str, min_duration: float = 2.0, @@ -207,13 +267,13 @@ def validate_reference_audio( ) -> Tuple[bool, Optional[str]]: """ Validate reference audio for voice cloning. - + Args: audio_path: Path to audio file min_duration: Minimum duration in seconds max_duration: Maximum duration in seconds min_rms: Minimum RMS level - + Returns: Tuple of (is_valid, error_message) """ @@ -231,26 +291,28 @@ def validate_and_load_reference_audio( ) -> Tuple[bool, Optional[str], Optional[np.ndarray], Optional[int]]: """ Validate and load reference audio in a single pass. - + + Applies :func:`preprocess_reference_audio` before checks so that + slightly-hot recordings aren't rejected as clipping. Duration and RMS + checks run on the preprocessed waveform. + Returns: Tuple of (is_valid, error_message, audio_array, sample_rate) """ try: audio, sr = load_audio(audio_path) + audio = preprocess_reference_audio(audio, sr) duration = len(audio) / sr - + if duration < min_duration: return False, f"Audio too short (minimum {min_duration} seconds)", None, None if duration > max_duration: return False, f"Audio too long (maximum {max_duration} seconds)", None, None - + rms = np.sqrt(np.mean(audio**2)) if rms < min_rms: return False, "Audio is too quiet or silent", None, None - - if np.abs(audio).max() > 0.99: - return False, "Audio is clipping (reduce input gain)", None, None - + return True, None, audio, sr except Exception as e: return False, f"Error validating audio: {str(e)}", None, None