Labyricorn/voicebox
1
0
Fork 0
mirror of https://github.com/jamiepine/voicebox.git synced 2026-09-16 13:20:39 -07:00
Code Issues Packages Projects Releases Wiki Activity

feat: chunked TTS generation for long text (engine-agnostic)

Browse Source 
Text exceeding max_chunk_chars (default 800) is automatically split at
sentence boundaries, generated per-chunk, and concatenated with a 50ms
crossfade.  Works with all engines (Qwen, LuxTTS, Chatterbox, Turbo).

- Abbreviation-aware sentence splitter (Dr., Mr., e.g., decimals)
- CJK sentence-ending punctuation support
- Paralinguistic tag preservation ([laugh], [cough], etc.)
- Per-chunk seed variation to avoid correlated RNG artefacts
- Per-chunk Chatterbox trim (catches hallucination at each boundary)
- max_chunk_chars exposed as per-request param on GenerationRequest
- Text max_length raised to 50,000 characters

Closes #99
This commit is contained in:
James Pine
2026-03-13 06:21:34 -07:00
parent c12b5d6f0a
commit 70ca7f66cb
3 changed files with 329 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/main.py
+29 -18
View File
@@ -824,18 +824,24 @@ async def generate_speech(
engine=engine,
)
audio, sample_rate = await tts_model.generate(
data.text,
voice_prompt,
data.language,
data.seed,
data.instruct,
)
from .utils.chunked_tts import generate_chunked
# Trim trailing silence/hallucination for Chatterbox output
# Resolve per-chunk trim function for engines that need it
trim_fn = None
if engine in ("chatterbox", "chatterbox_turbo"):
from .utils.audio import trim_tts_output
audio = trim_tts_output(audio, sample_rate)
trim_fn = trim_tts_output
audio, sample_rate = await generate_chunked(
tts_model,
data.text,
voice_prompt,
language=data.language,
seed=data.seed,
instruct=data.instruct,
max_chunk_chars=data.max_chunk_chars,
trim_fn=trim_fn,
)
# Calculate duration
duration = len(audio) / sample_rate
@@ -949,18 +955,23 @@ async def stream_speech(
data.profile_id, db, engine=engine,
)
audio, sample_rate = await tts_model.generate(
data.text,
voice_prompt,
data.language,
data.seed,
data.instruct,
)
from .utils.chunked_tts import generate_chunked
# Trim trailing silence/hallucination for Chatterbox output
trim_fn = None
if engine in ("chatterbox", "chatterbox_turbo"):
from .utils.audio import trim_tts_output
audio = trim_tts_output(audio, sample_rate)
trim_fn = trim_tts_output
audio, sample_rate = await generate_chunked(
tts_model,
data.text,
voice_prompt,
language=data.language,
seed=data.seed,
instruct=data.instruct,
max_chunk_chars=data.max_chunk_chars,
trim_fn=trim_fn,
)
wav_bytes = tts.audio_to_wav_bytes(audio, sample_rate)
backend/models.py
+2 -1
View File
@@ -52,12 +52,13 @@ class ProfileSampleResponse(BaseModel):
class GenerationRequest(BaseModel):
"""Request model for voice generation."""
profile_id: str
text: str = Field(..., min_length=1, max_length=5000)
text: str = Field(..., min_length=1, max_length=50000)
language: str = Field(default="en", pattern="^(zh|en|ja|ko|de|fr|ru|pt|es|it|he)$")
seed: Optional[int] = Field(None, ge=0)
model_size: Optional[str] = Field(default="1.7B", pattern="^(1\\.7B|0\\.6B)$")
instruct: Optional[str] = Field(None, max_length=500)
engine: Optional[str] = Field(default="qwen", pattern="^(qwen|luxtts|chatterbox|chatterbox_turbo)$")
max_chunk_chars: int = Field(default=800, ge=100, le=5000, description="Max characters per chunk for long text splitting")
class GenerationResponse(BaseModel):
backend/utils/chunked_tts.py
+298
View File
@@ -0,0 +1,298 @@
"""
Chunked TTS generation utilities.
Splits long text into sentence-boundary chunks, generates audio per-chunk
via any TTSBackend, and concatenates with crossfade. All logic is
engine-agnostic — it wraps the standard ``TTSBackend.generate()`` interface.
Short text (≤ max_chunk_chars) uses the single-shot fast path with zero
overhead.
"""
import logging
import re
from typing import List, Tuple
import numpy as np
logger = logging.getLogger("voicebox.chunked-tts")
# Default chunk size in characters. Can be overridden per-request via
# the ``max_chunk_chars`` field on GenerationRequest.
DEFAULT_MAX_CHUNK_CHARS = 800
# Common abbreviations that should NOT be treated as sentence endings.
# Lowercase for case-insensitive matching.
_ABBREVIATIONS = frozenset(
{
"mr",
"mrs",
"ms",
"dr",
"prof",
"sr",
"jr",
"st",
"ave",
"blvd",
"inc",
"ltd",
"corp",
"dept",
"est",
"approx",
"vs",
"etc",
"e.g",
"i.e",
"a.m",
"p.m",
"u.s",
"u.s.a",
"u.k",
}
)
# Paralinguistic tags used by Chatterbox Turbo. The splitter must never
# cut inside one of these.
_PARA_TAG_RE = re.compile(r"\[[^\]]*\]")
# ---------------------------------------------------------------------------
# Text splitting
# ---------------------------------------------------------------------------
def split_text_into_chunks(text: str, max_chars: int = DEFAULT_MAX_CHUNK_CHARS) -> List[str]:
"""Split *text* at natural boundaries into chunks of at most *max_chars*.
Priority: sentence-end (``.!?`` not preceded by an abbreviation and not
inside brackets) → clause boundary (``;:,—``) → whitespace → hard cut.
Paralinguistic tags like ``[laugh]`` are treated as atomic and will not
be split across chunks.
"""
text = text.strip()
if not text:
return []
if len(text) <= max_chars:
return [text]
chunks: List[str] = []
remaining = text
while remaining:
remaining = remaining.lstrip()
if not remaining:
break
if len(remaining) <= max_chars:
chunks.append(remaining)
break
segment = remaining[:max_chars]
# Try to split at the last real sentence ending
split_pos = _find_last_sentence_end(segment)
if split_pos == -1:
split_pos = _find_last_clause_boundary(segment)
if split_pos == -1:
split_pos = segment.rfind(" ")
if split_pos == -1:
# Absolute fallback: hard cut but avoid splitting inside a tag
split_pos = _safe_hard_cut(segment, max_chars)
chunk = remaining[: split_pos + 1].strip()
if chunk:
chunks.append(chunk)
remaining = remaining[split_pos + 1 :]
return chunks
def _find_last_sentence_end(text: str) -> int:
"""Return the index of the last sentence-ending punctuation in *text*.
Skips periods that follow common abbreviations (``Dr.``, ``Mr.``, etc.)
and periods inside bracket tags (``[laugh]``). Also handles CJK
sentence-ending punctuation (``。！？``).
"""
best = -1
# ASCII sentence ends
for m in re.finditer(r"[.!?](?:\s|$)", text):
pos = m.start()
char = text[pos]
# Skip periods after abbreviations
if char == ".":
# Walk backwards to find the preceding word
word_start = pos - 1
while word_start >= 0 and text[word_start].isalpha():
word_start -= 1
word = text[word_start + 1 : pos].lower()
if word in _ABBREVIATIONS:
continue
# Skip decimal numbers (digit immediately before the period)
if word_start >= 0 and text[word_start].isdigit():
continue
# Skip if we're inside a bracket tag
if _inside_bracket_tag(text, pos):
continue
best = pos
# CJK sentence-ending punctuation
for m in re.finditer(r"[\u3002\uff01\uff1f]", text):
if m.start() > best:
best = m.start()
return best
def _find_last_clause_boundary(text: str) -> int:
"""Return the index of the last clause-boundary punctuation."""
best = -1
for m in re.finditer(r"[;:,\u2014](?:\s|$)", text):
pos = m.start()
# Skip if inside a bracket tag
if _inside_bracket_tag(text, pos):
continue
best = pos
return best
def _inside_bracket_tag(text: str, pos: int) -> bool:
"""Return True if *pos* falls inside a ``[...]`` tag."""
for m in _PARA_TAG_RE.finditer(text):
if m.start() < pos < m.end():
return True
return False
def _safe_hard_cut(segment: str, max_chars: int) -> int:
"""Find a hard-cut position that doesn't split a ``[tag]``."""
cut = max_chars - 1
# Check if the cut falls inside a bracket tag; if so, move before it
for m in _PARA_TAG_RE.finditer(segment):
if m.start() < cut < m.end():
return m.start() - 1 if m.start() > 0 else cut
return cut
# ---------------------------------------------------------------------------
# Audio concatenation
# ---------------------------------------------------------------------------
def concatenate_audio_chunks(
chunks: List[np.ndarray],
sample_rate: int,
crossfade_ms: int = 50,
) -> np.ndarray:
"""Concatenate audio arrays with a short crossfade to eliminate clicks.
Each chunk is expected to be a 1-D float32 ndarray at *sample_rate* Hz.
"""
if not chunks:
return np.array([], dtype=np.float32)
if len(chunks) == 1:
return chunks[0]
crossfade_samples = int(sample_rate * crossfade_ms / 1000)
result = np.array(chunks[0], dtype=np.float32, copy=True)
for chunk in chunks[1:]:
if len(chunk) == 0:
continue
overlap = min(crossfade_samples, len(result), len(chunk))
if overlap > 0:
fade_out = np.linspace(1.0, 0.0, overlap, dtype=np.float32)
fade_in = np.linspace(0.0, 1.0, overlap, dtype=np.float32)
result[-overlap:] = result[-overlap:] * fade_out + chunk[:overlap] * fade_in
result = np.concatenate([result, chunk[overlap:]])
else:
result = np.concatenate([result, chunk])
return result
# ---------------------------------------------------------------------------
# Engine-agnostic chunked generation
# ---------------------------------------------------------------------------
async def generate_chunked(
backend,
text: str,
voice_prompt: dict,
language: str = "en",
seed: int | None = None,
instruct: str | None = None,
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
trim_fn=None,
) -> Tuple[np.ndarray, int]:
"""Generate audio with automatic chunking for long text.
For text shorter than *max_chunk_chars* this is a thin wrapper around
``backend.generate()`` with zero overhead.
For longer text the input is split at natural sentence boundaries,
each chunk is generated independently, optionally trimmed (useful for
Chatterbox engines that hallucinate trailing noise), and the results
are concatenated with a short crossfade.
Parameters
----------
backend : TTSBackend
Any backend implementing the ``generate()`` protocol.
text : str
Input text (may be arbitrarily long).
voice_prompt, language, seed, instruct
Forwarded to ``backend.generate()`` verbatim.
max_chunk_chars : int
Maximum characters per chunk (default 800).
trim_fn : callable | None
Optional ``(audio, sample_rate) -> audio`` post-processing
function applied to each chunk before concatenation (e.g.
``trim_tts_output`` for Chatterbox engines).
Returns
-------
(audio, sample_rate) : Tuple[np.ndarray, int]
"""
chunks = split_text_into_chunks(text, max_chunk_chars)
if len(chunks) <= 1:
# Short text — single-shot fast path
audio, sample_rate = await backend.generate(
text, voice_prompt, language, seed, instruct,
)
if trim_fn is not None:
audio = trim_fn(audio, sample_rate)
return audio, sample_rate
# Long text — chunked generation
logger.info(
"Splitting %d chars into %d chunks (max %d chars each)",
len(text), len(chunks), max_chunk_chars,
)
audio_chunks: List[np.ndarray] = []
sample_rate: int | None = None
for i, chunk_text in enumerate(chunks):
logger.info(
"Generating chunk %d/%d (%d chars)",
i + 1, len(chunks), len(chunk_text),
)
# Vary the seed per chunk to avoid correlated RNG artefacts,
# but keep it deterministic so the same (text, seed) pair
# always produces the same output.
chunk_seed = (seed + i) if seed is not None else None
chunk_audio, chunk_sr = await backend.generate(
chunk_text, voice_prompt, language, chunk_seed, instruct,
)
if trim_fn is not None:
chunk_audio = trim_fn(chunk_audio, chunk_sr)
audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
if sample_rate is None:
sample_rate = chunk_sr
audio = concatenate_audio_chunks(audio_chunks, sample_rate)
return audio, sample_rate