mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 12:50:42 -07:00
Jamie Pine
7b5e73cfa8
- Created a new .npmrc file to enforce bun usage. - Bumped version numbers for multiple packages to 0.1.9 in bun.lock. - Added react-sound-visualizer dependency to enhance audio visualization features. - Introduced convert:assets script in package.json for asset optimization. - Updated CONTRIBUTING.md with instructions for converting assets to web formats. - Added documentation files for API endpoints and developer guidelines in the docs directory.
300 lines
6.9 KiB
Plaintext
300 lines
6.9 KiB
Plaintext
---
|
|
title: "Transcription"
|
|
description: "How Whisper-based audio transcription works in Voicebox"
|
|
---
|
|
|
|
## Overview
|
|
|
|
Voicebox uses OpenAI's Whisper model for automatic speech recognition (ASR). This powers the transcription feature for creating reference text from audio recordings.
|
|
|
|
## Architecture
|
|
|
|
The transcription system is built around the `WhisperModel` class:
|
|
|
|
**Model Loading:** Lazy loading with HuggingFace Hub download.
|
|
|
|
**Audio Processing:** Resampling and preprocessing for Whisper.
|
|
|
|
**Inference:** Running transcription with optional language hints.
|
|
|
|
## WhisperModel Class
|
|
|
|
```python
|
|
class WhisperModel:
|
|
def __init__(self, model_size: str = "base"):
|
|
self.model = None
|
|
self.processor = None
|
|
self.model_size = model_size
|
|
self.device = self._get_device()
|
|
```
|
|
|
|
### Model Sizes
|
|
|
|
| Size | Parameters | VRAM | Speed | Quality |
|
|
|------|------------|------|-------|---------|
|
|
| tiny | 39M | ~1GB | Fastest | Basic |
|
|
| base | 74M | ~1GB | Fast | Good |
|
|
| small | 244M | ~2GB | Medium | Better |
|
|
| medium | 769M | ~5GB | Slow | High |
|
|
| large | 1550M | ~10GB | Slowest | Best |
|
|
|
|
Default is `base` for balance of speed and quality.
|
|
|
|
## Model Loading
|
|
|
|
Models are downloaded from HuggingFace Hub:
|
|
|
|
```python
|
|
def load_model(self, model_size: Optional[str] = None):
|
|
from transformers import WhisperProcessor, WhisperForConditionalGeneration
|
|
|
|
model_name = f"openai/whisper-{model_size}"
|
|
|
|
# Track download progress
|
|
progress_manager = get_progress_manager()
|
|
task_manager = get_task_manager()
|
|
task_manager.start_download(f"whisper-{model_size}")
|
|
|
|
# Load processor and model
|
|
with tracker.patch_download():
|
|
self.processor = WhisperProcessor.from_pretrained(model_name)
|
|
self.model = WhisperForConditionalGeneration.from_pretrained(model_name)
|
|
|
|
self.model.to(self.device)
|
|
|
|
# Mark complete
|
|
progress_manager.mark_complete(f"whisper-{model_size}")
|
|
task_manager.complete_download(f"whisper-{model_size}")
|
|
```
|
|
|
|
### Async Loading
|
|
|
|
Like TTS, loading runs in a thread pool:
|
|
|
|
```python
|
|
async def load_model_async(self, model_size: Optional[str] = None):
|
|
if self.model is not None and self.model_size == model_size:
|
|
return
|
|
await asyncio.to_thread(self.load_model, model_size)
|
|
```
|
|
|
|
## Transcription
|
|
|
|
### Basic Transcription
|
|
|
|
```python
|
|
async def transcribe(
|
|
self,
|
|
audio_path: str,
|
|
language: Optional[str] = None,
|
|
) -> str:
|
|
await self.load_model_async()
|
|
|
|
def _transcribe_sync():
|
|
# Load and resample to 16kHz (Whisper requirement)
|
|
audio, sr = load_audio(audio_path, sample_rate=16000)
|
|
|
|
# Process audio
|
|
inputs = self.processor(
|
|
audio,
|
|
sampling_rate=16000,
|
|
return_tensors="pt",
|
|
)
|
|
inputs = inputs.to(self.device)
|
|
|
|
# Set language hint if provided
|
|
forced_decoder_ids = None
|
|
if language:
|
|
forced_decoder_ids = self.processor.get_decoder_prompt_ids(
|
|
language=language,
|
|
task="transcribe",
|
|
)
|
|
|
|
# Generate
|
|
with torch.no_grad():
|
|
predicted_ids = self.model.generate(
|
|
inputs["input_features"],
|
|
forced_decoder_ids=forced_decoder_ids,
|
|
)
|
|
|
|
# Decode
|
|
transcription = self.processor.batch_decode(
|
|
predicted_ids,
|
|
skip_special_tokens=True,
|
|
)[0]
|
|
|
|
return transcription.strip()
|
|
|
|
return await asyncio.to_thread(_transcribe_sync)
|
|
```
|
|
|
|
### Supported Languages
|
|
|
|
Whisper supports 99+ languages. Common ones in Voicebox:
|
|
|
|
| Code | Language |
|
|
|------|----------|
|
|
| en | English |
|
|
| zh | Chinese |
|
|
| ja | Japanese |
|
|
| ko | Korean |
|
|
| de | German |
|
|
| fr | French |
|
|
| ru | Russian |
|
|
| pt | Portuguese |
|
|
| es | Spanish |
|
|
| it | Italian |
|
|
|
|
### Language Detection
|
|
|
|
When no language is specified, Whisper auto-detects:
|
|
|
|
```python
|
|
# Without language hint - auto-detect
|
|
transcription = await whisper.transcribe(audio_path)
|
|
|
|
# With language hint - more accurate for short clips
|
|
transcription = await whisper.transcribe(audio_path, language="en")
|
|
```
|
|
|
|
## Transcription with Timestamps
|
|
|
|
For advanced use cases, word-level timestamps are available:
|
|
|
|
```python
|
|
async def transcribe_with_timestamps(
|
|
self,
|
|
audio_path: str,
|
|
language: Optional[str] = None,
|
|
) -> List[Dict[str, any]]:
|
|
await self.load_model_async()
|
|
|
|
def _transcribe_timestamps_sync():
|
|
audio, sr = load_audio(audio_path, sample_rate=16000)
|
|
inputs = self.processor(audio, sampling_rate=16000, return_tensors="pt")
|
|
|
|
with torch.no_grad():
|
|
predicted_ids = self.model.generate(
|
|
inputs["input_features"],
|
|
return_timestamps=True,
|
|
)
|
|
|
|
# Parse timestamps
|
|
return [
|
|
{
|
|
"text": transcription,
|
|
"start": 0.0,
|
|
"end": len(audio) / sr,
|
|
}
|
|
]
|
|
|
|
return await asyncio.to_thread(_transcribe_timestamps_sync)
|
|
```
|
|
|
|
## Memory Management
|
|
|
|
### Unloading
|
|
|
|
Free memory when not needed:
|
|
|
|
```python
|
|
def unload_model(self):
|
|
if self.model is not None:
|
|
del self.model
|
|
del self.processor
|
|
self.model = None
|
|
self.processor = None
|
|
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
```
|
|
|
|
### Global Instance
|
|
|
|
A singleton pattern manages the model:
|
|
|
|
```python
|
|
_whisper_model: Optional[WhisperModel] = None
|
|
|
|
def get_whisper_model() -> WhisperModel:
|
|
global _whisper_model
|
|
if _whisper_model is None:
|
|
_whisper_model = WhisperModel()
|
|
return _whisper_model
|
|
```
|
|
|
|
## Audio Preprocessing
|
|
|
|
### Resampling
|
|
|
|
Whisper requires 16kHz audio:
|
|
|
|
```python
|
|
audio, sr = load_audio(audio_path, sample_rate=16000)
|
|
```
|
|
|
|
### Format Support
|
|
|
|
The `load_audio` utility handles:
|
|
- WAV
|
|
- MP3
|
|
- FLAC
|
|
- OGG
|
|
- M4A
|
|
|
|
All formats are converted to mono 16kHz.
|
|
|
|
## API Endpoints
|
|
|
|
| Method | Endpoint | Description |
|
|
|--------|----------|-------------|
|
|
| POST | `/transcribe` | Transcribe audio file |
|
|
|
|
### Request
|
|
|
|
Multipart form data:
|
|
|
|
```
|
|
POST /transcribe
|
|
Content-Type: multipart/form-data
|
|
|
|
file: <audio_file>
|
|
language: en (optional)
|
|
```
|
|
|
|
### Response
|
|
|
|
```json
|
|
{
|
|
"text": "Hello, this is a test transcription.",
|
|
"duration": 3.5
|
|
}
|
|
```
|
|
|
|
## Use Cases
|
|
|
|
### Reference Text for Voice Cloning
|
|
|
|
1. User records audio sample
|
|
2. Audio is sent to `/transcribe`
|
|
3. Transcription becomes `reference_text`
|
|
4. Both are added to voice profile
|
|
|
|
### Quality Tips
|
|
|
|
- Provide language hint for short audio
|
|
- Use clean audio with minimal noise
|
|
- Longer audio (>5s) improves accuracy
|
|
- Consider `small` or `medium` model for better quality
|
|
|
|
## Error Handling
|
|
|
|
Common issues:
|
|
|
|
| Error | Cause | Solution |
|
|
|-------|-------|----------|
|
|
| Model not found | First run, download failed | Retry with network |
|
|
| OOM | Model too large | Use smaller model |
|
|
| Empty result | No speech detected | Check audio has speech |
|
|
| Wrong language | Auto-detect failed | Provide language hint |
|