From 94487f32a5eb36ea6a660f445a9809c521d23412 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Thu, 29 Jan 2026 23:11:48 -0800 Subject: [PATCH] Enhance MLX and PyTorch Backend Integration - Added support for MLX backend on Apple Silicon, enabling optimized performance for TTS and STT tasks. - Implemented platform detection to dynamically select between MLX and PyTorch based on the runtime environment. - Updated build process to include MLX-specific dependencies and configurations for macOS. - Refactored backend code to improve model loading and inference logic, accommodating backend-specific requirements. - Enhanced documentation to clarify backend selection and performance benefits for different platforms. - Streamlined installation instructions and troubleshooting guidance for MLX-related issues. --- CONTRIBUTING.md | 40 +++++++------ Makefile | 5 ++ README.md | 11 +++- backend/README.md | 35 +++++++++--- backend/backends/__init__.py | 2 +- backend/build_binary.py | 5 +- backend/main.py | 2 +- backend/{platform.py => platform_detect.py} | 0 backend/voicebox-server.spec | 52 +++++++++++++++++ docs/TROUBLESHOOTING.md | 36 ++++++++++-- docs/developer/architecture.mdx | 6 +- docs/developer/building.mdx | 59 +++++++++++--------- docs/developer/setup.mdx | 15 ++--- package.json | 2 +- tauri/src-tauri/gen/Assets.car | Bin 3847048 -> 3847048 bytes tauri/src-tauri/src/audio_output.rs | 5 +- tauri/src-tauri/src/main.rs | 7 --- 17 files changed, 200 insertions(+), 82 deletions(-) rename backend/{platform.py => platform_detect.py} (100%) create mode 100644 backend/voicebox-server.spec diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7d06561..765da827 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,18 +66,17 @@ Thank you for your interest in contributing to Voicebox! This document provides # Install Python dependencies pip install -r requirements.txt + # Install MLX dependencies (Apple Silicon only - for faster inference) + # On Apple Silicon, this enables native Metal acceleration + if [[ $(uname -m) == "arm64" ]]; then + pip install -r requirements-mlx.txt + fi + # Install Qwen3-TTS (required for voice synthesis) pip install git+https://github.com/QwenLM/Qwen3-TTS.git ``` -4. **Initialize database** - ```bash - cd backend - python -c "from database import init_db; init_db()" - ``` - This creates the SQLite database at `data/voicebox.db`. - -5. **Start development servers** +4. **Start development servers** Development requires two terminals: one for the Python backend, one for the Tauri app. @@ -120,8 +119,22 @@ First-time usage will be slower due to model downloads, but subsequent runs will ### Building -**Build Python server binary:** +**Build everything (recommended):** ```bash +bun run build +``` +This automatically: +1. Builds the Python server binary (`./scripts/build-server.sh`) +2. Builds the Tauri desktop app (`cd tauri && bun run tauri build`) + +Creates platform-specific installers (`.dmg`, `.msi`, `.AppImage`) in `tauri/src-tauri/target/release/bundle/`. + +**Note:** The build process detects your platform and includes the appropriate backend (MLX for Apple Silicon, PyTorch for others). + +**Build server binary only:** +```bash +bun run build:server +# or ./scripts/build-server.sh ``` Creates platform-specific binary in `tauri/src-tauri/binaries/` @@ -132,18 +145,11 @@ If you're actively developing or modifying the Qwen3-TTS library, set the `QWEN_ ```bash export QWEN_TTS_PATH=~/path/to/your/Qwen3-TTS -./scripts/build-server.sh +bun run build:server ``` This makes PyInstaller use your local qwen-tts version instead of the pip-installed package. Useful when testing changes to the TTS library before they're published to PyPI or when using an editable install (`pip install -e`). -**Build Tauri desktop app:** -```bash -cd tauri -bun run tauri build -``` -Creates platform-specific installers (`.dmg`, `.msi`, `.AppImage`) - **Build web app:** ```bash cd web diff --git a/Makefile b/Makefile index 48de0784..620f6c8c 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,11 @@ setup-python: $(VENV)/bin/activate ## Set up Python virtual environment and depe @echo -e "$(BLUE)Installing Python dependencies...$(NC)" $(PIP) install --upgrade pip $(PIP) install -r $(BACKEND_DIR)/requirements.txt + @if [ "$$(uname -m)" = "arm64" ] && [ "$$(uname)" = "Darwin" ]; then \ + echo -e "$(BLUE)Detected Apple Silicon - installing MLX dependencies...$(NC)"; \ + $(PIP) install -r $(BACKEND_DIR)/requirements-mlx.txt; \ + echo -e "$(GREEN)✓ MLX backend enabled (native Metal acceleration)$(NC)"; \ + fi $(PIP) install git+https://github.com/QwenLM/Qwen3-TTS.git @echo -e "$(GREEN)✓ Python environment ready$(NC)" diff --git a/README.md b/README.md index d43750ed..725460ab 100644 --- a/README.md +++ b/README.md @@ -180,8 +180,9 @@ Full API documentation available at `http://localhost:8000/docs` when running. | Frontend | React, TypeScript, Tailwind CSS | | State | Zustand, React Query | | Backend | FastAPI (Python) | -| Voice Model | Qwen3-TTS | -| Transcription | Whisper | +| Voice Model | Qwen3-TTS (PyTorch or MLX) | +| Transcription | Whisper (PyTorch or MLX) | +| Inference Engine | MLX (Apple Silicon) / PyTorch (Windows/Linux/Intel) | | Database | SQLite | | Audio | WaveSurfer.js, librosa | @@ -257,7 +258,11 @@ cd backend && pip install -r requirements.txt && cd .. bun run dev ``` -**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org). CUDA-capable GPU recommended (CPU inference supported but slower). +**Prerequisites:** [Bun](https://bun.sh), [Rust](https://rustup.rs), [Python 3.11+](https://python.org). + +**Performance:** +- **Apple Silicon (M1/M2/M3)**: Uses MLX backend with native Metal acceleration for 4-5x faster inference +- **Windows/Linux/Intel Mac**: Uses PyTorch backend (CUDA GPU recommended, CPU supported but slower) ### Project Structure diff --git a/backend/README.md b/backend/README.md index 95b740e7..57163467 100644 --- a/backend/README.md +++ b/backend/README.md @@ -19,8 +19,13 @@ Production-quality FastAPI backend for Qwen3-TTS voice cloning. backend/ ├── main.py # FastAPI app with all routes ├── models.py # Pydantic request/response models -├── tts.py # Qwen3-TTS inference -├── transcribe.py # Whisper ASR +├── platform_detect.py # Platform detection for backend selection +├── tts.py # TTS backend abstraction (delegates to MLX or PyTorch) +├── transcribe.py # STT backend abstraction (delegates to MLX or PyTorch) +├── backends/ # Backend implementations +│ ├── __init__.py # Backend factory and protocols +│ ├── mlx_backend.py # MLX backend (Apple Silicon) +│ └── pytorch_backend.py # PyTorch backend (Windows/Linux/Intel) ├── profiles.py # Voice profile CRUD ├── history.py # Generation history ├── studio.py # Audio editing (TODO) @@ -31,6 +36,15 @@ backend/ └── validation.py # Input validation ``` +### Backend Selection + +Voicebox automatically selects the best backend based on platform: + +- **Apple Silicon (M1/M2/M3)**: Uses MLX backend with native Metal acceleration (4-5x faster) +- **Windows/Linux/Intel Mac**: Uses PyTorch backend (CUDA GPU if available, CPU fallback) + +The backend is detected at runtime via `platform_detect.py`. Both backends implement the same interface, so the API remains consistent across platforms. + ## API Endpoints ### Health & Info @@ -47,12 +61,20 @@ Health check with model status. "status": "healthy", "model_loaded": true, "gpu_available": true, - "vram_used_mb": 1024.5 + "gpu_type": "Metal (Apple Silicon via MLX)", + "backend_type": "mlx", + "vram_used_mb": null } ``` +**Backend Types:** +- `"mlx"` - MLX backend (Apple Silicon with Metal acceleration) +- `"pytorch"` - PyTorch backend (Windows/Linux/Intel Mac) + ### Voice Profiles +**Note:** The database is automatically initialized when the server starts. No manual setup required. + #### `POST /profiles` Create a new voice profile. @@ -266,13 +288,12 @@ data/ pip install -r requirements.txt ``` -### 2. Initialize Database - +**Note:** On Apple Silicon, also install MLX dependencies for faster inference: ```bash -python -c "from database import init_db; init_db()" +pip install -r requirements-mlx.txt ``` -### 3. Download Models (Automatic) +### 2. Download Models (Automatic) The Qwen3-TTS models are automatically downloaded from HuggingFace Hub on first use, similar to how Whisper models work. diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index d7167826..f7c47ba9 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -8,7 +8,7 @@ from typing import Protocol, Optional, Tuple, List from typing_extensions import runtime_checkable import numpy as np -from ..platform import get_backend_type +from ..platform_detect import get_backend_type @runtime_checkable diff --git a/backend/build_binary.py b/backend/build_binary.py index da63907f..de1e3a41 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -41,7 +41,7 @@ def build_server(): '--hidden-import', 'backend.history', '--hidden-import', 'backend.tts', '--hidden-import', 'backend.transcribe', - '--hidden-import', 'backend.platform', + '--hidden-import', 'backend.platform_detect', '--hidden-import', 'backend.backends', '--hidden-import', 'backend.backends.pytorch_backend', '--hidden-import', 'backend.utils.audio', @@ -83,6 +83,9 @@ def build_server(): '--hidden-import', 'mlx_audio.asr', '--collect-submodules', 'mlx', '--collect-submodules', 'mlx_audio', + # Collect MLX data files including Metal shader libraries (.metallib) + '--collect-data', 'mlx', + '--collect-data', 'mlx_audio', ]) else: print("Building for non-Apple Silicon platform - PyTorch only") diff --git a/backend/main.py b/backend/main.py index d34bad46..5662c154 100644 --- a/backend/main.py +++ b/backend/main.py @@ -27,7 +27,7 @@ from . import database, models, profiles, history, tts, transcribe, config, expo from .database import get_db, Generation as DBGeneration, VoiceProfile as DBVoiceProfile from .utils.progress import get_progress_manager from .utils.tasks import get_task_manager -from .platform import get_backend_type +from .platform_detect import get_backend_type app = FastAPI( title="voicebox API", diff --git a/backend/platform.py b/backend/platform_detect.py similarity index 100% rename from backend/platform.py rename to backend/platform_detect.py diff --git a/backend/voicebox-server.spec b/backend/voicebox-server.spec new file mode 100644 index 00000000..080fe068 --- /dev/null +++ b/backend/voicebox-server.spec @@ -0,0 +1,52 @@ +# -*- mode: python ; coding: utf-8 -*- +from PyInstaller.utils.hooks import collect_data_files +from PyInstaller.utils.hooks import collect_submodules +from PyInstaller.utils.hooks import copy_metadata + +datas = [] +hiddenimports = ['backend', 'backend.main', 'backend.config', 'backend.database', 'backend.models', 'backend.profiles', 'backend.history', 'backend.tts', 'backend.transcribe', 'backend.platform_detect', 'backend.backends', 'backend.backends.pytorch_backend', 'backend.utils.audio', 'backend.utils.cache', 'backend.utils.progress', 'backend.utils.hf_progress', 'backend.utils.validation', 'torch', 'transformers', 'fastapi', 'uvicorn', 'sqlalchemy', 'librosa', 'soundfile', 'qwen_tts', 'qwen_tts.inference', 'qwen_tts.inference.qwen3_tts_model', 'qwen_tts.inference.qwen3_tts_tokenizer', 'qwen_tts.core', 'qwen_tts.cli', 'pkg_resources.extern', 'backend.backends.mlx_backend', 'mlx', 'mlx.core', 'mlx.nn', 'mlx_audio', 'mlx_audio.tts', 'mlx_audio.asr'] +datas += collect_data_files('qwen_tts') +datas += collect_data_files('mlx') +datas += collect_data_files('mlx_audio') +datas += copy_metadata('qwen-tts') +hiddenimports += collect_submodules('qwen_tts') +hiddenimports += collect_submodules('jaraco') +hiddenimports += collect_submodules('mlx') +hiddenimports += collect_submodules('mlx_audio') + + +a = Analysis( + ['server.py'], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.datas, + [], + name='voicebox-server', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 2b930db4..6f0317a0 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -90,6 +90,26 @@ chmod +x voicebox-*.AppImage - Slower but works without GPU - Backend automatically falls back to CPU +### MLX "Failed to load the default metallib" error (Apple Silicon) + +**Symptoms:** Generation fails with "library not found" or "metallib" errors + +**Solutions:** +1. **Rebuild server binary** + ```bash + bun run build:server + ``` + The build script should automatically include MLX Metal shader libraries. + +2. **Check MLX installation** + ```bash + pip install -r backend/requirements-mlx.txt + ``` + +3. **Verify backend detection** + - Check server logs for "Backend: MLX" + - If showing "Backend: PYTORCH", MLX may not be installed correctly + ### Audio playback issues **Symptoms:** Generated audio won't play @@ -111,19 +131,27 @@ chmod +x voicebox-*.AppImage **Symptoms:** Generation takes >30 seconds **Solutions:** -1. **Use GPU** (if available) +1. **Check backend type** (Apple Silicon) + - Check Settings → Server Status + - Should show "Backend: MLX" on Apple Silicon + - If showing "Backend: PYTORCH", install MLX: `pip install -r backend/requirements-mlx.txt` + - MLX provides 4-5x faster inference on Apple Silicon + +2. **Use GPU** (if available) - Check Settings → Server Status - Should show "GPU available: true" + - Apple Silicon: Should show "Metal (Apple Silicon via MLX)" + - Windows/Linux: Should show "CUDA" if GPU available -2. **Enable caching** +3. **Enable caching** - Voice prompts are cached automatically - Second generation with same voice should be faster -3. **Use smaller model** +4. **Use smaller model** - 0.6B model is faster than 1.7B - Quality difference is minimal for most voices -4. **Check system resources** +5. **Check system resources** - Close other CPU/GPU intensive apps - Ensure adequate RAM (8GB+ recommended) diff --git a/docs/developer/architecture.mdx b/docs/developer/architecture.mdx index a6503f0d..367d566a 100644 --- a/docs/developer/architecture.mdx +++ b/docs/developer/architecture.mdx @@ -121,8 +121,12 @@ bun run dev ### Production ```bash +# Build everything (server binary + Tauri app) +bun run build + +# Or build separately: # 1. Build server binary (PyInstaller) -./scripts/build-server.sh +bun run build:server # 2. Build Tauri app (includes server) cd tauri && bun run tauri build diff --git a/docs/developer/building.mdx b/docs/developer/building.mdx index 860cf211..237805b9 100644 --- a/docs/developer/building.mdx +++ b/docs/developer/building.mdx @@ -10,46 +10,52 @@ Voicebox uses a multi-step build process to create platform-specific installers. ## Quick Build ```bash -# Build for your current platform +# Build for your current platform (automatically builds server binary first) make build # Or manually -cd tauri && bun run tauri build +bun run build ``` -## Build Steps +This automatically: +1. Builds the Python server binary (`bun run build:server`) +2. Builds the Tauri app (`cd tauri && bun run tauri build`) -### 1. Build Server Binary +## Build Process -The Python backend must be compiled into a standalone executable first: +The build process consists of two steps, but `bun run build` handles both automatically: -```bash -./scripts/build-server.sh -``` +### 1. Server Binary Build (Automatic) -This uses PyInstaller to create a binary in `tauri/src-tauri/binaries/`. +The Python backend is compiled into a standalone executable using PyInstaller. This happens automatically when you run `bun run build`. **Platform-specific binaries:** -- macOS: `voicebox-server-aarch64-apple-darwin` or `voicebox-server-x86_64-apple-darwin` -- Windows: `voicebox-server-x86_64-pc-windows-msvc.exe` -- Linux: `voicebox-server-x86_64-unknown-linux-gnu` +- macOS (Apple Silicon): `voicebox-server-aarch64-apple-darwin` (includes MLX backend) +- macOS (Intel): `voicebox-server-x86_64-apple-darwin` (PyTorch backend) +- Windows: `voicebox-server-x86_64-pc-windows-msvc.exe` (PyTorch backend) +- Linux: `voicebox-server-x86_64-unknown-linux-gnu` (PyTorch backend) - The build script automatically detects your platform and creates the appropriate binary. + The build script automatically detects your platform and includes the appropriate backend (MLX for Apple Silicon, PyTorch for others). -### 2. Build Tauri App - +**Manual build (if needed):** ```bash -cd tauri -bun run tauri build +bun run build:server ``` -This will: -1. Build the React frontend (Vite) -2. Compile the Rust backend -3. Bundle the server binary as a sidecar -4. Create platform-specific installers +### 2. Tauri App Build (Automatic) + +The Tauri app build is also handled automatically, which: +1. Builds the React frontend (Vite) +2. Compiles the Rust backend +3. Bundles the server binary as a sidecar +4. Creates platform-specific installers + +**Manual build (if needed):** +```bash +cd tauri && bun run tauri build +``` ### 3. Output @@ -91,7 +97,9 @@ If you're developing Qwen3-TTS locally: ```bash export QWEN_TTS_PATH=~/path/to/Qwen3-TTS -./scripts/build-server.sh +bun run build:server # Build server binary only +# or +bun run build # Build everything ``` This makes PyInstaller use your local version instead of the pip package. @@ -216,7 +224,7 @@ See [CONTRIBUTING.md](/development/contributing) for the full release workflow. **Common issues:** - Rust not installed: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` - - Server binary missing: Run `./scripts/build-server.sh` first + - Server binary missing: Usually auto-built, but can run manually: `./scripts/build-server.sh` - Node modules outdated: `bun install` **Solution:** @@ -225,8 +233,7 @@ See [CONTRIBUTING.md](/development/contributing) for the full release workflow. cd tauri/src-tauri cargo clean cd ../.. - ./scripts/build-server.sh - bun run tauri build + bun run build # Automatically builds server binary first ``` diff --git a/docs/developer/setup.mdx b/docs/developer/setup.mdx index b0737751..c2c44f24 100644 --- a/docs/developer/setup.mdx +++ b/docs/developer/setup.mdx @@ -80,19 +80,16 @@ venv\Scripts\activate # Windows # Install Python dependencies pip install -r requirements.txt +# Install MLX dependencies (Apple Silicon only - for faster inference) +# On Apple Silicon, this enables native Metal acceleration +if [[ $(uname -m) == "arm64" ]]; then + pip install -r requirements-mlx.txt +fi + # Install Qwen3-TTS pip install git+https://github.com/QwenLM/Qwen3-TTS.git ``` -### 3. Initialize Database - -```bash -cd backend -python -c "from database import init_db; init_db()" -``` - -This creates the SQLite database at `data/voicebox.db`. - ## Running in Development Development requires **two terminals**: one for the Python backend, one for the Tauri app. diff --git a/package.json b/package.json index 7e009135..6c3e9057 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "dev:landing": "cd landing && bun run dev", "dev:server": "uvicorn backend.main:app --reload --port 17493", "setup:dev": "bun run scripts/setup-dev-sidecar.js", - "build": "cd tauri && bun run tauri build", + "build": "./scripts/build-server.sh && cd tauri && bun run tauri build", "build:web": "cd web && bun run build", "build:landing": "cd landing && bun run build", "build:release": "./scripts/prepare-release.sh", diff --git a/tauri/src-tauri/gen/Assets.car b/tauri/src-tauri/gen/Assets.car index 7e4e205bbd72335766990847675b871c56710809..d921b3734599b2ac3b79aa7eba3df7342b2d43ff 100644 GIT binary patch delta 818 zcmZwFyG~S56b4|%5gD$d0*VL%D&7~f_t}@T7lns?JK+&1?d*)Dg@uX5(2|L<@c~F^ zOehYe4`9M$I5Ea(Me9$53QqF1|N7V2Yxl>`)-HTKUSIBxjEu~{ICNkVCSV$-4*#wn zclX5T;E&lFyt1oufOP#W_v|bv9?I*{wRC+LgA#7SbBBI(gEiLy#>Jm9{oInzFt$h(*onf8nL(UF^)a zDy_3~(q|G}Flp%wIV?!N^vMY5a%gejPY8R)`wxB)j|7Up0cZozFh&A0=La2J+f8ScRf ztil@HhX?Qw*5MI6h9~e8o;5q$&reROoSnxrTT03)(k=!HGD!+1#cZunB3*J8S;&;n z&Z57+^#TYspa%jZ6rezZY4*2l^SdgH%SD+YMSFT5&UxZoTJk;xT%TqR`odoKfZ_@QDxP1M^MS%;zBz=Fk~^?7mKGK!7P~Fk7#lBugvNy8 zqx1qyxC}SO7_Df1$V#Tsj|Vc@$1-+(JH4p(6UuEAtI@Xb8YiOS~U9Vc5Df+}Qk-Y_bm2%(Y* zI@!@V+&IIBb5`nJr}+^Fw#5bK9hs_RgtQ~aSSA^{@&UR+{)|><=`c$m;!mJ3@IC#P$e>> = Arc::new(Mutex::new(interleaved)); let position = Arc::new(AtomicUsize::new(0)); diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index 146b5cc2..255655aa 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -635,13 +635,6 @@ pub fn run() { } } - // Get all windows and open devtools on the first one - if let Some((_, window)) = app.webview_windows().iter().next() { - window.open_devtools(); - println!("Dev tools opened"); - } else { - println!("No window found to open dev tools"); - } Ok(()) }) .invoke_handler(tauri::generate_handler![