mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-16 13:20:39 -07:00
Add CUDA binary compression test script
GitHub has a 2GB limit on release assets, but the CUDA binary is ~2.5GB. Added compression test script to check if 7z can get it under the limit. If not, we'll need external hosting (S3/Azure).
This commit is contained in:
Jamie Pine
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test CUDA binary compression to verify it fits under GitHub's 2GB release asset limit.
|
||||
|
||||
Usage:
|
||||
python test_cuda_compression.py [path/to/voicebox-server-cuda.exe]
|
||||
|
||||
If no path provided, looks for the binary in ./dist/
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def format_size(bytes_size):
|
||||
"""Format bytes into human-readable size."""
|
||||
for unit in ['B', 'KB', 'MB', 'GB']:
|
||||
if bytes_size < 1024.0:
|
||||
return f"{bytes_size:.2f} {unit}"
|
||||
bytes_size /= 1024.0
|
||||
return f"{bytes_size:.2f} TB"
|
||||
|
||||
|
||||
def get_file_size(filepath):
|
||||
"""Get file size in bytes."""
|
||||
return os.path.getsize(filepath)
|
||||
|
||||
|
||||
def compress_with_7z(input_file, output_file):
|
||||
"""Compress file using 7z with maximum compression."""
|
||||
print(f"\nCompressing with 7z (maximum compression)...")
|
||||
print(f"This may take several minutes for a ~2.5GB file...\n")
|
||||
|
||||
cmd = [
|
||||
'7z', 'a',
|
||||
'-t7z', # 7z format
|
||||
'-m0=lzma2', # LZMA2 compression
|
||||
'-mx=9', # Maximum compression
|
||||
'-mfb=64', # Fast bytes
|
||||
'-md=32m', # Dictionary size
|
||||
'-ms=on', # Solid archive
|
||||
output_file,
|
||||
input_file
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error during compression: {e}")
|
||||
print(f"stderr: {e.stderr}")
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
print("ERROR: 7z not found. Please install 7-Zip:")
|
||||
print(" Windows: https://www.7-zip.org/download.html")
|
||||
print(" macOS: brew install p7zip")
|
||||
print(" Linux: apt-get install p7zip-full")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
# Find CUDA binary
|
||||
if len(sys.argv) > 1:
|
||||
cuda_binary = Path(sys.argv[1])
|
||||
else:
|
||||
# Look in dist directory
|
||||
dist_dir = Path(__file__).parent / 'dist'
|
||||
candidates = list(dist_dir.glob('voicebox-server-cuda*.exe'))
|
||||
|
||||
if not candidates:
|
||||
print("ERROR: CUDA binary not found in ./dist/")
|
||||
print("Please provide the path as an argument:")
|
||||
print(" python test_cuda_compression.py path/to/voicebox-server-cuda.exe")
|
||||
sys.exit(1)
|
||||
|
||||
cuda_binary = candidates[0]
|
||||
|
||||
if not cuda_binary.exists():
|
||||
print(f"ERROR: File not found: {cuda_binary}")
|
||||
sys.exit(1)
|
||||
|
||||
print("=" * 70)
|
||||
print("CUDA Binary Compression Test")
|
||||
print("=" * 70)
|
||||
|
||||
# Get original size
|
||||
original_size = get_file_size(cuda_binary)
|
||||
print(f"\nOriginal file: {cuda_binary.name}")
|
||||
print(f"Original size: {format_size(original_size)} ({original_size:,} bytes)")
|
||||
|
||||
# Check if already over 2GB
|
||||
github_limit = 2 * 1024 * 1024 * 1024 # 2GB in bytes
|
||||
print(f"GitHub limit: {format_size(github_limit)} ({github_limit:,} bytes)")
|
||||
|
||||
if original_size > github_limit:
|
||||
print(f"\n⚠️ Original file exceeds GitHub limit by {format_size(original_size - github_limit)}")
|
||||
else:
|
||||
print(f"\n✓ Original file is under GitHub limit")
|
||||
|
||||
# Compress
|
||||
output_file = cuda_binary.parent / f"{cuda_binary.stem}.7z"
|
||||
if output_file.exists():
|
||||
print(f"\nRemoving existing compressed file: {output_file.name}")
|
||||
output_file.unlink()
|
||||
|
||||
success = compress_with_7z(cuda_binary, output_file)
|
||||
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
|
||||
# Check compressed size
|
||||
compressed_size = get_file_size(output_file)
|
||||
compression_ratio = (1 - compressed_size / original_size) * 100
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("Compression Results")
|
||||
print("=" * 70)
|
||||
print(f"\nCompressed file: {output_file.name}")
|
||||
print(f"Compressed size: {format_size(compressed_size)} ({compressed_size:,} bytes)")
|
||||
print(f"Compression ratio: {compression_ratio:.1f}%")
|
||||
print(f"Space saved: {format_size(original_size - compressed_size)}")
|
||||
|
||||
if compressed_size <= github_limit:
|
||||
print(f"\n✅ SUCCESS: Compressed file fits under GitHub's 2GB limit!")
|
||||
print(f" Margin: {format_size(github_limit - compressed_size)} remaining")
|
||||
else:
|
||||
print(f"\n❌ FAILED: Compressed file still exceeds GitHub limit")
|
||||
print(f" Over by: {format_size(compressed_size - github_limit)}")
|
||||
print(f"\n Alternative: Host on external storage (S3, Azure Blob, etc.)")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
# GitHub 2GB Release Asset Limit Issue
|
||||
|
||||
## Problem
|
||||
|
||||
The CUDA server binary upload fails in CI with:
|
||||
```
|
||||
Error: File size (2543828017) is greater than 2 GiB
|
||||
```
|
||||
|
||||
GitHub release assets have a hard limit of 2GB per file. Our CUDA binary is ~2.5GB, which exceeds this limit.
|
||||
|
||||
## Background
|
||||
|
||||
The dual-server binary system (see `dual-server-binaries.md`) creates two binaries:
|
||||
- **CPU binary**: ~500MB ✅ Works fine
|
||||
- **CUDA binary**: ~2.5GB ❌ Exceeds GitHub limit
|
||||
|
||||
## Attempted Solution: Compression
|
||||
|
||||
We're testing 7z compression with maximum settings to see if we can squeeze the CUDA binary under 2GB.
|
||||
|
||||
### Test Script
|
||||
|
||||
Run `backend/test_cuda_compression.py` to test compression locally:
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
python test_cuda_compression.py
|
||||
```
|
||||
|
||||
This will:
|
||||
1. Find the CUDA binary in `dist/`
|
||||
2. Compress it with 7z (maximum compression)
|
||||
3. Report if the compressed size fits under 2GB
|
||||
|
||||
### Expected Compression
|
||||
|
||||
PyTorch CUDA binaries typically compress well since they contain:
|
||||
- Repeated patterns in neural network weights
|
||||
- Debug symbols and metadata
|
||||
- Redundant CUDA libraries
|
||||
|
||||
Estimated compression: 30-40% reduction
|
||||
- Original: ~2.5GB
|
||||
- Target: <2GB
|
||||
- Required compression: >20%
|
||||
|
||||
## Fallback: External Hosting
|
||||
|
||||
If compression doesn't work, we'll need to host the CUDA binary externally:
|
||||
|
||||
### Option 1: AWS S3
|
||||
```yaml
|
||||
- name: Upload CUDA binary to S3
|
||||
run: |
|
||||
aws s3 cp backend/cuda-release/voicebox-server-cuda-*.exe \
|
||||
s3://voicebox-releases/cuda-binaries/${{ github.ref_name }}/
|
||||
```
|
||||
|
||||
### Option 2: Azure Blob Storage
|
||||
```yaml
|
||||
- name: Upload to Azure Blob
|
||||
run: |
|
||||
az storage blob upload \
|
||||
--account-name voiceboxreleases \
|
||||
--container-name cuda-binaries \
|
||||
--file backend/cuda-release/voicebox-server-cuda-*.exe
|
||||
```
|
||||
|
||||
### Option 3: GitHub Packages (Container Registry)
|
||||
Package as a container image, though this adds complexity for desktop app distribution.
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
1. **Test compression locally** ← Current step
|
||||
2. **If compression works (<2GB)**:
|
||||
- Update CI to compress before upload
|
||||
- Update app to handle .7z downloads
|
||||
- Add extraction step in download manager
|
||||
|
||||
3. **If compression fails (≥2GB)**:
|
||||
- Set up external storage (likely S3)
|
||||
- Update CI to upload to S3
|
||||
- Provide download URL in release notes
|
||||
- Update app download manager to fetch from S3
|
||||
|
||||
## CI Workflow Changes (if compression works)
|
||||
|
||||
```yaml
|
||||
- name: Compress CUDA binary (Windows only)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
shell: bash
|
||||
run: |
|
||||
cd backend/cuda-release
|
||||
7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on \
|
||||
voicebox-server-cuda-x86_64-pc-windows-msvc.7z \
|
||||
voicebox-server-cuda-*.exe
|
||||
|
||||
- name: Upload compressed CUDA server (Windows only)
|
||||
if: matrix.platform == 'windows-latest'
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: backend/cuda-release/*.7z
|
||||
```
|
||||
|
||||
## User Experience Impact
|
||||
|
||||
### With Compression
|
||||
- Download: `voicebox-server-cuda-*.7z` (~1.5-1.8GB)
|
||||
- App extracts automatically
|
||||
- One extra step but manageable
|
||||
|
||||
### With External Hosting
|
||||
- Download from S3/Azure URL
|
||||
- No GitHub release asset dependency
|
||||
- Potentially faster download speeds (CDN)
|
||||
|
||||
## Status
|
||||
|
||||
🔄 **Testing compression locally to determine viability**
|
||||
|
||||
Results pending from local test run.
|
||||
Reference in New Issue
Block a user