Labyricorn/voicebox
1
0
Fork 0
mirror of https://github.com/jamiepine/voicebox.git synced 2026-10-03 09:05:17 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add .npmrc for bun usage and update dependencies

Browse Source 
- Created a new .npmrc file to enforce bun usage.
- Bumped version numbers for multiple packages to 0.1.9 in bun.lock.
- Added react-sound-visualizer dependency to enhance audio visualization features.
- Introduced convert:assets script in package.json for asset optimization.
- Updated CONTRIBUTING.md with instructions for converting assets to web formats.
- Added documentation files for API endpoints and developer guidelines in the docs directory.
This commit is contained in:
Jamie Pine
2026-01-29 18:56:10 -08:00
parent 462f104494
commit 7b5e73cfa8
58 changed files with 8014 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.npmrc
+2
View File
@@ -0,0 +1,2 @@
# Force bun usage
engine-strict=true
CONTRIBUTING.md
+20
View File
@@ -159,6 +159,26 @@ After starting the backend server:
``` ```
This downloads the OpenAPI schema and generates the TypeScript client in `app/src/lib/api/` This downloads the OpenAPI schema and generates the TypeScript client in `app/src/lib/api/`
### Convert Assets to Web Formats
To optimize images and videos for the web, run:
```bash
bun run convert:assets
```
This script:
- Converts PNG → WebP (better compression, same quality)
- Converts MOV → WebM (VP9 codec, smaller file size)
- Processes files in `landing/public/` and `docs/public/`
- **Deletes original files** after successful conversion
**Requirements:** Install `webp` and `ffmpeg`:
```bash
brew install webp ffmpeg
```
> **Note:** Run this before committing new images or videos to keep the repository size small.
## Development Workflow ## Development Workflow
### 1. Create a Branch ### 1. Create a Branch
app/package.json
+1
View File
@@ -48,6 +48,7 @@
"react": "^18.3.0", "react": "^18.3.0",
"react-dom": "^18.3.0", "react-dom": "^18.3.0",
"react-hook-form": "^7.53.0", "react-hook-form": "^7.53.0",
"react-sound-visualizer": "^1.4.0",
"tailwind-merge": "^2.5.4", "tailwind-merge": "^2.5.4",
"wavesurfer.js": "^7.0.0", "wavesurfer.js": "^7.0.0",
"zod": "^3.23.8", "zod": "^3.23.8",
app/src/components/VoiceProfiles/AudioSampleRecording.tsx
+72 -11
View File
@@ -1,8 +1,31 @@
import { Mic, Pause, Play, Square } from 'lucide-react'; import { Mic, Pause, Play, Square } from 'lucide-react';
import { memo, useEffect, useState } from 'react';
import { Visualizer } from 'react-sound-visualizer';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { FormControl, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; import { FormControl, FormItem, FormMessage } from '@/components/ui/form';
import { formatAudioDuration } from '@/lib/utils/audio'; import { formatAudioDuration } from '@/lib/utils/audio';
const MemoizedWaveform = memo(function MemoizedWaveform({
audioStream,
}: {
audioStream: MediaStream;
}) {
return (
<div className="absolute inset-0 pointer-events-none flex items-center justify-center opacity-30">
<Visualizer audio={audioStream} autoStart strokeColor="#b39a3d">
{({ canvasRef }) => (
<canvas
ref={canvasRef}
width={500}
height={150}
className="w-full h-full"
/>
)}
</Visualizer>
</div>
);
});
interface AudioSampleRecordingProps { interface AudioSampleRecordingProps {
file: File | null | undefined; file: File | null | undefined;
isRecording: boolean; isRecording: boolean;
@@ -14,6 +37,7 @@ interface AudioSampleRecordingProps {
onPlayPause: () => void; onPlayPause: () => void;
isPlaying: boolean; isPlaying: boolean;
isTranscribing?: boolean; isTranscribing?: boolean;
showWaveform?: boolean;
} }
export function AudioSampleRecording({ export function AudioSampleRecording({
@@ -27,29 +51,67 @@ export function AudioSampleRecording({
onPlayPause, onPlayPause,
isPlaying, isPlaying,
isTranscribing = false, isTranscribing = false,
showWaveform = true,
}: AudioSampleRecordingProps) { }: AudioSampleRecordingProps) {
const [audioStream, setAudioStream] = useState<MediaStream | null>(null);
// Request microphone access when component mounts
useEffect(() => {
if (!showWaveform) return;
let stream: MediaStream | null = null;
navigator.mediaDevices
.getUserMedia({ audio: true, video: false })
.then((s) => {
stream = s;
setAudioStream(s);
})
.catch((err) => {
console.warn('Could not access microphone for visualization:', err);
});
return () => {
if (stream) {
stream.getTracks().forEach((track) => {
track.stop();
});
}
};
}, [showWaveform]);
return ( return (
<FormItem> <FormItem>
<FormLabel>Record Audio</FormLabel>
<FormControl> <FormControl>
<div className="space-y-4"> <div className="space-y-4">
{!isRecording && !file && ( {!isRecording && !file && (
<div className="flex flex-col items-center justify-center gap-4 p-4 border-2 border-dashed rounded-lg min-h-[180px]"> <div className="relative flex flex-col items-center justify-center gap-4 p-4 border-2 border-dashed rounded-lg min-h-[180px] overflow-hidden">
<Button type="button" onClick={onStart} size="lg" className="flex items-center gap-2"> {showWaveform && audioStream && (
<MemoizedWaveform audioStream={audioStream} />
)}
<Button
type="button"
onClick={onStart}
size="lg"
className="relative z-10 flex items-center gap-2"
>
<Mic className="h-5 w-5" /> <Mic className="h-5 w-5" />
Start Recording Start Recording
</Button> </Button>
<p className="text-sm text-muted-foreground text-center"> <p className="relative z-10 text-sm text-muted-foreground text-center">
Click to start recording. Maximum duration: 30 seconds. Click to start recording. Maximum duration: 30 seconds.
</p> </p>
</div> </div>
)} )}
{isRecording && ( {isRecording && (
<div className="flex flex-col items-center justify-center gap-4 p-4 border-2 border-destructive rounded-lg bg-destructive/5 min-h-[180px]"> <div className="relative flex flex-col items-center justify-center gap-4 p-4 border-2 border-accent rounded-lg bg-accent/5 min-h-[180px] overflow-hidden">
<div className="flex items-center gap-4"> {showWaveform && audioStream && (
<MemoizedWaveform audioStream={audioStream} />
)}
<div className="relative z-10 flex items-center gap-4">
<div className="flex items-center gap-2"> <div className="flex items-center gap-2">
<div className="h-3 w-3 rounded-full bg-destructive animate-pulse" /> <div className="h-3 w-3 rounded-full bg-accent animate-pulse" />
<span className="text-lg font-mono font-semibold"> <span className="text-lg font-mono font-semibold">
{formatAudioDuration(duration)} {formatAudioDuration(duration)}
</span> </span>
@@ -58,13 +120,12 @@ export function AudioSampleRecording({
<Button <Button
type="button" type="button"
onClick={onStop} onClick={onStop}
variant="destructive" className="relative z-10 flex items-center gap-2 bg-accent text-accent-foreground hover:bg-accent/90"
className="flex items-center gap-2"
> >
<Square className="h-4 w-4" /> <Square className="h-4 w-4" />
Stop Recording Stop Recording
</Button> </Button>
<p className="text-sm text-muted-foreground text-center"> <p className="relative z-10 text-sm text-muted-foreground text-center">
{formatAudioDuration(30 - duration)} remaining {formatAudioDuration(30 - duration)} remaining
</p> </p>
</div> </div>
app/src/components/VoiceProfiles/AudioSampleSystem.tsx
+1 -2
View File
@@ -1,6 +1,6 @@
import { Mic, Monitor, Pause, Play, Square } from 'lucide-react'; import { Mic, Monitor, Pause, Play, Square } from 'lucide-react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { FormControl, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; import { FormControl, FormItem, FormMessage } from '@/components/ui/form';
import { formatAudioDuration } from '@/lib/utils/audio'; import { formatAudioDuration } from '@/lib/utils/audio';
interface AudioSampleSystemProps { interface AudioSampleSystemProps {
@@ -30,7 +30,6 @@ export function AudioSampleSystem({
}: AudioSampleSystemProps) { }: AudioSampleSystemProps) {
return ( return (
<FormItem> <FormItem>
<FormLabel>Capture System Audio</FormLabel>
<FormControl> <FormControl>
<div className="space-y-4"> <div className="space-y-4">
{!isRecording && !file && ( {!isRecording && !file && (
app/src/components/VoiceProfiles/AudioSampleUpload.tsx
+1 -2
View File
@@ -1,7 +1,7 @@
import { Mic, Pause, Play, Upload } from 'lucide-react'; import { Mic, Pause, Play, Upload } from 'lucide-react';
import { useRef, useState } from 'react'; import { useRef, useState } from 'react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { FormControl, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; import { FormControl, FormItem, FormMessage } from '@/components/ui/form';
interface AudioSampleUploadProps { interface AudioSampleUploadProps {
file: File | null | undefined; file: File | null | undefined;
@@ -31,7 +31,6 @@ export function AudioSampleUpload({
return ( return (
<FormItem> <FormItem>
<FormLabel>Audio File</FormLabel>
<FormControl> <FormControl>
<div className="flex flex-col gap-2"> <div className="flex flex-col gap-2">
<input <input
app/src/components/VoiceProfiles/ProfileForm.tsx
+68 -72
View File
@@ -110,7 +110,7 @@ export function ProfileForm() {
const addSample = useAddSample(); const addSample = useAddSample();
const transcribe = useTranscription(); const transcribe = useTranscription();
const { toast } = useToast(); const { toast } = useToast();
const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('upload'); const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('record');
const [audioDuration, setAudioDuration] = useState<number | null>(null); const [audioDuration, setAudioDuration] = useState<number | null>(null);
const [isValidatingAudio, setIsValidatingAudio] = useState(false); const [isValidatingAudio, setIsValidatingAudio] = useState(false);
const { isPlaying, playPause, cleanup: cleanupAudio } = useAudioPlayer(); const { isPlaying, playPause, cleanup: cleanupAudio } = useAudioPlayer();
@@ -282,7 +282,7 @@ export function ProfileForm() {
sampleFile: undefined, sampleFile: undefined,
referenceText: undefined, referenceText: undefined,
}); });
setSampleMode('upload'); setSampleMode('record');
} }
}, [editingProfile, profileFormDraft, open, form]); }, [editingProfile, profileFormDraft, open, form]);
@@ -488,9 +488,10 @@ export function ProfileForm() {
return ( return (
<Dialog open={open} onOpenChange={handleOpenChange}> <Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="max-w-4xl"> <DialogContent className="max-w-none w-screen h-screen left-0 top-0 translate-x-0 translate-y-0 rounded-none p-6 overflow-y-auto">
<div className="max-w-5xl max-h-[85vh] mx-auto my-auto w-full flex flex-col">
<DialogHeader> <DialogHeader>
<DialogTitle>{editingProfileId ? 'Edit Voice' : 'Create Voice Profile'}</DialogTitle> <DialogTitle className="text-2xl">{editingProfileId ? 'Edit Voice' : 'Clone voice'}</DialogTitle>
<DialogDescription> <DialogDescription>
{editingProfileId {editingProfileId
? 'Update your voice profile details and manage samples.' ? 'Update your voice profile details and manage samples.'
@@ -513,7 +514,7 @@ export function ProfileForm() {
sampleFile: undefined, sampleFile: undefined,
referenceText: '', referenceText: '',
}); });
setSampleMode('upload'); setSampleMode('record');
}} }}
> >
<X className="h-3 w-3 mr-1" /> <X className="h-3 w-3 mr-1" />
@@ -524,76 +525,14 @@ export function ProfileForm() {
</DialogHeader> </DialogHeader>
<Form {...form}> <Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)}> <form onSubmit={form.handleSubmit(onSubmit)} className="flex-1 min-h-0 flex flex-col">
<div className="grid gap-6 grid-cols-2"> <div className="grid gap-6 grid-cols-2 flex-1 overflow-y-auto min-h-0">
{/* Left column: Profile info */} {/* Left column: Sample management */}
<div className="space-y-4"> <div className="space-y-4 border-r pr-6">
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Name</FormLabel>
<FormControl>
<Input placeholder="My Voice" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>Description (Optional)</FormLabel>
<FormControl>
<Textarea placeholder="Describe this voice..." {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="language"
render={({ field }) => (
<FormItem>
<FormLabel>Language</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
{LANGUAGE_OPTIONS.map((lang) => (
<SelectItem key={lang.value} value={lang.value}>
{lang.label}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
</div>
{/* Right column: Sample management */}
<div className="space-y-4 border-l pl-6">
{isCreating ? ( {isCreating ? (
<> <>
<div>
<h3 className="text-sm font-medium mb-2">Add Sample</h3>
<p className="text-sm text-muted-foreground mb-4">
Provide an audio sample to clone the voice. You can add more samples later.
</p>
</div>
<Tabs <Tabs
className="pt-4"
value={sampleMode} value={sampleMode}
onValueChange={(v) => { onValueChange={(v) => {
const newMode = v as 'upload' | 'record' | 'system'; const newMode = v as 'upload' | 'record' | 'system';
@@ -720,6 +659,62 @@ export function ProfileForm() {
) )
)} )}
</div> </div>
{/* Right column: Profile info */}
<div className="space-y-4">
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Name</FormLabel>
<FormControl>
<Input placeholder="My Voice" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>Description (Optional)</FormLabel>
<FormControl>
<Textarea placeholder="Describe this voice..." {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="language"
render={({ field }) => (
<FormItem>
<FormLabel>Language</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
</FormControl>
<SelectContent>
{LANGUAGE_OPTIONS.map((lang) => (
<SelectItem key={lang.value} value={lang.value}>
{lang.label}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
</div>
</div> </div>
<div className="flex gap-2 justify-end mt-6 pt-4 border-t"> <div className="flex gap-2 justify-end mt-6 pt-4 border-t">
@@ -739,6 +734,7 @@ export function ProfileForm() {
</div> </div>
</form> </form>
</Form> </Form>
</div>
</DialogContent> </DialogContent>
</Dialog> </Dialog>
); );
bun.lock
+9 -4
View File
@@ -13,7 +13,7 @@
}, },
"app": { "app": {
"name": "@voicebox/app", "name": "@voicebox/app",
"version": "0.1.5", "version": "0.1.9",
"dependencies": { "dependencies": {
"@dnd-kit/core": "^6.3.1", "@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0", "@dnd-kit/sortable": "^10.0.0",
@@ -50,6 +50,7 @@
"react": "^18.3.0", "react": "^18.3.0",
"react-dom": "^18.3.0", "react-dom": "^18.3.0",
"react-hook-form": "^7.53.0", "react-hook-form": "^7.53.0",
"react-sound-visualizer": "^1.4.0",
"tailwind-merge": "^2.5.4", "tailwind-merge": "^2.5.4",
"wavesurfer.js": "^7.0.0", "wavesurfer.js": "^7.0.0",
"zod": "^3.23.8", "zod": "^3.23.8",
@@ -67,7 +68,7 @@
}, },
"landing": { "landing": {
"name": "@voicebox/landing", "name": "@voicebox/landing",
"version": "0.1.5", "version": "0.1.9",
"dependencies": { "dependencies": {
"@radix-ui/react-separator": "^1.1.8", "@radix-ui/react-separator": "^1.1.8",
"@radix-ui/react-slot": "^1.2.4", "@radix-ui/react-slot": "^1.2.4",
@@ -92,7 +93,7 @@
}, },
"tauri": { "tauri": {
"name": "@voicebox/tauri", "name": "@voicebox/tauri",
"version": "0.1.5", "version": "0.1.9",
"dependencies": { "dependencies": {
"@tauri-apps/api": "^2.0.0", "@tauri-apps/api": "^2.0.0",
"@tauri-apps/plugin-shell": "^2.0.0", "@tauri-apps/plugin-shell": "^2.0.0",
@@ -111,7 +112,7 @@
}, },
"web": { "web": {
"name": "@voicebox/web", "name": "@voicebox/web",
"version": "0.1.5", "version": "0.1.9",
"dependencies": { "dependencies": {
"@tanstack/react-query": "^5.0.0", "@tanstack/react-query": "^5.0.0",
"react": "^18.3.0", "react": "^18.3.0",
@@ -970,6 +971,8 @@
"react-remove-scroll-bar": ["[email protected]", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], "react-remove-scroll-bar": ["[email protected]", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="],
"react-sound-visualizer": ["[email protected]", "", { "dependencies": { "sound-visualizer": "^1.2.0" }, "peerDependencies": { "react": ">= 16" } }, "sha512-Qe7tFTd1owtQ8nYrUYXg7QLt8mw7iUy86mqj/+IwmXzSw+NlhnMnAGPuisb1Lk3ncliFnM+AQbZb3C4RQN9uMQ=="],
"react-style-singleton": ["[email protected]", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], "react-style-singleton": ["[email protected]", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="],
"read-cache": ["[email protected]", "", { "dependencies": { "pify": "^2.3.0" } }, "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA=="], "read-cache": ["[email protected]", "", { "dependencies": { "pify": "^2.3.0" } }, "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA=="],
@@ -1004,6 +1007,8 @@
"slash": ["[email protected]", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], "slash": ["[email protected]", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="],
"sound-visualizer": ["[email protected]", "", {}, "sha512-2+Un0PrrBgXylnCjrVYUoRW7KEDH29h7O8/MGzeDOgFGBPb9oX/2n/RGBxJXvVv2U3KFwX5olUWeJKf0Rr5TLQ=="],
"source-map-js": ["[email protected]", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], "source-map-js": ["[email protected]", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
"strip-ansi": ["[email protected]", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "strip-ansi": ["[email protected]", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
docs/.gitignore
+3
View File
@@ -0,0 +1,3 @@
node_modules
.mintlify
.DS_Store
docs/README.md
+64
View File
@@ -0,0 +1,64 @@
# Voicebox Documentation
This directory contains the documentation for Voicebox, built with [Mintlify](https://mintlify.com).
## Development
### Prerequisites
Install Mintlify globally using bun:
```bash
bun add -g mintlify
```
Or use the helper script:
```bash
bun run install:mintlify
```
### Running Locally
```bash
bun run dev
```
This will start the Mintlify dev server.
The docs will be available at `http://localhost:3000`
### Structure
```
docs/
├── mint.json # Mintlify configuration
├── custom.css # Custom styles
├── overview/ # Getting started & feature docs
├── guides/ # User guides
├── api/ # API reference
├── development/ # Developer documentation
├── logo/ # Logo assets
└── public/ # Static assets
```
### Writing Docs
- Use `.mdx` files for all documentation pages
- Follow the existing structure in `mint.json` for navigation
- Use Mintlify components for enhanced formatting (Card, CardGroup, Accordion, etc.)
- Reference the [Mintlify documentation](https://mintlify.com/docs) for available components
## Deployment
Docs are automatically deployed when changes are pushed to the main branch.
To manually deploy:
```bash
mintlify deploy
```
## Contributing
See [CONTRIBUTING.md](../CONTRIBUTING.md) for contribution guidelines.
docs/api/authentication.mdx
+55
View File
@@ -0,0 +1,55 @@
---
title: "Authentication"
description: "API authentication and security"
---
## Current Status
<Warning>
Authentication is not currently implemented in Voicebox. The API is intended for local use only.
</Warning>
## Local Usage
For local development and usage:
- API runs on `localhost:17493`
- No authentication required
- Access restricted to local machine
## Future Implementation
Authentication will be added in a future release for:
- Remote deployments
- Multi-user access
- Production environments
Planned authentication methods:
- API keys
- OAuth 2.0
- JWT tokens
## Security Best Practices
Until authentication is implemented:
<CardGroup cols={2}>
<Card title="Use VPN" icon="shield">
Use WireGuard or Tailscale for remote access
</Card>
<Card title="Reverse Proxy" icon="server">
Run behind nginx with basic auth
</Card>
<Card title="Firewall" icon="fire">
Restrict access to trusted IPs only
</Card>
<Card title="Local Only" icon="laptop">
Don't expose to public internet
</Card>
</CardGroup>
## Coming Soon
- API key management
- User accounts
- Rate limiting
- Access control
docs/api/generation.mdx
+119
View File
@@ -0,0 +1,119 @@
---
title: "Generation API"
description: "Generate speech from text"
---
## Generate Speech
```http
POST /generate
```
**Request:**
```json
{
"text": "Hello world",
"profile_id": "abc123",
"language": "en"
}
```
**Response:**
```json
{
"id": "gen123",
"text": "Hello world",
"profile_id": "abc123",
"language": "en",
"audio_url": "/audio/gen123.wav",
"duration": 2.3,
"created_at": "2024-01-29T12:00:00Z"
}
```
## List History
```http
GET /history
```
**Query Parameters:**
- `profile_id` (optional) - Filter by voice profile
- `limit` (optional) - Number of results (default: 50)
- `offset` (optional) - Pagination offset
**Response:**
```json
{
"generations": [
{
"id": "gen123",
"text": "Hello world",
"profile_id": "abc123",
"duration": 2.3,
"created_at": "2024-01-29T12:00:00Z"
}
],
"total": 100
}
```
## Get Generation
```http
GET /history/{id}
```
**Response:**
```json
{
"id": "gen123",
"text": "Hello world",
"profile_id": "abc123",
"language": "en",
"audio_url": "/audio/gen123.wav",
"duration": 2.3,
"created_at": "2024-01-29T12:00:00Z"
}
```
## Delete Generation
```http
DELETE /history/{id}
```
**Response:**
```json
{
"success": true
}
```
## TypeScript Example
```typescript
import { VoiceboxClient } from '@/lib/api'
const client = new VoiceboxClient({
baseUrl: 'http://localhost:17493'
})
// Generate speech
const generation = await client.generate({
text: 'Hello world',
profile_id: 'abc123',
language: 'en'
})
// Get audio URL
const audioUrl = generation.audio_url
// List history
const history = await client.listHistory({
profile_id: 'abc123',
limit: 20
})
```
For full API documentation, visit `http://localhost:17493/docs` when the server is running.
docs/api/overview.mdx
+219
View File
@@ -0,0 +1,219 @@
---
title: "API Overview"
description: "Integrate voice synthesis into your applications with the Voicebox REST API"
---
## Introduction
Voicebox exposes a full REST API that allows you to integrate voice synthesis into your own applications. The API runs on `http://localhost:17493` by default.
<Card title="Interactive API Docs" icon="book" href="http://localhost:17493/docs">
When Voicebox is running, visit the auto-generated API documentation at `http://localhost:17493/docs`
</Card>
## Base URL
```
http://localhost:17493
```
For remote deployments, replace `localhost` with your server's IP or hostname.
## Authentication
<Note>
Currently, the API does not require authentication for local development. Authentication will be added in a future release for production deployments.
</Note>
## Quick Example
Here's a simple example of generating speech:
```bash
# Generate speech
curl -X POST http://localhost:17493/generate \
-H "Content-Type: application/json" \
-d '{
"text": "Hello world",
"profile_id": "abc123",
"language": "en"
}'
```
## API Endpoints
The Voicebox API is organized into several categories:
<CardGroup cols={2}>
<Card title="Voice Profiles" icon="user" href="/api/voice-profiles">
Create, list, update, and delete voice profiles
</Card>
<Card title="Generation" icon="waveform" href="/api/generation">
Generate speech from text using voice profiles
</Card>
<Card title="Recordings" icon="microphone" href="/api/recordings">
Record and transcribe audio
</Card>
<Card title="Stories" icon="film">
Create and manage multi-voice stories (coming soon)
</Card>
</CardGroup>
## Core Endpoints
### Voice Profiles
```http
GET /profiles # List all profiles
POST /profiles # Create a new profile
GET /profiles/{id} # Get profile details
PUT /profiles/{id} # Update a profile
DELETE /profiles/{id} # Delete a profile
POST /profiles/{id}/samples # Add voice sample
```
### Generation
```http
POST /generate # Generate speech
GET /history # List generation history
GET /history/{id} # Get generation details
DELETE /history/{id} # Delete from history
```
### Recordings
```http
POST /recordings # Start recording
POST /recordings/stop # Stop recording
POST /transcribe # Transcribe audio
```
## Response Format
All API responses follow a consistent JSON format:
```json
{
"success": true,
"data": {
// Response data
},
"error": null
}
```
Error responses:
```json
{
"success": false,
"data": null,
"error": {
"message": "Error description",
"code": "ERROR_CODE"
}
}
```
## Data Models
### Voice Profile
```json
{
"id": "abc123",
"name": "John Smith",
"language": "en",
"description": "Professional narrator voice",
"created_at": "2024-01-29T12:00:00Z",
"samples": [
{
"id": "sample123",
"audio_path": "/path/to/sample.wav",
"duration": 15.5
}
]
}
```
### Generation
```json
{
"id": "gen123",
"text": "Hello world",
"profile_id": "abc123",
"language": "en",
"audio_path": "/path/to/output.wav",
"duration": 2.3,
"created_at": "2024-01-29T12:00:00Z"
}
```
## TypeScript Client
Voicebox provides an auto-generated TypeScript client with full type safety:
```typescript
import { VoiceboxClient } from '@/lib/api'
const client = new VoiceboxClient({
baseUrl: 'http://localhost:17493'
})
// Create a profile
const profile = await client.createProfile({
name: 'John Smith',
language: 'en'
})
// Generate speech
const generation = await client.generate({
text: 'Hello world',
profile_id: profile.id,
language: 'en'
})
```
The client is automatically generated from the OpenAPI schema. See [Development Setup](/development/setup#generate-openapi-client) for details.
## Rate Limiting
<Info>
Currently, there are no rate limits for local usage. Rate limiting will be added in a future release for production deployments.
</Info>
## WebSocket Support
<Note>
Real-time streaming generation via WebSockets is planned for a future release.
</Note>
## Use Cases
<CardGroup cols={2}>
<Card title="Game Development" icon="gamepad">
Generate dynamic dialogue for NPCs and characters
</Card>
<Card title="Content Creation" icon="video">
Automate voiceovers for videos and podcasts
</Card>
<Card title="Accessibility" icon="universal-access">
Build text-to-speech tools for visually impaired users
</Card>
<Card title="Voice Assistants" icon="robot">
Create custom voice interfaces
</Card>
</CardGroup>
## Next Steps
<CardGroup cols={2}>
<Card title="Voice Profiles API" icon="user" href="/api/voice-profiles">
Learn how to manage voice profiles
</Card>
<Card title="Generation API" icon="waveform" href="/api/generation">
Generate speech from text
</Card>
</CardGroup>
docs/api/recordings.mdx
+95
View File
@@ -0,0 +1,95 @@
---
title: "Recordings API"
description: "Record and transcribe audio"
---
## Start Recording
```http
POST /recordings/start
```
**Request:**
```json
{
"source": "microphone"
}
```
**Response:**
```json
{
"recording_id": "rec123",
"status": "recording"
}
```
## Stop Recording
```http
POST /recordings/stop
```
**Request:**
```json
{
"recording_id": "rec123"
}
```
**Response:**
```json
{
"recording_id": "rec123",
"audio_url": "/audio/rec123.wav",
"duration": 15.5
}
```
## Transcribe Audio
```http
POST /transcribe
```
**Request:** (multipart/form-data)
```
audio: <file>
language: "en" (optional)
```
**Response:**
```json
{
"text": "Transcribed speech text here",
"language": "en",
"duration": 15.5,
"confidence": 0.95
}
```
## TypeScript Example
```typescript
import { VoiceboxClient } from '@/lib/api'
const client = new VoiceboxClient({
baseUrl: 'http://localhost:17493'
})
// Start recording
const recording = await client.startRecording({
source: 'microphone'
})
// ... record audio ...
// Stop recording
const result = await client.stopRecording(recording.id)
// Transcribe
const transcription = await client.transcribe(audioFile, 'en')
console.log(transcription.text)
```
For full API documentation, visit `http://localhost:17493/docs` when the server is running.
docs/api/voice-profiles.mdx
+149
View File
@@ -0,0 +1,149 @@
---
title: "Voice Profiles API"
description: "Manage voice profiles programmatically"
---
## Endpoints
### List Profiles
```http
GET /profiles
```
**Response:**
```json
{
"profiles": [
{
"id": "abc123",
"name": "John Smith",
"language": "en",
"description": "Professional narrator",
"created_at": "2024-01-29T12:00:00Z",
"sample_count": 2
}
]
}
```
### Get Profile
```http
GET /profiles/{id}
```
**Response:**
```json
{
"id": "abc123",
"name": "John Smith",
"language": "en",
"description": "Professional narrator",
"created_at": "2024-01-29T12:00:00Z",
"samples": [
{
"id": "sample123",
"duration": 15.5,
"created_at": "2024-01-29T12:00:00Z"
}
]
}
```
### Create Profile
```http
POST /profiles
```
**Request:**
```json
{
"name": "John Smith",
"language": "en",
"description": "Professional narrator"
}
```
**Response:**
```json
{
"id": "abc123",
"name": "John Smith",
"language": "en",
"description": "Professional narrator",
"created_at": "2024-01-29T12:00:00Z"
}
```
### Update Profile
```http
PUT /profiles/{id}
```
**Request:**
```json
{
"name": "Updated Name",
"description": "Updated description"
}
```
### Delete Profile
```http
DELETE /profiles/{id}
```
**Response:**
```json
{
"success": true
}
```
### Add Voice Sample
```http
POST /profiles/{id}/samples
```
**Request:** (multipart/form-data)
```
audio: <file>
```
**Response:**
```json
{
"sample_id": "sample123",
"duration": 15.5
}
```
## TypeScript Example
```typescript
import { VoiceboxClient } from '@/lib/api'
const client = new VoiceboxClient({
baseUrl: 'http://localhost:17493'
})
// Create profile
const profile = await client.createProfile({
name: 'John Smith',
language: 'en',
description: 'Professional narrator'
})
// Add sample
await client.addSample(profile.id, audioFile)
// List all profiles
const profiles = await client.listProfiles()
```
For full API documentation, visit `http://localhost:17493/docs` when the server is running.
docs/bun.lock
+1831
View File
File diff suppressed because it is too large Load Diff
docs/custom.css
+15
View File
@@ -0,0 +1,15 @@
/* Anchor hover styles */
.nav-anchor:hover {
@apply text-[#BF9E40];
}
/* Icon wrapper on hover */
.nav-anchor:hover div {
background: #BF9E40 !important;
filter: brightness(1) !important;
}
/* Icon SVG on hover */
.nav-anchor:hover svg {
@apply bg-white !important;
}
docs/developer/architecture.mdx
+202
View File
@@ -0,0 +1,202 @@
---
title: "Architecture"
description: "Understanding Voicebox's technical architecture"
---
## System Overview
Voicebox uses a client-server architecture with a React frontend and Python backend. The desktop app is built with Tauri and contains two main layers:
**Frontend Layer:** A React application that handles the UI components, state management with Zustand, and data fetching with React Query (TanStack Query).
**Backend Layer:** A Python FastAPI server that provides the REST API, runs the TTS engine (Qwen3-TTS), manages the SQLite database, and handles audio processing.
These two layers communicate via HTTP, with the frontend making API requests to the backend.
## Frontend Architecture
### Tech Stack
- **Framework**: React 18 with TypeScript
- **State Management**: Zustand stores
- **Data Fetching**: React Query (TanStack Query)
- **Styling**: Tailwind CSS
- **Audio**: WaveSurfer.js
- **Desktop**: Tauri (Rust)
### Component Structure
```
app/src/
├── components/ # React components
│ ├── profiles/ # Voice profile UI
│ ├── generation/ # Speech generation UI
│ ├── stories/ # Timeline editor
│ └── shared/ # Reusable components
├── lib/ # Utilities
│ ├── api/ # Generated API client
│ └── utils/ # Helper functions
├── hooks/ # React hooks
└── stores/ # Zustand state stores
```
### State Management
```typescript
// Example: Profile store
const useProfileStore = create((set) => ({
profiles: [],
selectedProfile: null,
setProfiles: (profiles) => set({ profiles }),
selectProfile: (id) => set({ selectedProfile: id })
}))
```
## Backend Architecture
### Tech Stack
- **Framework**: FastAPI (Python 3.11+)
- **TTS Model**: Qwen3-TTS
- **Transcription**: Whisper
- **Database**: SQLite
- **Audio**: librosa, soundfile
### API Structure
```python
# main.py - API routes
@app.post("/generate")
async def generate_speech(request: GenerateRequest):
# 1. Validate request
# 2. Load voice profile
# 3. Generate audio with TTS
# 4. Save to database
# 5. Return response
```
### Data Model
The database uses three main tables:
**Profile Table:** Stores voice profiles with fields for id, name, and language.
**Sample Table:** Stores audio samples linked to profiles via profile_id, with fields for audio_path and duration.
**Generation Table:** Stores generated audio with fields for id, profile_id, text, and audio_path.
## Desktop App (Tauri)
### Rust Backend
```rust
// Sidecar process management
// File system access
// Native integrations
```
### Responsibilities
- Launch Python backend as sidecar process
- Native file dialogs
- System tray integration
- Auto-updates
- OS-specific features
## Build Process
### Development
```bash
# Frontend (Vite dev server)
cd app && bun run dev
# Backend (manual start)
cd backend && uvicorn main:app --reload
# Desktop app (connects to manual backend)
bun run dev
```
### Production
```bash
# 1. Build server binary (PyInstaller)
./scripts/build-server.sh
# 2. Build Tauri app (includes server)
cd tauri && bun run tauri build
```
## Data Flow
### Generation Flow
When a user generates speech, the data flows through the following stages:
1. **User Input** - User enters text in a React component
2. **State Update** - Text is stored in Zustand state
3. **API Request** - React Query mutation triggers an API call via fetch
4. **Backend Processing** - FastAPI endpoint receives the request
5. **TTS Generation** - Qwen3-TTS model generates the audio
6. **Storage** - Audio file is saved to disk and a database record is created
7. **Response** - Backend returns the audio URL
8. **Cache Update** - React Query updates its cache with the response
9. **UI Update** - Component re-renders with new data
10. **Playback** - User can play the generated audio
## Performance Considerations
### Frontend
- **Code splitting** - Lazy load routes
- **Memoization** - React.memo for heavy components
- **Virtual scrolling** - For large lists
- **Debouncing** - Search and input handling
### Backend
- **Async operations** - All I/O is async
- **Model caching** - Keep TTS model in memory
- **Voice prompt caching** - Reuse embeddings
- **Connection pooling** - Database connections
## Security
### Current
- Local-only by default
- No authentication (localhost trust)
- File system sandboxing via Tauri
### Planned
- API key authentication
- User accounts
- Rate limiting
- HTTPS support
## Deployment Modes
### Local Mode
- Backend runs as sidecar
- All data stays on device
- No network required
### Remote Mode
- Backend on separate machine
- Frontend connects via HTTP
- Shared infrastructure possible
## Next Steps
<CardGroup cols={2}>
<Card title="Development Setup" icon="code" href="/development/setup">
Set up your dev environment
</Card>
<Card title="Contributing" icon="code-pull-request" href="/development/contributing">
Contribute to Voicebox
</Card>
</CardGroup>
docs/developer/audio-channels.mdx
+310
View File
@@ -0,0 +1,310 @@
---
title: "Audio Channels"
description: "How audio output routing works in Voicebox"
---
## Overview
Audio channels allow routing voice output to different audio devices. This is useful for multi-output setups where different voices should play through different speakers or applications.
## Architecture
**Channel:** A named audio bus that can be assigned to output devices.
**Device Mapping:** Links channels to OS audio device identifiers.
**Profile Mapping:** Links voice profiles to channels (many-to-many).
## Data Model
### AudioChannel Table
```python
class AudioChannel(Base):
__tablename__ = "audio_channels"
id = Column(String, primary_key=True)
name = Column(String, nullable=False)
is_default = Column(Boolean, default=False)
created_at = Column(DateTime)
```
### ChannelDeviceMapping Table
```python
class ChannelDeviceMapping(Base):
__tablename__ = "channel_device_mappings"
id = Column(String, primary_key=True)
channel_id = Column(String, ForeignKey("audio_channels.id"))
device_id = Column(String) # OS device identifier
```
### ProfileChannelMapping Table
```python
class ProfileChannelMapping(Base):
__tablename__ = "profile_channel_mappings"
profile_id = Column(String, ForeignKey("profiles.id"), primary_key=True)
channel_id = Column(String, ForeignKey("audio_channels.id"), primary_key=True)
```
## Default Channel
A default channel is created on database initialization:
```python
def init_db():
# Create default channel if it doesn't exist
default_channel = db.query(AudioChannel).filter(
AudioChannel.is_default == True
).first()
if not default_channel:
default_channel = AudioChannel(
id=str(uuid.uuid4()),
name="Default",
is_default=True
)
db.add(default_channel)
# Assign all existing profiles to default channel
profiles = db.query(VoiceProfile).all()
for profile in profiles:
mapping = ProfileChannelMapping(
profile_id=profile.id,
channel_id=default_channel.id
)
db.add(mapping)
```
## Core Operations
### Creating a Channel
```python
async def create_channel(
data: AudioChannelCreate,
db: Session,
) -> AudioChannelResponse:
# Check name uniqueness
existing = db.query(DBAudioChannel).filter_by(name=data.name).first()
if existing:
raise ValueError(f"Channel with name '{data.name}' already exists")
# Create channel
channel = DBAudioChannel(
id=str(uuid.uuid4()),
name=data.name,
is_default=False,
)
db.add(channel)
# Add device mappings
for device_id in data.device_ids:
mapping = DBChannelDeviceMapping(
id=str(uuid.uuid4()),
channel_id=channel.id,
device_id=device_id,
)
db.add(mapping)
db.commit()
```
### Updating a Channel
```python
async def update_channel(
channel_id: str,
data: AudioChannelUpdate,
db: Session,
) -> AudioChannelResponse:
channel = db.query(DBAudioChannel).filter_by(id=channel_id).first()
# Cannot modify default channel
if channel.is_default:
raise ValueError("Cannot modify the default channel")
# Update name
if data.name is not None:
channel.name = data.name
# Update device mappings
if data.device_ids is not None:
# Delete existing
db.query(DBChannelDeviceMapping).filter_by(channel_id=channel_id).delete()
# Add new
for device_id in data.device_ids:
mapping = DBChannelDeviceMapping(
channel_id=channel.id,
device_id=device_id,
)
db.add(mapping)
db.commit()
```
### Deleting a Channel
```python
async def delete_channel(channel_id: str, db: Session) -> bool:
channel = db.query(DBAudioChannel).filter_by(id=channel_id).first()
# Cannot delete default channel
if channel.is_default:
raise ValueError("Cannot delete the default channel")
# Delete device mappings
db.query(DBChannelDeviceMapping).filter_by(channel_id=channel_id).delete()
# Delete profile-channel mappings
db.query(DBProfileChannelMapping).filter_by(channel_id=channel_id).delete()
# Delete channel
db.delete(channel)
db.commit()
```
## Voice Assignment
### Assigning Voices to Channel
```python
async def set_channel_voices(
channel_id: str,
data: ChannelVoiceAssignment,
db: Session,
) -> None:
# Verify channel exists
channel = db.query(DBAudioChannel).filter_by(id=channel_id).first()
if not channel:
raise ValueError(f"Channel {channel_id} not found")
# Verify all profiles exist
for profile_id in data.profile_ids:
profile = db.query(DBVoiceProfile).filter_by(id=profile_id).first()
if not profile:
raise ValueError(f"Profile {profile_id} not found")
# Delete existing mappings
db.query(DBProfileChannelMapping).filter_by(channel_id=channel_id).delete()
# Add new mappings
for profile_id in data.profile_ids:
mapping = DBProfileChannelMapping(
profile_id=profile_id,
channel_id=channel_id,
)
db.add(mapping)
db.commit()
```
### Assigning Channels to Voice
```python
async def set_profile_channels(
profile_id: str,
data: ProfileChannelAssignment,
db: Session,
) -> None:
# Verify profile exists
profile = db.query(DBVoiceProfile).filter_by(id=profile_id).first()
if not profile:
raise ValueError(f"Profile {profile_id} not found")
# Delete existing mappings
db.query(DBProfileChannelMapping).filter_by(profile_id=profile_id).delete()
# Add new mappings
for channel_id in data.channel_ids:
mapping = DBProfileChannelMapping(
profile_id=profile_id,
channel_id=channel_id,
)
db.add(mapping)
db.commit()
```
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| GET | `/channels` | List all channels |
| POST | `/channels` | Create a channel |
| GET | `/channels/{id}` | Get channel by ID |
| PUT | `/channels/{id}` | Update channel |
| DELETE | `/channels/{id}` | Delete channel |
| GET | `/channels/{id}/voices` | Get assigned voices |
| PUT | `/channels/{id}/voices` | Set assigned voices |
| GET | `/profiles/{id}/channels` | Get profile's channels |
| PUT | `/profiles/{id}/channels` | Set profile's channels |
## Request/Response Schemas
### AudioChannelCreate
```json
{
"name": "Speakers",
"device_ids": ["device_uuid_1", "device_uuid_2"]
}
```
### AudioChannelResponse
```json
{
"id": "channel_uuid",
"name": "Speakers",
"is_default": false,
"device_ids": ["device_uuid_1", "device_uuid_2"],
"created_at": "2024-01-15T10:30:00Z"
}
```
### ChannelVoiceAssignment
```json
{
"profile_ids": ["profile_1", "profile_2"]
}
```
## Use Cases
### Multi-Output Setup
**Scenario:** Stream with different voice characters
1. Create "Stream" channel → OBS virtual audio
2. Create "Monitor" channel → Headphones
3. Assign "Narrator" profile → Both channels
4. Assign "Character 1" profile → Stream only
### Virtual Audio Cables
Common device IDs for virtual audio:
- VB-Audio Virtual Cable
- BlackHole (macOS)
- Soundflower (macOS)
## Frontend Integration
The frontend needs to:
1. **Enumerate devices** using Web Audio API or Tauri
2. **Display channel list** with device assignments
3. **Allow profile assignment** via drag/drop or dropdown
4. **Route playback** to correct device based on profile's channel
## Limitations
- Device IDs are OS-specific
- Hot-plugging may invalidate device IDs
- Default channel cannot be modified/deleted
- Frontend handles actual audio routing (backend just stores config)
docs/developer/autoupdater.mdx
+84
View File
@@ -0,0 +1,84 @@
---
title: "Auto-Updater"
description: "Configure and use the Tauri auto-updater"
---
## Overview
Voicebox uses Tauri's built-in auto-updater to deliver updates to users automatically.
## Quick Reference
For detailed setup instructions, see the existing documentation:
- [AUTOUPDATER_QUICKSTART.md](https://github.com/jamiepine/voicebox/blob/main/docs/AUTOUPDATER_QUICKSTART.md)
- [AUTOUPDATER.md](https://github.com/jamiepine/voicebox/blob/main/docs/AUTOUPDATER.md)
## How It Works
The auto-updater follows a secure update process:
1. **Check for Updates** - The Voicebox app periodically checks GitHub Releases for new versions
2. **Download Update** - If a new version is found, the update package is downloaded
3. **Verify Signature** - The downloaded package is cryptographically verified using the public key
4. **Install** - After verification, the update is installed
5. **Restart** - The app restarts with the new version
## Configuration
Updates are configured in `tauri/src-tauri/tauri.conf.json`:
```json
{
"updater": {
"active": true,
"endpoints": [
"https://github.com/jamiepine/voicebox/releases/latest/download/latest.json"
],
"dialog": true,
"pubkey": "YOUR_PUBLIC_KEY"
}
}
```
## Generating Keys
```bash
# Generate signing keys
bun run generate:keys
# Keys saved to ~/.tauri/voicebox.key
```
<Warning>
Keep your private key secure! Never commit it to the repository.
</Warning>
## Release Process
1. **Bump version** using bumpversion
2. **Push tag** to trigger CI/CD
3. **GitHub Actions** builds and signs releases
4. **Users** receive update notification
## User Experience
When an update is available:
1. User sees a notification dialog
2. User clicks "Update"
3. Update downloads in background
4. App restarts with new version
## For Developers
See the full documentation files for:
- Setting up signing keys
- Configuring GitHub releases
- Testing updates locally
- Troubleshooting update failures
<Card title="View Full Docs" href="https://github.com/jamiepine/voicebox/tree/main/docs">
Access AUTOUPDATER.md and AUTOUPDATER_QUICKSTART.md in the repository
</Card>
docs/developer/building.mdx
+263
View File
@@ -0,0 +1,263 @@
---
title: "Building"
description: "Build Voicebox for production"
---
## Overview
Voicebox uses a multi-step build process to create platform-specific installers.
## Quick Build
```bash
# Build for your current platform
make build
# Or manually
cd tauri && bun run tauri build
```
## Build Steps
### 1. Build Server Binary
The Python backend must be compiled into a standalone executable first:
```bash
./scripts/build-server.sh
```
This uses PyInstaller to create a binary in `tauri/src-tauri/binaries/`.
**Platform-specific binaries:**
- macOS: `voicebox-server-aarch64-apple-darwin` or `voicebox-server-x86_64-apple-darwin`
- Windows: `voicebox-server-x86_64-pc-windows-msvc.exe`
- Linux: `voicebox-server-x86_64-unknown-linux-gnu`
<Note>
The build script automatically detects your platform and creates the appropriate binary.
</Note>
### 2. Build Tauri App
```bash
cd tauri
bun run tauri build
```
This will:
1. Build the React frontend (Vite)
2. Compile the Rust backend
3. Bundle the server binary as a sidecar
4. Create platform-specific installers
### 3. Output
Installers are created in `tauri/src-tauri/target/release/bundle/`:
**macOS:**
- `dmg/` - Disk image installer
- `macos/` - App bundle
**Windows:**
- `msi/` - MSI installer
- `nsis/` - NSIS installer
**Linux:**
- `deb/` - Debian package
- `appimage/` - AppImage
## Advanced Options
### Building for Specific Platform
```bash
# Build for macOS (Apple Silicon)
bun run tauri build -- --target aarch64-apple-darwin
# Build for macOS (Intel)
bun run tauri build -- --target x86_64-apple-darwin
# Build for Windows
bun run tauri build -- --target x86_64-pc-windows-msvc
# Build for Linux
bun run tauri build -- --target x86_64-unknown-linux-gnu
```
### Using Local Qwen3-TTS
If you're developing Qwen3-TTS locally:
```bash
export QWEN_TTS_PATH=~/path/to/Qwen3-TTS
./scripts/build-server.sh
```
This makes PyInstaller use your local version instead of the pip package.
### Debug Build
```bash
cd tauri
bun run tauri build --debug
```
Creates a debug build with symbols and logging.
## Build Configuration
### Tauri Config
Edit `tauri/src-tauri/tauri.conf.json`:
```json
{
"bundle": {
"identifier": "com.voicebox.app",
"icon": [
"icons/32x32.png",
"icons/128x128.png",
"icons/icon.icns",
"icons/icon.ico"
]
}
}
```
### Sidecar Configuration
The Python server is bundled as a sidecar:
```json
{
"tauri": {
"bundle": {
"externalBin": [
"binaries/voicebox-server"
]
}
}
}
```
## Code Signing
### macOS
To sign the app for distribution:
```bash
# Set signing identity
export APPLE_SIGNING_IDENTITY="Developer ID Application: Your Name"
# Build with signing
bun run tauri build
```
For notarization:
```bash
# Set credentials
export APPLE_ID="[email protected]"
export APPLE_PASSWORD="app-specific-password"
# Build and notarize
bun run tauri build
```
### Windows
For Windows code signing:
```bash
# Set certificate
export WINDOWS_CERTIFICATE_PATH="/path/to/cert.pfx"
export WINDOWS_CERTIFICATE_PASSWORD="password"
# Build with signing
bun run tauri build
```
## Release Process
The full release process is automated:
```bash
# 1. Bump version
bumpversion patch # or minor/major
# 2. Build all platforms (CI/CD handles this)
git push --tags
# 3. GitHub Actions creates releases
```
See [CONTRIBUTING.md](/development/contributing) for the full release workflow.
## Troubleshooting
<AccordionGroup>
<Accordion title="Server Binary Build Fails">
**Common issues:**
- Missing Python dependencies: `pip install -r requirements.txt`
- PyInstaller not found: `pip install pyinstaller`
- Qwen3-TTS not installed: `pip install git+https://github.com/QwenLM/Qwen3-TTS.git`
**Solution:**
```bash
cd backend
source venv/bin/activate
pip install -r requirements.txt
pip install pyinstaller
```
</Accordion>
<Accordion title="Tauri Build Fails">
**Common issues:**
- Rust not installed: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
- Server binary missing: Run `./scripts/build-server.sh` first
- Node modules outdated: `bun install`
**Solution:**
```bash
# Clean and rebuild
cd tauri/src-tauri
cargo clean
cd ../..
./scripts/build-server.sh
bun run tauri build
```
</Accordion>
<Accordion title="App Won't Launch After Build">
**Check:**
- Server binary has execute permissions
- All dependencies are bundled
- Check logs in the app's data directory
**macOS:**
```bash
tail -f ~/Library/Application\ Support/com.voicebox.app/logs/server.log
```
**Windows:**
```bash
type %APPDATA%\com.voicebox.app\logs\server.log
```
</Accordion>
</AccordionGroup>
## CI/CD
GitHub Actions automatically builds releases when tags are pushed:
```yaml
# .github/workflows/release.yml
on:
push:
tags:
- 'v*'
```
See the [repository](https://github.com/jamiepine/voicebox) for the full CI/CD configuration.
docs/developer/contributing.mdx
+326
View File
@@ -0,0 +1,326 @@
---
title: "Contributing"
description: "How to contribute to Voicebox"
---
Thank you for your interest in contributing to Voicebox! This guide will help you get started.
## Code of Conduct
- Be respectful and inclusive
- Welcome newcomers and help them learn
- Focus on constructive feedback
- Respect different viewpoints and experiences
## Getting Started
Before you start contributing, make sure you have:
1. **Read the documentation** to understand how Voicebox works
2. **Set up your development environment** - see [Development Setup](/development/setup)
3. **Explored the codebase** to understand the project structure
4. **Checked existing issues** to see if someone else is working on something similar
## Ways to Contribute
<CardGroup cols={2}>
<Card title="Report Bugs" icon="bug">
Found a bug? Open an issue with reproduction steps
</Card>
<Card title="Request Features" icon="lightbulb">
Have an idea? Start a discussion or open an issue
</Card>
<Card title="Improve Docs" icon="book">
Fix typos, add examples, or clarify instructions
</Card>
<Card title="Write Code" icon="code">
Fix bugs, add features, or optimize performance
</Card>
</CardGroup>
## Development Workflow
### 1. Fork & Clone
```bash
# Fork the repository on GitHub
# Then clone your fork
git clone https://github.com/YOUR_USERNAME/voicebox.git
cd voicebox
```
### 2. Create a Branch
Use descriptive branch names:
```bash
# For features
git checkout -b feature/voice-effects
# For bug fixes
git checkout -b fix/audio-playback-issue
# For documentation
git checkout -b docs/api-examples
```
### 3. Make Your Changes
Follow these guidelines:
<AccordionGroup>
<Accordion title="Code Style">
**TypeScript/React:**
- Use TypeScript strict mode
- Prefer functional components with hooks
- Use named exports
- Format with Biome (runs automatically)
**Python:**
- Follow PEP 8
- Use type hints
- Use async/await for I/O
- Document functions with docstrings
**Rust:**
- Follow Rust conventions
- Use meaningful names
- Handle errors explicitly
- Run `rustfmt`
</Accordion>
<Accordion title="Commit Messages">
Write clear, descriptive commit messages:
```bash
# Good
git commit -m "Add voice profile export feature"
git commit -m "Fix audio playback stopping after 30 seconds"
# Avoid
git commit -m "Update code"
git commit -m "Fix bug"
```
Format:
- Use imperative mood ("Add feature" not "Added feature")
- Keep first line under 50 characters
- Add detailed description if needed
</Accordion>
<Accordion title="Testing">
- Test your changes manually in the app
- Ensure backend API endpoints work
- Check for TypeScript/Python errors
- Verify UI components render correctly
- Add automated tests when possible
</Accordion>
</AccordionGroup>
### 4. Push & Create PR
```bash
# Push your branch
git push origin feature/your-feature-name
# Then create a pull request on GitHub
```
## Pull Request Guidelines
When creating a pull request:
<Steps>
<Step title="Use a Clear Title">
Examples:
- "Add voice profile export functionality"
- "Fix audio playback stopping after 30 seconds"
- "Improve generation speed with caching"
</Step>
<Step title="Provide Description">
Include:
- What changes you made
- Why you made them
- How to test them
- Screenshots (for UI changes)
- Reference related issues
</Step>
<Step title="Update Documentation">
- Update relevant docs if behavior changes
- Add API documentation for new endpoints
- Update README if needed
</Step>
<Step title="Check the Checklist">
- [ ] Code follows style guidelines
- [ ] Documentation updated
- [ ] Changes tested
- [ ] No breaking changes (or documented)
- [ ] CHANGELOG.md updated
</Step>
</Steps>
## Project Structure
Understanding the codebase:
```
voicebox/
├── app/ # Shared React frontend
│ ├── src/
│ │ ├── components/ # UI components
│ │ ├── lib/ # Utilities and API client
│ │ ├── hooks/ # React hooks
│ │ └── stores/ # Zustand state stores
├── backend/ # Python FastAPI server
│ ├── main.py # API routes
│ ├── tts.py # Voice synthesis logic
│ ├── database.py # SQLite operations
│ └── models.py # Pydantic models
├── tauri/ # Desktop app wrapper
│ └── src-tauri/ # Rust backend
├── web/ # Web deployment
├── landing/ # Marketing website
└── scripts/ # Build & release scripts
```
## Areas for Contribution
### Bug Fixes
- Check [existing issues](https://github.com/jamiepine/voicebox/issues) for bugs
- Test your fix thoroughly
- Add regression tests if possible
### New Features
- Check the [roadmap](https://github.com/jamiepine/voicebox#roadmap) for planned features
- Discuss major features in an issue first
- Keep features focused and well-scoped
### Documentation
- Improve clarity and fix typos
- Add code examples
- Create tutorials or guides
- Document API endpoints
### UI/UX Improvements
- Improve accessibility
- Enhance visual design
- Optimize performance
- Add animations/transitions
### Infrastructure
- Improve build process
- Add CI/CD improvements
- Optimize bundle size
- Add testing infrastructure
## API Development
When adding new API endpoints:
<Steps>
<Step title="Add Route">
In `backend/main.py`:
```python
@app.post("/api/new-endpoint")
async def new_endpoint(data: RequestModel) -> ResponseModel:
"""Endpoint description."""
# Implementation
return response
```
</Step>
<Step title="Create Models">
In `backend/models.py`:
```python
class RequestModel(BaseModel):
field: str
class ResponseModel(BaseModel):
result: str
```
</Step>
<Step title="Regenerate Client">
```bash
bun run generate:api
```
This updates the TypeScript client with type-safe bindings.
</Step>
<Step title="Update Docs">
Add documentation in `/docs/api/`
</Step>
</Steps>
## Testing
Currently testing is primarily manual. When adding tests:
**Backend:**
```bash
cd backend
pytest
```
**Frontend:**
```bash
bun run test
```
**E2E (future):**
```bash
bun run test:e2e
```
## Release Process
Releases are managed by maintainers using `bumpversion`:
```bash
# Bump version (patch, minor, or major)
bumpversion patch
# Push with tags
git push && git push --tags
```
GitHub Actions automatically builds and publishes releases when tags are pushed.
## Community
- **GitHub Issues:** Bug reports and feature requests
- **GitHub Discussions:** General questions and ideas
- **Discord:** Real-time chat (coming soon)
## Recognition
Contributors are recognized in:
- [CHANGELOG.md](https://github.com/jamiepine/voicebox/blob/main/CHANGELOG.md)
- GitHub contributor list
- Release notes
## License
By contributing, you agree that your contributions will be licensed under the MIT License.
## Questions?
If you have questions:
1. Check the [documentation](/overview/introduction)
2. Search [existing issues](https://github.com/jamiepine/voicebox/issues)
3. Open a new issue or discussion
4. See [CONTRIBUTING.md](https://github.com/jamiepine/voicebox/blob/main/CONTRIBUTING.md) in the repo
Thank you for contributing to Voicebox! 🎉
docs/developer/history.mdx
+260
View File
@@ -0,0 +1,260 @@
---
title: "Generation History"
description: "How generation history tracking works in Voicebox"
---
## Overview
The history module tracks all generated audio, providing a searchable record of past generations. Each generation stores the text, settings, and a reference to the audio file.
## Data Model
### Generation Table
```python
class Generation(Base):
__tablename__ = "generations"
id = Column(String, primary_key=True)
profile_id = Column(String, ForeignKey("profiles.id"))
text = Column(Text, nullable=False)
language = Column(String, default="en")
audio_path = Column(String, nullable=False)
duration = Column(Float, nullable=False)
seed = Column(Integer)
instruct = Column(Text)
created_at = Column(DateTime)
```
## File Storage
Generated audio is stored in:
```
data/
└── generations/
└── {generation_id}.wav
```
## Core Functions
### Creating a Generation Record
After TTS generates audio, a history entry is created:
```python
async def create_generation(
profile_id: str,
text: str,
language: str,
audio_path: str,
duration: float,
seed: Optional[int],
db: Session,
instruct: Optional[str] = None,
) -> GenerationResponse:
db_generation = DBGeneration(
id=str(uuid.uuid4()),
profile_id=profile_id,
text=text,
language=language,
audio_path=audio_path,
duration=duration,
seed=seed,
instruct=instruct,
created_at=datetime.utcnow(),
)
db.add(db_generation)
db.commit()
return GenerationResponse.model_validate(db_generation)
```
### Listing Generations
Supports filtering and pagination:
```python
async def list_generations(
query: HistoryQuery,
db: Session,
) -> HistoryListResponse:
# Build query with profile name join
q = db.query(
DBGeneration,
DBVoiceProfile.name.label('profile_name')
).join(
DBVoiceProfile,
DBGeneration.profile_id == DBVoiceProfile.id
)
# Apply filters
if query.profile_id:
q = q.filter(DBGeneration.profile_id == query.profile_id)
if query.search:
q = q.filter(DBGeneration.text.like(f"%{query.search}%"))
# Order and paginate
total = q.count()
q = q.order_by(DBGeneration.created_at.desc())
q = q.offset(query.offset).limit(query.limit)
return HistoryListResponse(items=results, total=total)
```
### Getting Statistics
Aggregate statistics for the dashboard:
```python
async def get_generation_stats(db: Session) -> dict:
total = db.query(func.count(DBGeneration.id)).scalar()
total_duration = db.query(func.sum(DBGeneration.duration)).scalar()
by_profile = db.query(
DBGeneration.profile_id,
func.count(DBGeneration.id).label('count')
).group_by(DBGeneration.profile_id).all()
return {
"total_generations": total,
"total_duration_seconds": total_duration,
"generations_by_profile": {
profile_id: count for profile_id, count in by_profile
},
}
```
## Deletion
Deleting a generation removes both the database record and audio file:
```python
async def delete_generation(generation_id: str, db: Session) -> bool:
generation = db.query(DBGeneration).filter_by(id=generation_id).first()
if not generation:
return False
# Delete audio file
audio_path = Path(generation.audio_path)
if audio_path.exists():
audio_path.unlink()
# Delete database record
db.delete(generation)
db.commit()
return True
```
### Cascade Delete
When deleting a profile, all its generations are also deleted:
```python
async def delete_generations_by_profile(profile_id: str, db: Session) -> int:
generations = db.query(DBGeneration).filter_by(profile_id=profile_id).all()
for generation in generations:
Path(generation.audio_path).unlink(missing_ok=True)
db.delete(generation)
db.commit()
return len(generations)
```
## Export/Import
### Exporting a Generation
Generations can be exported as ZIP archives:
```
generation_export.zip
├── generation.json # Metadata
└── audio.wav # Audio file
```
### Importing a Generation
The import process:
1. Extract ZIP archive
2. Validate metadata and audio
3. Create new generation ID
4. Copy audio to generations directory
5. Create database record
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| GET | `/history` | List generations with filters |
| GET | `/history/stats` | Get aggregate statistics |
| GET | `/history/{id}` | Get generation by ID |
| DELETE | `/history/{id}` | Delete generation |
| GET | `/history/{id}/export` | Export as ZIP |
| GET | `/history/{id}/export-audio` | Export audio only |
| POST | `/history/import` | Import from ZIP |
### Query Parameters
```
GET /history?profile_id=uuid&search=hello&limit=50&offset=0
```
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `profile_id` | string | null | Filter by profile |
| `search` | string | null | Search in text |
| `limit` | int | 50 | Results per page |
| `offset` | int | 0 | Pagination offset |
### Response Schema
```json
{
"items": [
{
"id": "uuid",
"profile_id": "uuid",
"profile_name": "My Voice",
"text": "Hello world",
"language": "en",
"audio_path": "/path/to/audio.wav",
"duration": 1.5,
"seed": 42,
"instruct": null,
"created_at": "2024-01-15T10:30:00Z"
}
],
"total": 150
}
```
## Usage in Stories
Generations can be added to stories for multi-voice narratives. The story system references generations by ID:
```python
class StoryItem(Base):
generation_id = Column(String, ForeignKey("generations.id"))
```
This allows the same generation to be reused across multiple stories without duplicating audio files.
## Storage Considerations
### Disk Usage
Each generation creates a WAV file. For a 10-second clip at 24kHz:
- ~480KB per file (mono, 16-bit)
### Cleanup Strategy
Consider implementing:
- Automatic cleanup of old generations
- Storage quota per profile
- Compression for archival
docs/developer/model-management.mdx
+341
View File
@@ -0,0 +1,341 @@
---
title: "Model Management"
description: "How model downloading, loading, and status tracking works in Voicebox"
---
## Overview
Voicebox manages two types of models:
**TTS Models:** Qwen3-TTS for voice cloning (0.6B and 1.7B variants).
**ASR Models:** Whisper for transcription (tiny through large).
Models are downloaded from HuggingFace Hub on first use and cached locally.
## Available Models
### TTS Models
| Model | HuggingFace ID | Size | VRAM |
|-------|----------------|------|------|
| 0.6B | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ~1.2GB | ~2GB |
| 1.7B | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ~3.4GB | ~6GB |
### Whisper Models
| Model | HuggingFace ID | Size | VRAM |
|-------|----------------|------|------|
| tiny | `openai/whisper-tiny` | ~150MB | ~1GB |
| base | `openai/whisper-base` | ~300MB | ~1GB |
| small | `openai/whisper-small` | ~500MB | ~2GB |
| medium | `openai/whisper-medium` | ~1.5GB | ~5GB |
| large | `openai/whisper-large` | ~3GB | ~10GB |
## Model Storage
Models are cached in the HuggingFace cache directory:
```
~/.cache/huggingface/hub/
├── models--Qwen--Qwen3-TTS-12Hz-1.7B-Base/
├── models--Qwen--Qwen3-TTS-12Hz-0.6B-Base/
├── models--openai--whisper-base/
└── ...
```
## Progress Tracking
### Progress Manager
Tracks download progress across all models:
```python
class ProgressManager:
def __init__(self):
self._progress = {} # model_name -> progress_info
def update_progress(
self,
model_name: str,
current: int,
total: int,
filename: str,
status: str,
):
self._progress[model_name] = {
"current": current,
"total": total,
"filename": filename,
"status": status, # downloading, complete, error
"updated_at": datetime.utcnow(),
}
def get_progress(self, model_name: str) -> Optional[dict]:
return self._progress.get(model_name)
```
### HuggingFace Progress Callback
Hooks into HuggingFace's download system:
```python
class HFProgressTracker:
def __init__(self, callback):
self.callback = callback
@contextmanager
def patch_download(self):
"""Context manager to intercept HF downloads."""
original_download = hf_hub_download
def patched_download(*args, **kwargs):
# Intercept progress
result = original_download(*args, **kwargs)
self.callback(progress_info)
return result
# Apply patch
with patch('huggingface_hub.hf_hub_download', patched_download):
yield
```
### Server-Sent Events (SSE)
Progress is streamed to the frontend:
```python
@app.get("/models/progress/{model_name}")
async def get_model_progress(model_name: str):
async def event_generator():
while True:
progress = progress_manager.get_progress(model_name)
if progress:
yield f"data: {json.dumps(progress)}\n\n"
if progress and progress["status"] in ["complete", "error"]:
break
await asyncio.sleep(0.5)
return StreamingResponse(
event_generator(),
media_type="text/event-stream"
)
```
## Task Manager
Tracks active downloads and generations:
```python
class TaskManager:
def __init__(self):
self._active_downloads = {}
self._active_generations = {}
def start_download(self, model_name: str):
self._active_downloads[model_name] = {
"status": "downloading",
"started_at": datetime.utcnow(),
}
def complete_download(self, model_name: str):
if model_name in self._active_downloads:
del self._active_downloads[model_name]
def get_active_tasks(self) -> dict:
return {
"downloads": list(self._active_downloads.values()),
"generations": list(self._active_generations.values()),
}
```
## Model Status
Check which models are downloaded and loaded:
```python
@app.get("/models/status")
async def get_model_status() -> ModelStatusListResponse:
models = []
# Check TTS models
for size, hf_id in [("1.7B", "Qwen/Qwen3-TTS-12Hz-1.7B-Base"), ...]:
downloaded = is_model_downloaded(hf_id)
loaded = tts_model._current_model_size == size
models.append(ModelStatus(
model_name=f"qwen-tts-{size}",
display_name=f"Qwen3-TTS {size}",
downloaded=downloaded,
size_mb=get_model_size_mb(hf_id),
loaded=loaded,
))
# Check Whisper models
for size in ["tiny", "base", "small", "medium", "large"]:
hf_id = f"openai/whisper-{size}"
downloaded = is_model_downloaded(hf_id)
models.append(ModelStatus(
model_name=f"whisper-{size}",
display_name=f"Whisper {size}",
downloaded=downloaded,
size_mb=get_model_size_mb(hf_id),
loaded=False, # Whisper is loaded on-demand
))
return ModelStatusListResponse(models=models)
```
## Manual Model Operations
### Load Model
```python
@app.post("/models/load")
async def load_model(model_size: str = "1.7B"):
tts_model = get_tts_model()
await tts_model.load_model_async(model_size)
return {"status": "loaded", "model_size": model_size}
```
### Unload Model
```python
@app.post("/models/unload")
async def unload_model():
tts_model = get_tts_model()
tts_model.unload_model()
return {"status": "unloaded"}
```
### Trigger Download
```python
@app.post("/models/download")
async def trigger_model_download(request: ModelDownloadRequest):
# This triggers the download in background
# Progress is tracked via /models/progress/{model_name}
if request.model_name.startswith("qwen-tts"):
size = request.model_name.split("-")[-1]
asyncio.create_task(download_tts_model(size))
elif request.model_name.startswith("whisper"):
size = request.model_name.split("-")[-1]
asyncio.create_task(download_whisper_model(size))
return {"status": "downloading"}
```
### Delete Model
```python
@app.delete("/models/{model_name}")
async def delete_model(model_name: str):
# Find and delete from HuggingFace cache
cache_dir = Path.home() / ".cache" / "huggingface" / "hub"
model_dirs = list(cache_dir.glob(f"models--*--{model_name}*"))
for model_dir in model_dirs:
shutil.rmtree(model_dir)
return {"status": "deleted"}
```
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| GET | `/models/status` | Get status of all models |
| POST | `/models/load` | Load TTS model |
| POST | `/models/unload` | Unload TTS model |
| POST | `/models/download` | Trigger model download |
| GET | `/models/progress/{name}` | Stream download progress (SSE) |
| DELETE | `/models/{name}` | Delete downloaded model |
| GET | `/tasks/active` | Get active downloads/generations |
## Response Schemas
### ModelStatus
```json
{
"model_name": "qwen-tts-1.7B",
"display_name": "Qwen3-TTS 1.7B",
"downloaded": true,
"size_mb": 3400,
"loaded": true
}
```
### ActiveTasksResponse
```json
{
"downloads": [
{
"model_name": "whisper-medium",
"status": "downloading",
"started_at": "2024-01-15T10:30:00Z"
}
],
"generations": [
{
"task_id": "uuid",
"profile_id": "uuid",
"text_preview": "Hello world...",
"started_at": "2024-01-15T10:30:00Z"
}
]
}
```
## Frontend Integration
### Progress Display
```typescript
// Subscribe to download progress via SSE
const eventSource = new EventSource(`/models/progress/${modelName}`);
eventSource.onmessage = (event) => {
const progress = JSON.parse(event.data);
updateProgressBar(progress.current / progress.total);
if (progress.status === 'complete') {
eventSource.close();
}
};
```
### Model Status UI
```typescript
// Fetch model status
const { data: models } = useQuery({
queryKey: ['models', 'status'],
queryFn: () => api.getModelStatus(),
});
// Display download/load buttons based on status
models.map(model => (
<ModelCard
name={model.display_name}
downloaded={model.downloaded}
loaded={model.loaded}
onDownload={() => triggerDownload(model.model_name)}
onLoad={() => loadModel(model.model_name)}
/>
));
```
## Error Handling
| Error | Cause | Solution |
|-------|-------|----------|
| Download failed | Network issue | Retry download |
| OOM on load | Model too large | Use smaller model |
| Model not found | Cache corrupted | Re-download |
| Slow download | HF rate limit | Wait and retry |
docs/developer/setup.mdx
+242
View File
@@ -0,0 +1,242 @@
---
title: "Development Setup"
description: "Set up your local development environment for Voicebox"
---
## Prerequisites
Before you begin, ensure you have the following installed:
<CardGroup cols={3}>
<Card title="Bun" icon="package">
[Download Bun](https://bun.sh)
```bash
curl -fsSL https://bun.sh/install | bash
```
</Card>
<Card title="Python 3.11+" icon="python">
[Download Python](https://python.org)
```bash
python --version
```
</Card>
<Card title="Rust" icon="rust">
[Install Rust](https://rustup.rs)
```bash
rustc --version
```
</Card>
</CardGroup>
## Clone the Repository
```bash
git clone https://github.com/jamiepine/voicebox.git
cd voicebox
```
## Quick Setup (Recommended)
The easiest way to get started is using the Makefile:
```bash
# Setup everything
make setup
# Start development
make dev
```
<Note>
The Makefile is available on macOS and Linux. Windows users should follow the manual setup below.
</Note>
## Manual Setup
### 1. Install JavaScript Dependencies
```bash
bun install
```
This installs dependencies for:
- `app/` - Shared React frontend
- `tauri/` - Tauri desktop wrapper
- `web/` - Web deployment wrapper
### 2. Set Up Python Backend
```bash
cd backend
# Create virtual environment
python -m venv venv
# Activate virtual environment
source venv/bin/activate # macOS/Linux
# or
venv\Scripts\activate # Windows
# Install Python dependencies
pip install -r requirements.txt
# Install Qwen3-TTS
pip install git+https://github.com/QwenLM/Qwen3-TTS.git
```
### 3. Initialize Database
```bash
cd backend
python -c "from database import init_db; init_db()"
```
This creates the SQLite database at `data/voicebox.db`.
## Running in Development
Development requires **two terminals**: one for the Python backend, one for the Tauri app.
<Tabs>
<Tab title="Terminal 1: Backend">
Start the Python server first:
```bash
cd backend
source venv/bin/activate # Activate venv
bun run dev:server
```
Or manually:
```bash
uvicorn main:app --reload --port 17493
```
Backend will be available at `http://localhost:17493`
</Tab>
<Tab title="Terminal 2: Desktop App">
Then start the Tauri app:
```bash
bun run dev
```
This will:
- Create a placeholder sidecar binary
- Start Vite dev server on port 5173
- Launch Tauri window
- Enable hot reload
</Tab>
</Tabs>
<Info>
In dev mode, the app connects to your manually-started Python server. The bundled server binary is only used in production builds.
</Info>
### Optional: Web App
```bash
bun run dev:web
```
Web app will be available at `http://localhost:5174`
## Model Downloads
Models are automatically downloaded from HuggingFace Hub on first use:
- **Whisper** (transcription): Auto-downloads on first transcription
- **Qwen3-TTS** (voice cloning): Auto-downloads on first generation (~2-4GB)
<Warning>
First-time usage will be slower due to model downloads, but subsequent runs will use cached models.
</Warning>
## Project Structure
```
voicebox/
├── app/ # Shared React frontend
│ └── src/
│ ├── components/ # UI components
│ ├── lib/ # Utilities and API client
│ └── hooks/ # React hooks
├── backend/ # Python FastAPI server
│ ├── main.py # API routes
│ ├── tts.py # Voice synthesis
│ └── database.py # SQLite operations
├── tauri/ # Desktop app wrapper
│ └── src-tauri/ # Rust backend
├── web/ # Web deployment
├── landing/ # Marketing website
└── scripts/ # Build & release scripts
```
## Available Make Commands
Run `make help` to see all available commands:
```bash
make setup # Install all dependencies
make dev # Start development servers
make dev-web # Start web development server
make build # Build desktop app
make build-web # Build web app
make clean # Clean build artifacts
make test # Run tests
```
## Generate OpenAPI Client
After starting the backend server, generate the TypeScript API client:
```bash
./scripts/generate-api.sh
# or
bun run generate:api
```
This downloads the OpenAPI schema and generates the TypeScript client in `app/src/lib/api/`
## Next Steps
<CardGroup cols={2}>
<Card title="Architecture" icon="diagram-project" href="/development/architecture">
Understand the system architecture
</Card>
<Card title="Contributing" icon="code-pull-request" href="/development/contributing">
Read the contribution guidelines
</Card>
<Card title="Building" icon="hammer" href="/development/building">
Learn how to build production releases
</Card>
<Card title="API Reference" icon="code" href="/api/overview">
Explore the REST API
</Card>
</CardGroup>
## Troubleshooting
<AccordionGroup>
<Accordion title="Backend won't start">
- Check Python version (must be 3.11+)
- Ensure virtual environment is activated
- Verify all dependencies are installed: `pip install -r requirements.txt`
- Check if port 17493 is available
</Accordion>
<Accordion title="Tauri build fails">
- Ensure Rust is installed: `rustc --version`
- Clean the build: `cd tauri/src-tauri && cargo clean`
- Try rebuilding: `bun run dev`
</Accordion>
<Accordion title="OpenAPI client generation fails">
- Ensure backend is running: `curl http://localhost:17493/openapi.json`
- Check network connectivity
- Verify the backend is accessible at localhost:17493
</Accordion>
</AccordionGroup>
See the full [Troubleshooting Guide](/guides/troubleshooting) for more issues and solutions.
docs/developer/stories.mdx
+320
View File
@@ -0,0 +1,320 @@
---
title: "Stories & Timeline"
description: "How the multi-voice timeline editor works in Voicebox"
---
## Overview
Stories allow users to arrange multiple voice generations on a timeline to create multi-voice narratives. The system supports tracks, trimming, splitting, and audio mixing.
## Architecture
**Story:** A container that holds story items with metadata.
**Story Item:** Links a generation to a story with timeline position, track, and trim data.
**Export:** Combines all items into a single mixed audio file.
## Data Model
### Story Table
```python
class Story(Base):
__tablename__ = "stories"
id = Column(String, primary_key=True)
name = Column(String, nullable=False)
description = Column(Text)
created_at = Column(DateTime)
updated_at = Column(DateTime)
```
### StoryItem Table
```python
class StoryItem(Base):
__tablename__ = "story_items"
id = Column(String, primary_key=True)
story_id = Column(String, ForeignKey("stories.id"))
generation_id = Column(String, ForeignKey("generations.id"))
start_time_ms = Column(Integer, default=0) # Timeline position
track = Column(Integer, default=0) # Track number
trim_start_ms = Column(Integer, default=0) # Trim from start
trim_end_ms = Column(Integer, default=0) # Trim from end
created_at = Column(DateTime)
```
## Timeline Concepts
### Start Time
`start_time_ms` defines when an item begins on the timeline:
```
Timeline (ms): 0----1000----2000----3000----4000
Item 1: [======]
Item 2: [==========]
Item 3: [====]
```
### Tracks
Multiple tracks allow overlapping audio:
```
Track 0: [Item 1] [Item 3]
Track 1: [Item 2]
```
### Trimming
Trim values cut audio from the start or end without destroying the original:
```
Original: [=========AUDIO=========]
trim_start: ^^
trim_end: ^^
Result: [=====AUDIO=====]
```
## Core Operations
### Adding Items
When adding a generation to a story:
```python
async def add_item_to_story(
story_id: str,
data: StoryItemCreate,
db: Session,
) -> StoryItemDetail:
# Calculate start time if not provided
if data.start_time_ms is None:
# Find the end of all existing items
existing_items = get_items_with_durations(story_id, db)
max_end_time_ms = max(
item.start_time_ms + int(gen.duration * 1000)
for item, gen in existing_items
)
start_time_ms = max_end_time_ms + 200 # 200ms gap
# Create the item
item = DBStoryItem(
id=str(uuid.uuid4()),
story_id=story_id,
generation_id=data.generation_id,
start_time_ms=start_time_ms,
track=data.track or 0,
)
db.add(item)
db.commit()
```
### Moving Items
Update position and/or track:
```python
async def move_story_item(
story_id: str,
item_id: str,
data: StoryItemMove,
db: Session,
) -> StoryItemDetail:
item = get_item(story_id, item_id, db)
item.start_time_ms = data.start_time_ms
item.track = data.track
db.commit()
```
### Trimming Items
Non-destructive trimming:
```python
async def trim_story_item(
story_id: str,
item_id: str,
data: StoryItemTrim,
db: Session,
) -> StoryItemDetail:
item = get_item(story_id, item_id, db)
generation = get_generation(item.generation_id, db)
# Validate trim doesn't exceed duration
max_duration_ms = int(generation.duration * 1000)
if data.trim_start_ms + data.trim_end_ms >= max_duration_ms:
return None # Invalid trim
item.trim_start_ms = data.trim_start_ms
item.trim_end_ms = data.trim_end_ms
db.commit()
```
### Splitting Items
Split one item into two at a specific time:
```python
async def split_story_item(
story_id: str,
item_id: str,
data: StoryItemSplit,
db: Session,
) -> List[StoryItemDetail]:
item = get_item(story_id, item_id, db)
generation = get_generation(item.generation_id, db)
# Calculate split point
current_trim_start = item.trim_start_ms
current_trim_end = item.trim_end_ms
original_duration_ms = int(generation.duration * 1000)
absolute_split_ms = current_trim_start + data.split_time_ms
# Update original: trim from end
item.trim_end_ms = original_duration_ms - absolute_split_ms
# Create new item: trim from start
new_item = DBStoryItem(
generation_id=item.generation_id, # Same generation
start_time_ms=item.start_time_ms + data.split_time_ms,
track=item.track,
trim_start_ms=absolute_split_ms,
trim_end_ms=current_trim_end,
)
db.add(new_item)
db.commit()
return [item, new_item]
```
### Duplicating Items
Create a copy with all properties:
```python
async def duplicate_story_item(
story_id: str,
item_id: str,
db: Session,
) -> StoryItemDetail:
original = get_item(story_id, item_id, db)
generation = get_generation(original.generation_id, db)
# Calculate effective duration for positioning
effective_duration_ms = (
int(generation.duration * 1000)
- original.trim_start_ms
- original.trim_end_ms
)
# Place copy after original with 200ms gap
new_item = DBStoryItem(
generation_id=original.generation_id,
start_time_ms=original.start_time_ms + effective_duration_ms + 200,
track=original.track,
trim_start_ms=original.trim_start_ms,
trim_end_ms=original.trim_end_ms,
)
db.add(new_item)
db.commit()
```
## Audio Export
### Mixing Algorithm
The export function mixes all items into a single audio file:
```python
async def export_story_audio(story_id: str, db: Session) -> bytes:
items = get_all_items_with_generations(story_id, db)
# Calculate total duration
max_end_time_ms = max(
data['start_time_ms'] + data['duration_ms']
for data in audio_data
)
# Create output buffer
total_samples = int((max_end_time_ms / 1000.0) * sample_rate)
final_audio = np.zeros(total_samples, dtype=np.float32)
# Mix each item at its position
for data in audio_data:
audio = data['audio']
start_sample = int((data['start_time_ms'] / 1000.0) * sample_rate)
# Apply trim
trimmed_audio = audio[trim_start_sample:len(audio) - trim_end_sample]
# Add to buffer (overlapping items sum together)
final_audio[start_sample:start_sample + len(trimmed_audio)] += trimmed_audio
# Normalize to prevent clipping
max_val = np.abs(final_audio).max()
if max_val > 1.0:
final_audio = final_audio / max_val
return audio_to_bytes(final_audio, sample_rate)
```
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| GET | `/stories` | List all stories |
| POST | `/stories` | Create a story |
| GET | `/stories/{id}` | Get story with items |
| PUT | `/stories/{id}` | Update story metadata |
| DELETE | `/stories/{id}` | Delete story |
| POST | `/stories/{id}/items` | Add item to story |
| DELETE | `/stories/{id}/items/{item_id}` | Remove item |
| PUT | `/stories/{id}/items/{item_id}/move` | Move item |
| PUT | `/stories/{id}/items/{item_id}/trim` | Trim item |
| POST | `/stories/{id}/items/{item_id}/split` | Split item |
| POST | `/stories/{id}/items/{item_id}/duplicate` | Duplicate item |
| PUT | `/stories/{id}/items/times` | Batch update times |
| PUT | `/stories/{id}/items/reorder` | Reorder items |
| GET | `/stories/{id}/export-audio` | Export mixed audio |
## Response Schemas
### StoryItemDetail
```json
{
"id": "item_uuid",
"story_id": "story_uuid",
"generation_id": "generation_uuid",
"start_time_ms": 1500,
"track": 0,
"trim_start_ms": 200,
"trim_end_ms": 100,
"profile_id": "profile_uuid",
"profile_name": "Narrator",
"text": "Hello world",
"audio_path": "/path/to/audio.wav",
"duration": 2.5,
"created_at": "2024-01-15T10:30:00Z"
}
```
## Frontend Integration
The timeline UI needs to:
1. **Fetch story** with all items
2. **Render waveforms** for each item
3. **Handle drag/drop** to move items
4. **Handle edge drag** for trimming
5. **Sync playhead** across all tracks
6. **Export** when user clicks download
docs/developer/transcription.mdx
+299
View File
@@ -0,0 +1,299 @@
---
title: "Transcription"
description: "How Whisper-based audio transcription works in Voicebox"
---
## Overview
Voicebox uses OpenAI's Whisper model for automatic speech recognition (ASR). This powers the transcription feature for creating reference text from audio recordings.
## Architecture
The transcription system is built around the `WhisperModel` class:
**Model Loading:** Lazy loading with HuggingFace Hub download.
**Audio Processing:** Resampling and preprocessing for Whisper.
**Inference:** Running transcription with optional language hints.
## WhisperModel Class
```python
class WhisperModel:
def __init__(self, model_size: str = "base"):
self.model = None
self.processor = None
self.model_size = model_size
self.device = self._get_device()
```
### Model Sizes
| Size | Parameters | VRAM | Speed | Quality |
|------|------------|------|-------|---------|
| tiny | 39M | ~1GB | Fastest | Basic |
| base | 74M | ~1GB | Fast | Good |
| small | 244M | ~2GB | Medium | Better |
| medium | 769M | ~5GB | Slow | High |
| large | 1550M | ~10GB | Slowest | Best |
Default is `base` for balance of speed and quality.
## Model Loading
Models are downloaded from HuggingFace Hub:
```python
def load_model(self, model_size: Optional[str] = None):
from transformers import WhisperProcessor, WhisperForConditionalGeneration
model_name = f"openai/whisper-{model_size}"
# Track download progress
progress_manager = get_progress_manager()
task_manager = get_task_manager()
task_manager.start_download(f"whisper-{model_size}")
# Load processor and model
with tracker.patch_download():
self.processor = WhisperProcessor.from_pretrained(model_name)
self.model = WhisperForConditionalGeneration.from_pretrained(model_name)
self.model.to(self.device)
# Mark complete
progress_manager.mark_complete(f"whisper-{model_size}")
task_manager.complete_download(f"whisper-{model_size}")
```
### Async Loading
Like TTS, loading runs in a thread pool:
```python
async def load_model_async(self, model_size: Optional[str] = None):
if self.model is not None and self.model_size == model_size:
return
await asyncio.to_thread(self.load_model, model_size)
```
## Transcription
### Basic Transcription
```python
async def transcribe(
self,
audio_path: str,
language: Optional[str] = None,
) -> str:
await self.load_model_async()
def _transcribe_sync():
# Load and resample to 16kHz (Whisper requirement)
audio, sr = load_audio(audio_path, sample_rate=16000)
# Process audio
inputs = self.processor(
audio,
sampling_rate=16000,
return_tensors="pt",
)
inputs = inputs.to(self.device)
# Set language hint if provided
forced_decoder_ids = None
if language:
forced_decoder_ids = self.processor.get_decoder_prompt_ids(
language=language,
task="transcribe",
)
# Generate
with torch.no_grad():
predicted_ids = self.model.generate(
inputs["input_features"],
forced_decoder_ids=forced_decoder_ids,
)
# Decode
transcription = self.processor.batch_decode(
predicted_ids,
skip_special_tokens=True,
)[0]
return transcription.strip()
return await asyncio.to_thread(_transcribe_sync)
```
### Supported Languages
Whisper supports 99+ languages. Common ones in Voicebox:
| Code | Language |
|------|----------|
| en | English |
| zh | Chinese |
| ja | Japanese |
| ko | Korean |
| de | German |
| fr | French |
| ru | Russian |
| pt | Portuguese |
| es | Spanish |
| it | Italian |
### Language Detection
When no language is specified, Whisper auto-detects:
```python
# Without language hint - auto-detect
transcription = await whisper.transcribe(audio_path)
# With language hint - more accurate for short clips
transcription = await whisper.transcribe(audio_path, language="en")
```
## Transcription with Timestamps
For advanced use cases, word-level timestamps are available:
```python
async def transcribe_with_timestamps(
self,
audio_path: str,
language: Optional[str] = None,
) -> List[Dict[str, any]]:
await self.load_model_async()
def _transcribe_timestamps_sync():
audio, sr = load_audio(audio_path, sample_rate=16000)
inputs = self.processor(audio, sampling_rate=16000, return_tensors="pt")
with torch.no_grad():
predicted_ids = self.model.generate(
inputs["input_features"],
return_timestamps=True,
)
# Parse timestamps
return [
{
"text": transcription,
"start": 0.0,
"end": len(audio) / sr,
}
]
return await asyncio.to_thread(_transcribe_timestamps_sync)
```
## Memory Management
### Unloading
Free memory when not needed:
```python
def unload_model(self):
if self.model is not None:
del self.model
del self.processor
self.model = None
self.processor = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
```
### Global Instance
A singleton pattern manages the model:
```python
_whisper_model: Optional[WhisperModel] = None
def get_whisper_model() -> WhisperModel:
global _whisper_model
if _whisper_model is None:
_whisper_model = WhisperModel()
return _whisper_model
```
## Audio Preprocessing
### Resampling
Whisper requires 16kHz audio:
```python
audio, sr = load_audio(audio_path, sample_rate=16000)
```
### Format Support
The `load_audio` utility handles:
- WAV
- MP3
- FLAC
- OGG
- M4A
All formats are converted to mono 16kHz.
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| POST | `/transcribe` | Transcribe audio file |
### Request
Multipart form data:
```
POST /transcribe
Content-Type: multipart/form-data
file: <audio_file>
language: en (optional)
```
### Response
```json
{
"text": "Hello, this is a test transcription.",
"duration": 3.5
}
```
## Use Cases
### Reference Text for Voice Cloning
1. User records audio sample
2. Audio is sent to `/transcribe`
3. Transcription becomes `reference_text`
4. Both are added to voice profile
### Quality Tips
- Provide language hint for short audio
- Use clean audio with minimal noise
- Longer audio (>5s) improves accuracy
- Consider `small` or `medium` model for better quality
## Error Handling
Common issues:
| Error | Cause | Solution |
|-------|-------|----------|
| Model not found | First run, download failed | Retry with network |
| OOM | Model too large | Use smaller model |
| Empty result | No speech detected | Check audio has speech |
| Wrong language | Auto-detect failed | Provide language hint |
docs/developer/tts-generation.mdx
+283
View File
@@ -0,0 +1,283 @@
---
title: "TTS Generation"
description: "How text-to-speech generation works in Voicebox"
---
## Overview
Voicebox uses Qwen3-TTS for voice cloning and text-to-speech generation. The TTS module handles model loading, voice prompt creation, and audio synthesis.
## Architecture
The TTS system is built around the `TTSModel` class which manages:
**Model Loading:** Lazy loading with automatic HuggingFace Hub download.
**Voice Prompts:** Converting reference audio into embeddings.
**Generation:** Synthesizing speech from text using voice prompts.
## TTSModel Class
```python
class TTSModel:
def __init__(self, model_size: str = "1.7B"):
self.model = None
self.model_size = model_size
self.device = self._get_device() # cuda, mps, or cpu
```
### Device Selection
The model automatically selects the best available device:
```python
def _get_device(self) -> str:
if torch.cuda.is_available():
return "cuda"
elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
return "cpu" # MPS can have issues, use CPU for stability
return "cpu"
```
## Model Loading
Models are downloaded from HuggingFace Hub on first use:
```python
def load_model(self, model_size: Optional[str] = None):
# Model IDs on HuggingFace Hub
hf_model_map = {
"1.7B": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
"0.6B": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
}
# Load with progress tracking
with tracker.patch_download():
self.model = Qwen3TTSModel.from_pretrained(
model_path,
device_map=self.device,
torch_dtype=torch.bfloat16, # float32 on CPU
)
```
### Async Loading
Loading runs in a thread pool to avoid blocking the event loop:
```python
async def load_model_async(self, model_size: Optional[str] = None):
if self.model is not None and self._current_model_size == model_size:
return
await asyncio.to_thread(self.load_model, model_size)
```
## Voice Prompt Creation
Voice prompts are created from reference audio and cached for reuse:
```python
async def create_voice_prompt(
self,
audio_path: str,
reference_text: str,
use_cache: bool = True,
) -> Tuple[dict, bool]:
await self.load_model_async()
# Check cache
if use_cache:
cache_key = get_cache_key(audio_path, reference_text)
cached = get_cached_voice_prompt(cache_key)
if cached:
return cached, True
# Create prompt (blocking, run in thread pool)
voice_prompt = await asyncio.to_thread(
self.model.create_voice_clone_prompt,
ref_audio=audio_path,
ref_text=reference_text,
)
# Cache the result
cache_voice_prompt(cache_key, voice_prompt)
return voice_prompt, False
```
### Combining Multiple Samples
When a profile has multiple samples, they're combined:
```python
async def combine_voice_prompts(
self,
audio_paths: List[str],
reference_texts: List[str],
) -> Tuple[np.ndarray, str]:
combined_audio = []
for audio_path in audio_paths:
audio, sr = load_audio(audio_path)
audio = normalize_audio(audio)
combined_audio.append(audio)
# Concatenate and normalize
mixed = np.concatenate(combined_audio)
mixed = normalize_audio(mixed)
# Combine texts
combined_text = " ".join(reference_texts)
return mixed, combined_text
```
## Speech Generation
The core generation function:
```python
async def generate(
self,
text: str,
voice_prompt: dict,
language: str = "en",
seed: Optional[int] = None,
instruct: Optional[str] = None,
) -> Tuple[np.ndarray, int]:
await self.load_model_async()
def _generate_sync():
# Set seed for reproducibility
if seed is not None:
torch.manual_seed(seed)
# Generate audio
wavs, sample_rate = self.model.generate_voice_clone(
text=text,
voice_clone_prompt=voice_prompt,
instruct=instruct, # Natural language delivery control
)
return wavs[0], sample_rate
# Run in thread pool
return await asyncio.to_thread(_generate_sync)
```
### Instruct Feature
The `instruct` parameter allows natural language control over speech delivery:
```python
# Examples:
instruct = "Speak slowly and clearly"
instruct = "Sound excited and enthusiastic"
instruct = "Whisper softly"
```
## Caching Strategy
Voice prompts are cached to avoid recomputation:
```python
def get_cache_key(audio_path: str, reference_text: str) -> str:
"""Generate cache key from audio hash and text."""
audio_hash = hashlib.md5(Path(audio_path).read_bytes()).hexdigest()
text_hash = hashlib.md5(reference_text.encode()).hexdigest()
return f"{audio_hash}_{text_hash}"
```
Cache is stored in `data/cache/voice_prompts/`.
## Memory Management
### Unloading Models
Free VRAM/RAM when not needed:
```python
def unload_model(self):
if self.model is not None:
del self.model
self.model = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
```
### Model Switching
When switching between model sizes (1.7B ↔ 0.6B):
```python
# Unload existing model first
if self.model is not None and self._current_model_size != model_size:
self.unload_model()
```
## Generation Flow
1. **Request** → Validate text and profile ID
2. **Profile** → Load profile samples from database
3. **Voice Prompt** → Create or retrieve cached prompt
4. **Generate** → Run TTS inference
5. **Save** → Write audio to generations directory
6. **Record** → Create history entry in database
7. **Response** → Return audio path and metadata
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| POST | `/generate` | Generate speech from text |
| GET | `/audio/{id}` | Serve generated audio file |
### Request Schema
```json
{
"profile_id": "uuid",
"text": "Text to synthesize",
"language": "en",
"seed": 42,
"model_size": "1.7B",
"instruct": "Speak clearly"
}
```
### Response Schema
```json
{
"id": "generation_uuid",
"profile_id": "profile_uuid",
"text": "Text to synthesize",
"language": "en",
"audio_path": "/path/to/audio.wav",
"duration": 3.5,
"seed": 42,
"instruct": "Speak clearly",
"created_at": "2024-01-15T10:30:00Z"
}
```
## Performance Considerations
### GPU Acceleration
- CUDA provides fastest inference
- MPS (Apple Silicon) has stability issues, uses CPU fallback
- CPU inference is slower but always works
### Batch Size
Currently generates one utterance at a time. For long texts, consider:
- Splitting into sentences
- Sequential generation
- Concatenating results
### Memory Usage
| Model | VRAM/RAM Required |
|-------|-------------------|
| 0.6B | ~2GB |
| 1.7B | ~6GB |
docs/developer/voice-profiles.mdx
+202
View File
@@ -0,0 +1,202 @@
---
title: "Voice Profiles"
description: "How voice profile management works in Voicebox"
---
## Overview
Voice profiles are the foundation of Voicebox's voice cloning capability. Each profile stores reference audio samples and metadata that the TTS model uses to clone a voice.
## Architecture
The voice profile system consists of three main components:
**Database Layer:** SQLite tables store profile metadata and sample references.
**File Storage:** Audio samples are stored on disk in a structured directory format.
**Profile Module:** The `profiles.py` module provides the business logic for CRUD operations.
## Data Model
### VoiceProfile Table
```python
class VoiceProfile(Base):
__tablename__ = "profiles"
id = Column(String, primary_key=True)
name = Column(String, unique=True, nullable=False)
description = Column(Text)
language = Column(String, default="en")
created_at = Column(DateTime)
updated_at = Column(DateTime)
```
### ProfileSample Table
```python
class ProfileSample(Base):
__tablename__ = "profile_samples"
id = Column(String, primary_key=True)
profile_id = Column(String, ForeignKey("profiles.id"))
audio_path = Column(String, nullable=False)
reference_text = Column(Text, nullable=False)
```
## File Structure
Profiles are stored in the data directory:
```
data/
└── profiles/
└── {profile_id}/
├── {sample_id_1}.wav
├── {sample_id_2}.wav
└── ...
```
## Core Functions
### Creating a Profile
```python
async def create_profile(data: VoiceProfileCreate, db: Session) -> VoiceProfileResponse:
# 1. Create database record
db_profile = DBVoiceProfile(
id=str(uuid.uuid4()),
name=data.name,
description=data.description,
language=data.language,
)
db.add(db_profile)
db.commit()
# 2. Create profile directory
profile_dir = profiles_dir / db_profile.id
profile_dir.mkdir(parents=True, exist_ok=True)
return VoiceProfileResponse.model_validate(db_profile)
```
### Adding Samples
When a sample is added, the audio is validated and copied to the profile directory:
```python
async def add_profile_sample(
profile_id: str,
audio_path: str,
reference_text: str,
db: Session,
) -> ProfileSampleResponse:
# 1. Validate audio (duration, format, quality)
is_valid, error_msg = validate_reference_audio(audio_path)
if not is_valid:
raise ValueError(f"Invalid reference audio: {error_msg}")
# 2. Copy to profile directory
sample_id = str(uuid.uuid4())
dest_path = profile_dir / f"{sample_id}.wav"
audio, sr = load_audio(audio_path)
save_audio(audio, str(dest_path), sr)
# 3. Create database record
db_sample = DBProfileSample(
id=sample_id,
profile_id=profile_id,
audio_path=str(dest_path),
reference_text=reference_text,
)
db.add(db_sample)
db.commit()
```
### Voice Prompt Creation
When generating speech, samples are combined into a voice prompt:
```python
async def create_voice_prompt_for_profile(
profile_id: str,
db: Session,
) -> dict:
samples = db.query(DBProfileSample).filter_by(profile_id=profile_id).all()
if len(samples) == 1:
# Single sample - use directly
voice_prompt, _ = await tts_model.create_voice_prompt(
sample.audio_path,
sample.reference_text,
)
else:
# Multiple samples - combine them
combined_audio, combined_text = await tts_model.combine_voice_prompts(
[s.audio_path for s in samples],
[s.reference_text for s in samples],
)
voice_prompt, _ = await tts_model.create_voice_prompt(
combined_audio_path,
combined_text,
)
return voice_prompt
```
## Audio Validation
Reference audio is validated before being accepted:
- **Duration:** 3-30 seconds recommended
- **Format:** WAV, MP3, FLAC, OGG supported
- **Sample Rate:** Resampled to 24kHz
- **Channels:** Converted to mono if stereo
## Export/Import
Profiles can be exported as ZIP archives for sharing:
```
profile_export.zip
├── profile.json # Metadata
├── samples/
│ ├── sample_1.wav
│ └── sample_1.json # Reference text
└── ...
```
## API Endpoints
| Method | Endpoint | Description |
|--------|----------|-------------|
| GET | `/profiles` | List all profiles |
| POST | `/profiles` | Create a profile |
| GET | `/profiles/{id}` | Get profile by ID |
| PUT | `/profiles/{id}` | Update profile |
| DELETE | `/profiles/{id}` | Delete profile |
| GET | `/profiles/{id}/samples` | Get profile samples |
| POST | `/profiles/{id}/samples` | Add sample to profile |
| PUT | `/profiles/samples/{id}` | Update sample text |
| DELETE | `/profiles/samples/{id}` | Delete sample |
| GET | `/profiles/{id}/export` | Export as ZIP |
| POST | `/profiles/import` | Import from ZIP |
## Best Practices
### Sample Quality
- Use clean audio with minimal background noise
- Ensure the reference text exactly matches what is spoken
- Multiple samples (3-5) improve voice cloning quality
### Language Matching
- Set the profile language to match the reference audio
- Supported languages: en, zh, ja, ko, de, fr, ru, pt, es, it
### Naming Conventions
- Use descriptive names that identify the voice
- Avoid special characters that may cause filesystem issues
docs/images/app-screenshot-1.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 134 KiB

docs/images/app-screenshot-2.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 129 KiB

docs/images/app-screenshot-3.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 108 KiB

docs/logo/icon-dark.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

docs/logo/icon-light.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

docs/mint.json
+102
View File
@@ -0,0 +1,102 @@
{
"$schema": "https://mintlify.com/schema.json",
"name": "Voicebox",
"logo": {
"light": "/logo/icon-light.png",
"dark": "/logo/icon-dark.png"
},
"favicon": "/favicon.png",
"colors": {
"primary": "#BF9E40",
"light": "#D4B560",
"dark": "#A68A35"
},
"styles": {
"css": ["/custom.css"]
},
"anchors": [
{
"name": "Overview",
"icon": "book-open",
"url": "overview"
},
{
"name": "API",
"icon": "code",
"url": "api"
},
{
"name": "Developer",
"icon": "book-open-cover",
"url": "developer"
},
{
"name": "GitHub",
"icon": "github",
"url": "https://github.com/jamiepine/voicebox"
}
],
"navigation": [
{
"group": "Getting Started",
"icon": "rocket",
"pages": ["overview/introduction", "overview/installation", "overview/quick-start"]
},
{
"group": "Features",
"icon": "sparkles",
"pages": [
"overview/voice-cloning",
"overview/stories-editor",
"overview/recording-transcription",
"overview/generation-history",
"overview/remote-mode"
]
},
{
"group": "User Guides",
"icon": "compass",
"pages": [
"overview/creating-voice-profiles",
"overview/generating-speech",
"overview/building-stories",
"overview/troubleshooting"
]
},
{
"group": "Development",
"icon": "wrench",
"pages": [
"developer/setup",
"developer/architecture",
"developer/contributing",
"developer/building",
"developer/autoupdater"
]
},
{
"group": "API Reference",
"icon": "code",
"pages": [
"api/overview",
"api/authentication",
"api/voice-profiles",
"api/generation",
"api/recordings"
]
},
{
"group": "Architecture",
"icon": "book-open-cover",
"pages": [
"developer/voice-profiles",
"developer/tts-generation",
"developer/history",
"developer/stories",
"developer/transcription",
"developer/audio-channels",
"developer/model-management"
]
}
]
}
docs/overview/building-stories.mdx
+37
View File
@@ -0,0 +1,37 @@
---
title: "Building Stories"
description: "Create multi-voice narratives with the Stories Editor"
---
## Getting Started
The Stories Editor is perfect for creating podcasts, audiobooks, and multi-speaker content.
<Steps>
<Step title="Create Story">
**Stories** → **+ New Story**
</Step>
<Step title="Add Tracks">
Create tracks for each speaker
</Step>
<Step title="Add Clips">
Generate or drag audio to tracks
</Step>
<Step title="Arrange">
Position and trim clips on timeline
</Step>
<Step title="Export">
Render final audio
</Step>
</Steps>
## Use Cases
- Multi-host podcasts
- Audiobook narration with character voices
- Game dialogue scenes
- Educational content with multiple speakers
## Coming Soon
Full timeline editor documentation will be added as features are finalized.
docs/overview/creating-voice-profiles.mdx
+296
View File
@@ -0,0 +1,296 @@
---
title: "Creating Voice Profiles"
description: "Advanced guide to creating high-quality voice profiles"
---
## Overview
Voice profiles are the foundation of voice cloning in Voicebox. This guide covers best practices for creating professional-quality voice profiles.
## Quick Start
<Steps>
<Step title="Prepare Audio">
10-30 seconds of clear speech
</Step>
<Step title="Create Profile">
**Profiles** → **+ New Profile**
</Step>
<Step title="Upload Sample">
Add your audio file
</Step>
<Step title="Generate">
Use the profile to generate speech
</Step>
</Steps>
## Audio Requirements
### Ideal Sample Characteristics
<CardGroup cols={2}>
<Card title="Duration" icon="clock">
**10-30 seconds**
Too short: Poor quality
Too long: Unnecessary
</Card>
<Card title="Clarity" icon="volume">
**Clear speech**
No background noise
No music or overlapping voices
</Card>
<Card title="Quality" icon="sparkles">
**High fidelity**
44.1kHz or 48kHz sample rate
Minimal compression
</Card>
<Card title="Content" icon="microphone">
**Natural speech**
Conversational tone
Complete sentences
</Card>
</CardGroup>
### File Formats
Supported formats:
- **WAV** (recommended) - Lossless quality
- **MP3** - Acceptable, minimal compression
- **M4A** - Acceptable
- **FLAC** - Lossless alternative
<Tip>
Use WAV for best results. Avoid heavily compressed formats.
</Tip>
## Recording Tips
### Environment
<AccordionGroup>
<Accordion title="Quiet Space">
- Record in a quiet room
- Turn off fans, AC, appliances
- Close windows to reduce outside noise
- Use soft furnishings to reduce echo
</Accordion>
<Accordion title="Microphone Placement">
- 6-12 inches from mouth
- Slight angle to reduce plosives (p, b, t)
- Use a pop filter if available
- Maintain consistent distance
</Accordion>
<Accordion title="Recording Settings">
- 44.1kHz or 48kHz sample rate
- 16-bit or 24-bit depth
- Mono is fine (stereo will be converted)
- Avoid automatic gain control
</Accordion>
</AccordionGroup>
### Speaking
- **Natural pace** - Don't rush or speak too slowly
- **Clear articulation** - Pronounce words clearly
- **Consistent volume** - Maintain steady loudness
- **Normal tone** - Speak as you normally would
- **Complete sentences** - Avoid fragments or "ums"
## Multiple Samples
Adding multiple samples can significantly improve quality:
### Why Multiple Samples?
<CardGroup cols={2}>
<Card title="Robustness" icon="shield">
Model learns a more complete representation
</Card>
<Card title="Versatility" icon="palette">
Handles different speaking styles better
</Card>
<Card title="Quality" icon="star">
Reduces artifacts and improves naturalness
</Card>
<Card title="Consistency" icon="check">
More reliable across different texts
</Card>
</CardGroup>
### Sample Variety
Consider adding samples with:
1. **Different tones**
- Casual conversation
- Professional/formal
- Excited/enthusiastic
- Calm/serious
2. **Different content**
- Narratives
- Questions
- Statements
- Emotions (happy, sad, neutral)
3. **Different recording conditions**
- Studio quality
- Phone call quality (if needed)
- Room acoustics
<Warning>
All samples should be from the **same speaker**. Mixing voices will produce poor results.
</Warning>
## Processing Existing Audio
If you have existing audio (podcasts, videos, etc.):
### Extracting Clean Segments
<Steps>
<Step title="Find Clean Speech">
Look for segments with:
- Just the target speaker
- No background music
- Minimal noise
</Step>
<Step title="Use Audio Editor">
Tools like Audacity or Adobe Audition:
- Cut out clean 10-30s segments
- Remove silence at start/end
- Normalize volume if needed
</Step>
<Step title="Export as WAV">
Save as high-quality WAV file
</Step>
</Steps>
### Noise Reduction
If you have light background noise:
```
1. Use noise reduction in Audacity:
- Select noise-only section
- Get Noise Profile
- Select full audio
- Apply noise reduction (gentle settings)
2. Avoid over-processing:
- Can introduce artifacts
- May reduce voice quality
```
## Testing & Iteration
### Test Your Profile
After creating a profile:
<Steps>
<Step title="Generate Test">
Generate a simple phrase:
```
"Hello, this is a test of my voice profile."
```
</Step>
<Step title="Evaluate Quality">
Listen for:
- Natural tone
- Clear pronunciation
- Proper prosody
- Lack of artifacts
</Step>
<Step title="Iterate">
If quality is poor:
- Add more samples
- Try different source audio
- Check sample quality
</Step>
</Steps>
### Common Issues
<AccordionGroup>
<Accordion title="Robotic Voice">
**Cause**: Poor quality samples or too short
**Fix**: Use longer, higher quality samples
</Accordion>
<Accordion title="Wrong Tone">
**Cause**: Sample tone doesn't match desired output
**Fix**: Record samples in the style you want to generate
</Accordion>
<Accordion title="Artifacts/Glitches">
**Cause**: Background noise or audio issues in samples
**Fix**: Clean up samples or re-record in quieter environment
</Accordion>
</AccordionGroup>
## Advanced Tips
### Celebrity/Character Voices
For cloning public figures or characters:
1. **Legal considerations** - Ensure you have rights or it's fair use
2. **Source quality** - Find high-quality interview audio or clean clips
3. **Consistency** - Use clips where they speak similarly
4. **Multiple samples** - Very important for recognizable voices
### Accent & Dialect
The model will preserve accent and dialect:
- British English will generate British English
- Southern accent will produce Southern accent
- Regional pronunciations will be maintained
### Emotion Transfer
The emotional tone of samples affects generation:
- Energetic samples → Energetic output
- Calm samples → Calm output
- Mix samples for versatile profile
## Managing Profiles
### Organization
- **Descriptive names** - "John Smith - Professional Narrator"
- **Add descriptions** - Note recording conditions, use cases
- **Language tags** - Mark the primary language
- **Archive unused** - Keep profile list manageable
### Export/Import
- **Export** profiles to share or backup
- **Import** from colleagues or teammates
- Profiles include voice embeddings, not original audio
## Next Steps
<CardGroup cols={2}>
<Card title="Generate Speech" icon="waveform" href="/guides/generating-speech">
Use your profile to generate speech
</Card>
<Card title="Build Stories" icon="film" href="/guides/building-stories">
Create multi-voice narratives
</Card>
</CardGroup>
docs/overview/generating-speech.mdx
+65
View File
@@ -0,0 +1,65 @@
---
title: "Generating Speech"
description: "Generate high-quality speech from text"
---
## Basic Generation
<Steps>
<Step title="Select Profile">
Choose a voice profile from the dropdown
</Step>
<Step title="Enter Text">
Type or paste your text
</Step>
<Step title="Generate">
Click **Generate** and wait a few seconds
</Step>
<Step title="Play & Export">
Preview and download the result
</Step>
</Steps>
## Text Formatting Tips
The way you format text affects the output quality.
### Punctuation
Use proper punctuation for natural pauses:
```
Good: "Hello! How are you today? I'm doing great."
Bad: "Hello how are you today Im doing great"
```
### Emphasis
Use formatting to suggest emphasis:
```
- ALL CAPS for louder/emphasized: "That was AMAZING!"
- Italics for subtle emphasis: "I *really* enjoyed that"
- Bold for strong emphasis: "This is **very** important"
```
<Note>
The model interprets these hints but results may vary.
</Note>
## Advanced Features
### Batch Generation
For long-form content, split into smaller chunks for better control and faster processing.
### Voice Caching
Voicebox caches voice prompts for faster re-generation with the same profile.
## Coming Soon
- Real-time streaming
- Word-level timing control
- Emotion and style controls
- SSML support
docs/overview/generation-history.mdx
+88
View File
@@ -0,0 +1,88 @@
---
title: "Generation History"
description: "Track and manage all your generated audio"
---
## Overview
Voicebox keeps a complete history of all generated audio, making it easy to find, reuse, and manage your creations.
## Features
<CardGroup cols={2}>
<Card title="Full History" icon="clock">
Every generation is automatically saved
</Card>
<Card title="Search & Filter" icon="search">
Find by text, voice, or date
</Card>
<Card title="Re-generate" icon="rotate">
Regenerate any past generation with one click
</Card>
<Card title="Export" icon="download">
Download individual or batch exports
</Card>
</CardGroup>
## Viewing History
Navigate to the **History** tab to see all your generations.
Each entry shows:
- Generated text
- Voice profile used
- Timestamp
- Audio duration
- Language
## Actions
### Play
Click any generation to play it immediately.
### Re-generate
Regenerate with the same settings or modify the text/voice.
### Download
Export as WAV, MP3, or M4A.
### Delete
Remove unwanted generations to free up space.
### Add to Story
Drag generations to the Stories Editor timeline.
## Search & Filter
<Tabs>
<Tab title="By Text">
Search for specific text content
```
"Hello world"
```
</Tab>
<Tab title="By Voice">
Filter by voice profile
```
Select from dropdown
```
</Tab>
<Tab title="By Date">
Filter by date range
```
Last 7 days, Last 30 days, Custom range
```
</Tab>
</Tabs>
## Storage
History is stored locally:
- **macOS**: `~/Library/Application Support/com.voicebox.app/data/`
- **Windows**: `%APPDATA%/com.voicebox.app/data/`
- **Linux**: `~/.config/com.voicebox.app/data/`
<Warning>
Deleting the data directory will remove all history. Export important files first.
</Warning>
docs/overview/installation.mdx
+119
View File
@@ -0,0 +1,119 @@
---
title: "Installation"
description: "Download and install Voicebox on macOS, Windows, or Linux"
---
## Download
Voicebox is available for macOS and Windows, with Linux builds coming soon.
<CardGroup cols={2}>
<Card title="macOS" icon="apple">
Download for Apple Silicon or Intel Macs
</Card>
<Card title="Windows" icon="windows">
Download MSI installer or Setup executable
</Card>
</CardGroup>
### macOS
<Tabs>
<Tab title="Apple Silicon">
Download: [voicebox_aarch64.app.tar.gz](https://github.com/jamiepine/voicebox/releases/latest/download/voicebox_aarch64.app.tar.gz)
```bash
# Extract the archive
tar -xzf voicebox_aarch64.app.tar.gz
# Move to Applications
mv Voicebox.app /Applications/
```
</Tab>
<Tab title="Intel">
Download: [voicebox_x64.app.tar.gz](https://github.com/jamiepine/voicebox/releases/latest/download/voicebox_x64.app.tar.gz)
```bash
# Extract the archive
tar -xzf voicebox_x64.app.tar.gz
# Move to Applications
mv Voicebox.app /Applications/
```
</Tab>
</Tabs>
### Windows
<Tabs>
<Tab title="MSI Installer">
Download: [voicebox_x64_en-US.msi](https://github.com/jamiepine/voicebox/releases/latest/download/voicebox_x64_en-US.msi)
Double-click the MSI file and follow the installation wizard.
</Tab>
<Tab title="Setup Executable">
Download: [voicebox_x64-setup.exe](https://github.com/jamiepine/voicebox/releases/latest/download/voicebox_x64-setup.exe)
Run the executable and follow the installation wizard.
</Tab>
</Tabs>
### Linux
<Note>
Linux builds are coming soon. Currently blocked by GitHub runner disk space limitations.
</Note>
## First Launch
When you launch Voicebox for the first time:
1. **Model Download** — Qwen3-TTS model (~2-4GB) will download automatically on first use
2. **Data Directory** — Voice profiles and generated audio are stored in:
- macOS: `~/Library/Application Support/com.voicebox.app/`
- Windows: `%APPDATA%/com.voicebox.app/`
- Linux: `~/.config/com.voicebox.app/`
3. **Backend Server** — The bundled Python server starts automatically
<Tip>
First generation will be slower due to model downloads. Subsequent runs use cached models.
</Tip>
## System Requirements
### Minimum
- **OS:** macOS 11+, Windows 10+, or Linux
- **RAM:** 8GB
- **Storage:** 5GB free space (for models and data)
- **CPU:** Modern multi-core processor
### Recommended
- **RAM:** 16GB+
- **GPU:** CUDA-capable NVIDIA GPU (for faster generation)
- **Storage:** 10GB+ free space
<Note>
CPU inference is supported but significantly slower than GPU. A CUDA-capable GPU is highly recommended for real-time workflows.
</Note>
## Verification
After installation, verify everything works:
1. Launch Voicebox
2. Check the server status indicator in the bottom-left corner (should be green)
3. Navigate to **Profiles** and create a test profile
4. Generate a short audio clip to verify the TTS engine works
<Check>
If you see a green status indicator and can generate audio, you're all set!
</Check>
## Next Steps
<Card title="Quick Start Guide" icon="rocket" href="/overview/quick-start">
Create your first voice profile and generate speech
</Card>
docs/overview/introduction.mdx
+58
View File
@@ -0,0 +1,58 @@
---
title: "Introduction"
description: "Welcome to Voicebox - the open-source voice synthesis studio"
---
## What is Voicebox?
Voicebox is a **local-first voice cloning studio** with DAW-like features for professional voice synthesis. Think of it as the **Ollama for voice** — download models, clone voices, and generate speech entirely on your machine.
<Frame>
<img src="/images/app-screenshot-1.webp" alt="Voicebox App Screenshot" />
</Frame>
Unlike cloud services that lock your voice data behind subscriptions, Voicebox gives you:
- **Complete privacy** — models and voice data stay on your machine
- **Professional tools** — multi-track timeline editor, audio trimming, conversation mixing
- **Model flexibility** — currently powered by Qwen3-TTS, with support for XTTS, Bark, and other models coming soon
- **API-first** — use the desktop app or integrate voice synthesis into your own projects
- **Native performance** — built with Tauri (Rust), not Electron
Download a voice model, clone any voice from a few seconds of audio, and compose multi-voice projects with studio-grade editing tools. No Python install required, no cloud dependency, no limits.
## Key Features
<CardGroup cols={2}>
<Card title="Voice Cloning" icon="microphone">
Instant cloning from just a few seconds of audio with Qwen3-TTS
</Card>
<Card title="Stories Editor" icon="film">
Multi-track timeline for creating conversations and narratives
</Card>
<Card title="Full API" icon="code">
REST API for integrating voice synthesis into your apps
</Card>
<Card title="Local-First" icon="shield">
Everything runs on your machine - complete privacy
</Card>
</CardGroup>
## Use Cases
- **Game Development** — Generate dynamic dialogue for characters
- **Content Creation** — Produce podcasts and video voiceovers
- **Accessibility** — Build text-to-speech tools
- **Voice Assistants** — Create custom voice interfaces
- **Production Pipelines** — Automate voiceover workflows
## Next Steps
<CardGroup cols={2}>
<Card title="Installation" icon="download" href="/overview/installation">
Download and install Voicebox on your machine
</Card>
<Card title="Quick Start" icon="rocket" href="/overview/quick-start">
Get up and running in 5 minutes
</Card>
</CardGroup>
docs/overview/quick-start.mdx
+154
View File
@@ -0,0 +1,154 @@
---
title: "Quick Start"
description: "Get started with Voicebox in 5 minutes"
---
This guide will walk you through creating your first voice profile and generating speech.
## Prerequisites
Make sure you have [installed Voicebox](/overview/installation) and launched the app.
## Step 1: Create a Voice Profile
Voice profiles are the foundation of Voicebox. Each profile contains voice samples that the AI uses to clone the voice.
<Steps>
<Step title="Navigate to Profiles">
Click the **Profiles** tab in the sidebar
</Step>
<Step title="Create New Profile">
Click the **+ New Profile** button
Fill in the details:
- **Name:** A descriptive name (e.g., "John Smith")
- **Language:** Select the primary language
- **Description:** Optional notes about the voice
</Step>
<Step title="Add Voice Sample">
You have two options:
**Option A: Upload Audio**
- Click **Upload Sample**
- Select an audio file (WAV, MP3, or M4A)
- Ideal length: 10-30 seconds of clear speech
**Option B: Record Live**
- Click **Record Sample**
- Speak clearly for 10-30 seconds
- Click stop when finished
</Step>
<Step title="Save Profile">
Click **Create Profile** to save
</Step>
</Steps>
<Tip>
For best results, use clean audio with minimal background noise and consistent speaking tone.
</Tip>
## Step 2: Generate Speech
Now let's use your new voice profile to generate speech.
<Steps>
<Step title="Go to Generation">
Click the **Generate** tab in the sidebar
</Step>
<Step title="Select Voice Profile">
Choose your newly created profile from the dropdown
</Step>
<Step title="Enter Text">
Type or paste the text you want to generate:
```
Hello! This is my first voice generation with Voicebox.
```
</Step>
<Step title="Generate">
Click **Generate** and wait a few seconds
<Note>
First generation may take longer due to model initialization. Subsequent generations will be faster.
</Note>
</Step>
<Step title="Play & Download">
- Click **Play** to preview the audio
- Click **Download** to save the audio file
- The generation is also saved to your **History**
</Step>
</Steps>
## Step 3: Build a Story (Optional)
The Stories Editor lets you create multi-voice narratives with a timeline-based interface.
<Steps>
<Step title="Create New Story">
Navigate to **Stories** and click **+ New Story**
</Step>
<Step title="Add Voice Tracks">
Click **+ Add Track** to create tracks for different speakers
</Step>
<Step title="Add Audio Clips">
- Drag generated audio from your History
- Or generate new clips directly in the timeline
- Arrange clips on the timeline
</Step>
<Step title="Edit & Export">
- Trim clips by dragging edges
- Adjust timing and spacing
- Click **Export** to render the final audio
</Step>
</Steps>
## What's Next?
<CardGroup cols={2}>
<Card title="Voice Cloning Guide" icon="microphone" href="/guides/creating-voice-profiles">
Learn advanced techniques for high-quality voice cloning
</Card>
<Card title="API Integration" icon="code" href="/api/overview">
Integrate Voicebox into your own applications
</Card>
<Card title="Stories Editor" icon="film" href="/overview/stories-editor">
Master the multi-track timeline editor
</Card>
<Card title="Remote Mode" icon="server" href="/overview/remote-mode">
Connect to a GPU server for faster generation
</Card>
</CardGroup>
## Tips for Success
<AccordionGroup>
<Accordion title="Getting the Best Voice Quality">
- Use 10-30 seconds of clear, consistent speech
- Avoid background noise and echo
- Multiple samples from the same speaker improve quality
- Match the speaking style you want to generate
</Accordion>
<Accordion title="Improving Generation Speed">
- Use a CUDA-capable GPU for 5-10x faster generation
- Enable voice prompt caching for repeated generations
- Consider running the backend on a remote GPU server
</Accordion>
<Accordion title="Troubleshooting Common Issues">
- **Server won't start:** Check if port 17493 is available
- **Poor audio quality:** Try adding more voice samples
- **Slow generation:** Verify GPU acceleration is enabled
- See the full [Troubleshooting Guide](/guides/troubleshooting) for more
</Accordion>
</AccordionGroup>
docs/overview/recording-transcription.mdx
+64
View File
@@ -0,0 +1,64 @@
---
title: "Recording & Transcription"
description: "Record audio and transcribe speech with Whisper"
---
## Recording
Voicebox includes built-in recording capabilities for creating voice samples and capturing audio.
### Features
- **Microphone input** - Record from any audio input device
- **System audio capture** - Record desktop audio (macOS/Windows)
- **Waveform visualization** - See audio levels in real-time
- **Multiple formats** - Export as WAV, MP3, or M4A
### How to Record
<Steps>
<Step title="Select Input">
Choose your microphone or system audio
</Step>
<Step title="Start Recording">
Click the record button and speak clearly
</Step>
<Step title="Stop & Save">
Click stop when finished
</Step>
<Step title="Use or Export">
Use as voice sample or export to file
</Step>
</Steps>
## Transcription
Automatic speech-to-text powered by OpenAI's Whisper model.
### Features
- **High accuracy** - Industry-leading speech recognition
- **Multiple languages** - Supports 50+ languages
- **Automatic detection** - Language auto-detection
- **Timestamps** - Word-level timing information
### How to Transcribe
<Steps>
<Step title="Select Audio">
Choose a recording or upload an audio file
</Step>
<Step title="Choose Language">
Select language or use auto-detect
</Step>
<Step title="Transcribe">
Click transcribe and wait for processing
</Step>
<Step title="Review & Export">
Review text and export as needed
</Step>
</Steps>
<Tip>
Transcription is useful for creating voice samples from existing audio or generating subtitles.
</Tip>
docs/overview/remote-mode.mdx
+138
View File
@@ -0,0 +1,138 @@
---
title: "Remote Mode"
description: "Connect to a GPU server for faster generation"
---
## Overview
Remote Mode allows you to run the Voicebox backend on a separate machine (like a GPU server) while using the desktop app on your local machine.
## Use Cases
- **No local GPU** - Use a cloud GPU or remote workstation
- **Faster generation** - Leverage powerful remote hardware
- **Shared infrastructure** - Multiple users connect to one server
- **Laptop workflows** - Keep your laptop cool and battery-efficient
## Architecture
In Remote Mode, the Voicebox desktop app (running on your local machine) communicates with the backend server (running on a remote machine) via HTTP. The local app provides only the user interface, while the remote server handles all the heavy processing including the TTS models, API endpoints, and audio generation.
## Setting Up Remote Mode
### On the Server
<Steps>
<Step title="Install Dependencies">
```bash
# Clone the repo
git clone https://github.com/jamiepine/voicebox.git
cd voicebox/backend
# Install Python dependencies
pip install -r requirements.txt
pip install git+https://github.com/QwenLM/Qwen3-TTS.git
```
</Step>
<Step title="Start the Server">
```bash
# Allow external connections
uvicorn main:app --host 0.0.0.0 --port 17493
```
<Warning>
This exposes the server to your network. Use a firewall or VPN for security.
</Warning>
</Step>
<Step title="Open Firewall">
```bash
# Ubuntu/Debian
sudo ufw allow 17493
# Or use your cloud provider's firewall settings
```
</Step>
</Steps>
### On the Client
<Steps>
<Step title="Open Settings">
In Voicebox, go to **Settings → Server**
</Step>
<Step title="Enable Remote Mode">
Toggle **Use Remote Server**
</Step>
<Step title="Enter Server URL">
```
http://<server-ip>:17493
```
Replace `<server-ip>` with your server's IP address
</Step>
<Step title="Test Connection">
Click **Test Connection** to verify
</Step>
</Steps>
## Cloud Deployment
### AWS EC2
```bash
# Launch a GPU instance (e.g., g4dn.xlarge)
# Install dependencies
# Start server with --host 0.0.0.0
```
### Vast.ai
```bash
# Rent a GPU instance
# SSH in and clone repo
# Start server
```
### RunPod
```bash
# Deploy a pod with CUDA support
# Install Voicebox backend
# Expose port 17493
```
## Security Considerations
<Warning>
The API currently has no authentication. Only use on trusted networks or with a VPN.
</Warning>
**Best Practices:**
- Use a VPN (WireGuard, Tailscale) instead of exposing to the internet
- Run behind a reverse proxy with authentication (nginx + basic auth)
- Use HTTPS with SSL certificates
- Firewall rules to limit access to specific IPs
## Performance
Expected performance on various GPUs:
| GPU | Generation Speed |
|-----|------------------|
| RTX 4090 | ~2-3s per 10 words |
| RTX 3090 | ~3-4s per 10 words |
| RTX 3060 | ~5-7s per 10 words |
| CPU (12-core) | ~20-30s per 10 words |
<Tip>
A GPU with 8GB+ VRAM is recommended for best performance.
</Tip>
## Troubleshooting
See the [Troubleshooting Guide](/guides/troubleshooting#remote-mode-issues) for common remote mode issues.
docs/overview/stories-editor.mdx
+64
View File
@@ -0,0 +1,64 @@
---
title: "Stories Editor"
description: "Create multi-voice narratives with a timeline-based editor"
---
## Overview
The Stories Editor is a DAW-like timeline interface for creating multi-voice narratives, podcasts, and conversations.
## Features
<CardGroup cols={2}>
<Card title="Multi-Track Timeline" icon="timeline">
Arrange multiple voice tracks in parallel
</Card>
<Card title="Inline Editing" icon="scissors">
Trim and split clips directly in the timeline
</Card>
<Card title="Auto-Playback" icon="play">
Preview with synchronized playhead
</Card>
<Card title="Voice Mixing" icon="users">
Build conversations with multiple speakers
</Card>
</CardGroup>
## Creating a Story
<Steps>
<Step title="Create New Story">
Navigate to **Stories** and click **+ New Story**
</Step>
<Step title="Add Tracks">
Create separate tracks for each voice/speaker
</Step>
<Step title="Add Clips">
- Drag from generation history
- Generate new clips inline
- Upload audio files
</Step>
<Step title="Arrange & Edit">
- Position clips on timeline
- Trim clip edges
- Adjust spacing and timing
</Step>
<Step title="Export">
Render the final mixed audio
</Step>
</Steps>
## Use Cases
- **Podcasts**: Multi-host conversations
- **Audiobooks**: Narrator + character voices
- **Game Dialogue**: Character interactions
- **Video Voiceovers**: Multiple speakers
- **Audio Drama**: Full voice casts
## Coming Soon
- Word-level editing
- Crossfades and transitions
- Audio effects (reverb, EQ)
- Real-time collaboration
docs/overview/troubleshooting.mdx
+477
View File
@@ -0,0 +1,477 @@
---
title: "Troubleshooting"
description: "Common issues and solutions for Voicebox"
---
This guide covers common issues you might encounter when using or developing Voicebox, along with solutions.
## Installation Issues
### macOS: "App is damaged and can't be opened"
This occurs because the app isn't signed with an Apple Developer certificate.
**Solution:**
```bash
# Remove the quarantine attribute
xattr -cr /Applications/Voicebox.app
```
### Windows: SmartScreen Warning
Windows SmartScreen may warn that the app is unrecognized.
**Solution:**
- Click "More info"
- Click "Run anyway"
<Note>
This is expected for unsigned applications. We're working on code signing for future releases.
</Note>
## Server Issues
### Backend Server Won't Start
**Symptoms:**
- Red status indicator in bottom-left corner
- "Failed to connect to server" error
**Solutions:**
<AccordionGroup>
<Accordion title="Port Already in Use">
Check if port 17493 is already in use:
```bash
# macOS/Linux
lsof -i :17493
# Windows
netstat -ano | findstr :17493
```
Kill the process using the port:
```bash
# macOS/Linux
kill -9 <PID>
# Windows
taskkill /PID <PID> /F
```
</Accordion>
<Accordion title="Permission Issues">
The server binary might not have execute permissions:
```bash
# macOS/Linux
chmod +x ~/Library/Application\ Support/com.voicebox.app/backend/voicebox-server
```
</Accordion>
<Accordion title="Check Logs">
View server logs for errors:
**macOS:**
```bash
tail -f ~/Library/Application\ Support/com.voicebox.app/logs/server.log
```
**Windows:**
```bash
type %APPDATA%\com.voicebox.app\logs\server.log
```
</Accordion>
</AccordionGroup>
### Connection Timeout
**Symptoms:**
- Long loading times
- "Connection timeout" errors
**Solution:**
- Restart the app
- Check your firewall settings
- Ensure localhost is accessible
## Generation Issues
### First Generation is Very Slow
**Symptoms:**
- First generation takes 2-5 minutes
- Progress indicator stuck at "Loading model..."
**Explanation:**
This is expected behavior. The first generation downloads the Qwen3-TTS model (~2-4GB) and initializes it.
**Solution:**
- Wait for the initial download to complete
- Subsequent generations will be much faster
- Check your internet connection
### Poor Voice Quality
**Symptoms:**
- Robotic or unnatural voice
- Missing emotion or prosody
- Pronunciation errors
**Solutions:**
<Steps>
<Step title="Improve Voice Samples">
- Use 10-30 seconds of clear audio
- Avoid background noise
- Ensure consistent speaking tone
- Add multiple samples from the same speaker
</Step>
<Step title="Match Speaking Style">
The generated voice will mimic the tone and style of your samples. If your sample is monotone, the generation will be too.
</Step>
<Step title="Adjust Text Formatting">
- Use proper punctuation
- Add commas for natural pauses
- Capitalize proper nouns
</Step>
</Steps>
### Generation Fails with "Out of Memory"
**Symptoms:**
- Generation crashes
- "CUDA out of memory" or "RuntimeError: out of memory"
**Solutions:**
<AccordionGroup>
<Accordion title="Free GPU Memory">
Close other GPU-intensive applications:
- Games
- Video editors
- Multiple browser tabs with WebGL
Then restart Voicebox.
</Accordion>
<Accordion title="Use CPU Mode">
If your GPU doesn't have enough VRAM (need 6GB+), use CPU mode:
Settings → Generation → Use CPU instead of GPU
<Warning>
CPU generation is 5-10x slower but uses system RAM instead of VRAM.
</Warning>
</Accordion>
<Accordion title="Reduce Batch Size">
For long text, split it into smaller chunks instead of generating all at once.
</Accordion>
</AccordionGroup>
## Audio Issues
### No Audio Playback
**Symptoms:**
- Generated audio won't play
- Playback button doesn't respond
**Solutions:**
- Check system audio settings
- Ensure audio output device is connected
- Try exporting and playing in a media player
### Crackling or Distorted Audio
**Symptoms:**
- Audio has static or distortion
- Clipping sounds
**Solutions:**
- Check if your input samples have distortion
- Reduce playback volume
- Re-generate with cleaner voice samples
## Development Issues
### Backend Won't Start in Dev Mode
**Symptoms:**
- `bun run dev:server` fails
- Import errors or module not found
**Solutions:**
<AccordionGroup>
<Accordion title="Python Version">
Ensure Python 3.11 or higher:
```bash
python --version
```
If not, install Python 3.11+ and recreate the virtual environment.
</Accordion>
<Accordion title="Virtual Environment">
Ensure venv is activated:
```bash
# macOS/Linux
source backend/venv/bin/activate
# Windows
backend\venv\Scripts\activate
```
You should see `(venv)` in your prompt.
</Accordion>
<Accordion title="Dependencies">
Reinstall dependencies:
```bash
cd backend
pip install -r requirements.txt
pip install git+https://github.com/QwenLM/Qwen3-TTS.git
```
</Accordion>
</AccordionGroup>
### Tauri Build Fails
**Symptoms:**
- `bun run tauri build` fails
- Rust compilation errors
**Solutions:**
```bash
# Clean build artifacts
cd tauri/src-tauri
cargo clean
# Update Rust
rustup update
# Try building again
cd ../..
bun run tauri build
```
### OpenAPI Client Generation Fails
**Symptoms:**
- `./scripts/generate-api.sh` fails
- "Failed to fetch schema" error
**Solutions:**
<Steps>
<Step title="Ensure Backend is Running">
```bash
curl http://localhost:17493/openapi.json
```
Should return JSON. If not, start the backend.
</Step>
<Step title="Check Port">
Ensure nothing else is using port 17493
</Step>
<Step title="Regenerate Manually">
```bash
cd backend
source venv/bin/activate
uvicorn main:app --reload --port 17493
# In another terminal
./scripts/generate-api.sh
```
</Step>
</Steps>
## Database Issues
### "Database is locked" Error
**Symptoms:**
- Profile or generation operations fail
- SQLite lock errors
**Solutions:**
- Close all Voicebox instances
- Delete the lock file:
```bash
# macOS
rm ~/Library/Application\ Support/com.voicebox.app/data/voicebox.db-shm
rm ~/Library/Application\ Support/com.voicebox.app/data/voicebox.db-wal
```
### Corrupted Database
**Symptoms:**
- App crashes on launch
- Data missing or corrupted
**Solutions:**
<Warning>
This will delete all your voice profiles and generation history. Export important profiles first if possible.
</Warning>
```bash
# macOS
rm ~/Library/Application\ Support/com.voicebox.app/data/voicebox.db
# Windows
del %APPDATA%\com.voicebox.app\data\voicebox.db
```
Restart the app to create a fresh database.
## Model Issues
### Model Download Fails
**Symptoms:**
- "Failed to download model" error
- Stuck at "Downloading..."
**Solutions:**
- Check your internet connection
- Check HuggingFace Hub status
- Try using a VPN if HuggingFace is blocked in your region
- Manually download and place in cache directory
### Wrong Model Version
**Symptoms:**
- Generation quality suddenly degraded
- Different voice output
**Solutions:**
Clear the model cache and re-download:
```bash
# macOS
rm -rf ~/.cache/huggingface/hub/models--Qwen*
# Windows
rmdir /s %USERPROFILE%\.cache\huggingface\hub\models--Qwen*
```
## Performance Issues
### Slow Generation on GPU
**Symptoms:**
- Generation slower than expected
- GPU not being utilized
**Solutions:**
<AccordionGroup>
<Accordion title="Verify CUDA Installation">
```bash
nvidia-smi
```
Should show your GPU. If not, install CUDA drivers.
</Accordion>
<Accordion title="Check GPU Selection">
If you have multiple GPUs, ensure Voicebox is using the right one.
Settings → Generation → GPU Device
</Accordion>
<Accordion title="Update GPU Drivers">
Outdated drivers can cause performance issues. Update to the latest NVIDIA drivers.
</Accordion>
</AccordionGroup>
### High Memory Usage
**Symptoms:**
- App uses excessive RAM
- System becomes sluggish
**Solutions:**
- Close unused voice profiles
- Clear generation history
- Restart the app periodically
## Remote Mode Issues
### Can't Connect to Remote Server
**Symptoms:**
- "Connection refused" error
- Remote server not found
**Solutions:**
<Steps>
<Step title="Check Server Status">
Ensure the remote server is running:
```bash
curl http://<server-ip>:17493/health
```
</Step>
<Step title="Check Firewall">
Ensure port 17493 is open on the remote server:
```bash
# Allow port on Ubuntu/Debian
sudo ufw allow 17493
```
</Step>
<Step title="Verify Network">
- Ensure both machines are on the same network (for local servers)
- Use IP address instead of hostname
- Try pinging the server: `ping <server-ip>`
</Step>
</Steps>
## Still Having Issues?
If you're still experiencing problems:
1. **Check GitHub Issues:** [github.com/jamiepine/voicebox/issues](https://github.com/jamiepine/voicebox/issues)
2. **Open a New Issue:** Provide:
- Operating system and version
- Voicebox version
- Steps to reproduce
- Error messages or logs
3. **Join Discord:** [discord.gg/voicebox](https://discord.gg/voicebox) (coming soon)
## Diagnostic Information
When reporting issues, include this information:
```bash
# Voicebox version
# Check Help → About in the app
# Operating system
uname -a # macOS/Linux
systeminfo # Windows
# Python version (for dev issues)
python --version
# GPU info (if generation issues)
nvidia-smi # NVIDIA GPUs
```
For more detailed troubleshooting, see the [TROUBLESHOOTING.md](https://github.com/jamiepine/voicebox/blob/main/docs/TROUBLESHOOTING.md) file in the repository.
docs/overview/voice-cloning.mdx
+75
View File
@@ -0,0 +1,75 @@
---
title: "Voice Cloning"
description: "Clone any voice from just a few seconds of audio"
---
## Overview
Voicebox uses **Qwen3-TTS** from Alibaba to achieve near-perfect voice cloning from just a few seconds of audio. The model captures prosody, emotion, and natural cadence.
## How It Works
<Steps>
<Step title="Upload or Record Sample">
Provide 10-30 seconds of clear speech from the target voice
</Step>
<Step title="Model Analysis">
Qwen3-TTS analyzes vocal characteristics, tone, and speaking patterns
</Step>
<Step title="Voice Profile Created">
The model generates a voice embedding for synthesis
</Step>
<Step title="Generate Speech">
Use the profile to generate any text in the cloned voice
</Step>
</Steps>
## Best Practices
### Sample Quality
<CardGroup cols={2}>
<Card title="Do" icon="check">
- Use 10-30 seconds of audio
- Clear, consistent speaking
- Minimal background noise
- Natural speaking pace
</Card>
<Card title="Don't" icon="xmark">
- Very short clips (< 5 seconds)
- Heavy background noise
- Music or overlapping voices
- Heavily processed audio
</Card>
</CardGroup>
### Multiple Samples
Adding multiple samples from the same speaker can improve quality:
- Different speaking styles (casual, formal)
- Different emotions (happy, serious)
- Different recording conditions
<Tip>
The model will learn a more robust representation from diverse samples.
</Tip>
## Supported Languages
Currently supported:
- English
- Chinese (Mandarin)
More languages coming soon.
## Limitations
<Warning>
Voice cloning should only be used with consent. Ensure you have permission to clone someone's voice.
</Warning>
- Quality depends on sample clarity
- Works best with consistent speaking tone
- May struggle with extreme accents or speech impediments
- Background noise reduces quality
docs/package.json
+15
View File
@@ -0,0 +1,15 @@
{
"name": "voicebox-docs",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "mintlify dev",
"install:mintlify": "bun add -g mintlify"
},
"devDependencies": {
"mintlify": "latest"
},
"engines": {
"bun": ">=1.0.0"
}
}
docs/plans/OPENAI_SUPPORT.md
+235
View File
@@ -0,0 +1,235 @@
# OpenAI API Compatibility
**Status:** Planned for v0.2.0
**Issue:** [#10 OpenAI API compatibility](https://github.com/jamiepine/voicebox/issues/10)
## Overview
This feature exposes OpenAI-compatible endpoints from Voicebox, allowing any tool, library, or application that speaks the OpenAI Audio API to use Voicebox as a drop-in local replacement.
```mermaid
flowchart LR
subgraph clients [External Clients]
SDK[OpenAI SDK]
Curl[curl / HTTP]
Apps[Third-party Apps]
end
subgraph voicebox [Voicebox Server]
OpenAI["/v1/audio/* endpoints"]
TTS[TTSModel]
Whisper[WhisperModel]
Profiles[Voice Profiles]
end
SDK --> OpenAI
Curl --> OpenAI
Apps --> OpenAI
OpenAI --> TTS
OpenAI --> Whisper
OpenAI --> Profiles
```
## Use Cases
- **OpenAI SDK users**: `openai.audio.speech.create()` works with Voicebox
- **LLM frameworks**: LangChain, AutoGen, etc. can use Voicebox for TTS
- **Shell scripts**: `curl` commands copy-pasted from OpenAI docs work
- **Existing integrations**: Any tool expecting OpenAI's API works without code changes
## Endpoints to Implement
### 1. `POST /v1/audio/speech` (TTS)
OpenAI spec: https://platform.openai.com/docs/api-reference/audio/createSpeech
**Request:**
```json
{
"model": "tts-1",
"input": "Hello world!",
"voice": "alloy",
"response_format": "mp3",
"speed": 1.0
}
```
**Response:** Audio file (mp3, wav, opus, aac, flac, pcm)
**Voice Mapping Strategy:**
- `voice` parameter maps to Voicebox profile names (case-insensitive)
- If no match, use a configurable default profile
- Support special syntax: `voice: "profile:uuid"` for explicit profile ID
### 2. `POST /v1/audio/transcriptions` (Whisper)
OpenAI spec: https://platform.openai.com/docs/api-reference/audio/createTranscription
**Request:** (multipart/form-data)
- `file`: Audio file
- `model`: "whisper-1"
- `language`: Optional language hint
- `response_format`: json, text, srt, verbose_json, vtt
**Response:**
```json
{
"text": "Hello world!"
}
```
## Implementation Details
### New File: `backend/openai_compat.py`
Create a dedicated module with an APIRouter for OpenAI-compatible endpoints:
```python
from fastapi import APIRouter, UploadFile, File, Form, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from typing import Literal, Optional
router = APIRouter(prefix="/v1/audio", tags=["OpenAI Compatible"])
class SpeechRequest(BaseModel):
model: str = "tts-1"
input: str
voice: str = "alloy"
response_format: Literal["mp3", "wav", "opus", "aac", "flac", "pcm"] = "mp3"
speed: float = 1.0
@router.post("/speech")
async def create_speech(request: SpeechRequest, db: Session = Depends(get_db)):
# 1. Map voice name to profile
# 2. Generate audio using existing TTSModel
# 3. Convert to requested format
# 4. Return audio stream
...
@router.post("/transcriptions")
async def create_transcription(
file: UploadFile = File(...),
model: str = Form("whisper-1"),
language: Optional[str] = Form(None),
response_format: str = Form("json"),
):
# 1. Save uploaded file
# 2. Transcribe using existing WhisperModel
# 3. Return in requested format
...
```
### Voice Profile Resolution
Add helper in [backend/profiles.py](backend/profiles.py):
```python
async def resolve_voice_for_openai(voice: str, db: Session) -> Optional[VoiceProfile]:
"""
Resolve OpenAI voice parameter to a Voicebox profile.
Priority:
1. Exact profile name match (case-insensitive)
2. Profile ID match (if voice starts with "profile:")
3. Default profile from config
4. First available profile
"""
...
```
### Audio Format Conversion
Add conversion utilities in [backend/utils/audio.py](backend/utils/audio.py):
```python
def convert_audio_format(
audio: np.ndarray,
sample_rate: int,
target_format: str, # mp3, wav, opus, aac, flac, pcm
) -> bytes:
"""Convert audio to target format using ffmpeg or pydub."""
...
```
### Configuration
Add to [backend/config.py](backend/config.py):
```python
# OpenAI API Compatibility
OPENAI_COMPAT_ENABLED = True
OPENAI_COMPAT_DEFAULT_VOICE = None # Profile ID or name for default voice
OPENAI_COMPAT_REQUIRE_AUTH = False # Require API key validation
OPENAI_COMPAT_API_KEY = None # If set, validate against this
```
### Integration with main.py
In [backend/main.py](backend/main.py), include the router:
```python
from . import openai_compat
# Add OpenAI-compatible routes
if config.OPENAI_COMPAT_ENABLED:
app.include_router(openai_compat.router)
```
## Streaming Support (Future Enhancement)
Initial implementation returns complete audio. Streaming can be added later:
```python
@router.post("/speech")
async def create_speech(request: SpeechRequest):
if request.stream:
return StreamingResponse(
generate_audio_chunks(request),
media_type=f"audio/{request.response_format}"
)
...
```
## Testing
Example usage after implementation:
```bash
# TTS with curl
curl http://localhost:8000/v1/audio/speech \
-H "Content-Type: application/json" \
-d '{"model": "tts-1", "input": "Hello!", "voice": "MyProfile"}' \
--output speech.mp3
# With OpenAI Python SDK
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8000/v1", api_key="unused")
response = client.audio.speech.create(
model="tts-1",
voice="MyProfile",
input="Hello world!"
)
response.stream_to_file("output.mp3")
# Transcription
curl http://localhost:8000/v1/audio/transcriptions \
-F file=@audio.mp3 \
-F model="whisper-1"
```
## Security Considerations
- Optional API key validation (for shared deployments)
- Rate limiting on endpoints
- Input length limits (same as existing `/generate` endpoint)
## Dependencies
- `pydub` or `ffmpeg-python` for audio format conversion (mp3, opus, etc.)
- No changes to existing TTS/Whisper model code
docs/public/app-screenshot-1.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 134 KiB

docs/public/app-screenshot-2.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 129 KiB

docs/public/app-screenshot-3.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 108 KiB

package.json
+6 -1
View File
@@ -22,6 +22,7 @@
"generate:keys": "cd tauri && bun tauri signer generate -w ~/.tauri/voicebox.key", "generate:keys": "cd tauri && bun tauri signer generate -w ~/.tauri/voicebox.key",
"build:server": "./scripts/build-server.sh", "build:server": "./scripts/build-server.sh",
"update:icons": "./scripts/update-icons.sh", "update:icons": "./scripts/update-icons.sh",
"convert:assets": "./scripts/convert-assets.sh",
"lint": "biome lint .", "lint": "biome lint .",
"lint:fix": "biome lint --write .", "lint:fix": "biome lint --write .",
"format": "biome format --write .", "format": "biome format --write .",
@@ -35,5 +36,9 @@
"@types/node": "^20.0.0", "@types/node": "^20.0.0",
"tailwindcss": "^4.1.18", "tailwindcss": "^4.1.18",
"typescript": "^5.6.0" "typescript": "^5.6.0"
} },
"engines": {
"bun": ">=1.0.0"
},
"packageManager": "[email protected]"
} }
scripts/convert-assets.sh
Executable
+154
View File
@@ -0,0 +1,154 @@
#!/bin/bash
set -e
# Asset Conversion Script
# Converts PNG → WebP and MOV → WebM in public folders
# Deletes original files after successful conversion
cd "$(dirname "$0")/.."
# Directories to process
DIRS=(
"landing/public"
"docs/public"
)
# Track counts
png_converted=0
mov_converted=0
png_failed=0
mov_failed=0
echo "🔄 Converting assets to web-optimized formats..."
echo ""
# Check for required tools
check_dependencies() {
local missing=()
if ! command -v cwebp &> /dev/null && ! command -v ffmpeg &> /dev/null; then
missing+=("cwebp or ffmpeg (for PNG→WebP)")
fi
if ! command -v ffmpeg &> /dev/null; then
missing+=("ffmpeg (for MOV→WebM)")
fi
if [ ${#missing[@]} -ne 0 ]; then
echo "❌ Missing required tools:"
for tool in "${missing[@]}"; do
echo " - $tool"
done
echo ""
echo "Install with: brew install webp ffmpeg"
exit 1
fi
}
# Convert PNG to WebP
convert_png() {
local input="$1"
local output="${input%.png}.webp"
echo " Converting: $input"
if command -v cwebp &> /dev/null; then
# Use cwebp for best quality/size ratio
if cwebp -q 90 "$input" -o "$output" 2>/dev/null; then
rm "$input"
echo " ✓ → $output"
return 0
fi
elif command -v ffmpeg &> /dev/null; then
# Fallback to ffmpeg
if ffmpeg -i "$input" -c:v libwebp -quality 90 "$output" -y 2>/dev/null; then
rm "$input"
echo " ✓ → $output"
return 0
fi
fi
echo " ✗ Failed to convert"
return 1
}
# Convert MOV to WebM
convert_mov() {
local input="$1"
local output="${input%.mov}.webm"
echo " Converting: $input"
if ffmpeg -i "$input" -c:v libvpx-vp9 -crf 30 -b:v 0 -c:a libopus "$output" -y 2>/dev/null; then
rm "$input"
echo " ✓ → $output"
return 0
fi
echo " ✗ Failed to convert"
return 1
}
# Main execution
check_dependencies
for dir in "${DIRS[@]}"; do
if [ ! -d "$dir" ]; then
echo "⚠ Directory not found: $dir (skipping)"
continue
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "📁 Processing: $dir"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# Find and convert PNGs (recursively)
while IFS= read -r -d '' file; do
if convert_png "$file"; then
((png_converted++))
else
((png_failed++))
fi
done < <(find "$dir" -type f -name "*.png" -print0 2>/dev/null)
# Find and convert MOVs (recursively)
while IFS= read -r -d '' file; do
if convert_mov "$file"; then
((mov_converted++))
else
((mov_failed++))
fi
done < <(find "$dir" -type f -name "*.mov" -print0 2>/dev/null)
# Also check for uppercase extensions
while IFS= read -r -d '' file; do
if convert_png "$file"; then
((png_converted++))
else
((png_failed++))
fi
done < <(find "$dir" -type f -name "*.PNG" -print0 2>/dev/null)
while IFS= read -r -d '' file; do
if convert_mov "$file"; then
((mov_converted++))
else
((mov_failed++))
fi
done < <(find "$dir" -type f -name "*.MOV" -print0 2>/dev/null)
echo ""
done
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "✅ Conversion complete!"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "Results:"
echo " PNG → WebP: $png_converted converted, $png_failed failed"
echo " MOV → WebM: $mov_converted converted, $mov_failed failed"
if [ $png_converted -eq 0 ] && [ $mov_converted -eq 0 ]; then
echo ""
echo "No PNG or MOV files found to convert."
fi
tauri/src-tauri/Cargo.lock
Generated
+1 -1
View File
@@ -5041,7 +5041,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]] [[package]]
name = "voicebox" name = "voicebox"
version = "0.1.8" version = "0.1.9"
dependencies = [ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"core-foundation-sys", "core-foundation-sys",
tauri/src-tauri/gen/Assets.car
BIN
View File
Binary file not shown.
tauri/src-tauri/src/main.rs
-3
View File
@@ -635,8 +635,6 @@ pub fn run() {
} }
} }
#[cfg(debug_assertions)]
{
// Get all windows and open devtools on the first one // Get all windows and open devtools on the first one
if let Some((_, window)) = app.webview_windows().iter().next() { if let Some((_, window)) = app.webview_windows().iter().next() {
window.open_devtools(); window.open_devtools();
@@ -644,7 +642,6 @@ pub fn run() {
} else { } else {
println!("No window found to open dev tools"); println!("No window found to open dev tools");
} }
}
Ok(()) Ok(())
}) })
.invoke_handler(tauri::generate_handler![ .invoke_handler(tauri::generate_handler![