macos audio capture for sample creation

This commit is contained in:
Jamie Pine
2026-01-26 16:05:42 -08:00
parent c7c401b98c
commit 446182e16c
13 changed files with 1187 additions and 24 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ import { UpdateNotification } from '@/components/UpdateNotification';
import { Toaster } from '@/components/ui/toaster';
import { ProfileList } from '@/components/VoiceProfiles/ProfileList';
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
import { useRestoreActiveTasks, MODEL_DISPLAY_NAMES } from '@/lib/hooks/useRestoreActiveTasks';
import { MODEL_DISPLAY_NAMES, useRestoreActiveTasks } from '@/lib/hooks/useRestoreActiveTasks';
import { isMacOS, isTauri, setupWindowCloseHandler, startServer } from '@/lib/tauri';
// Track if server is starting to prevent duplicate starts
@@ -39,9 +39,11 @@ import {
} from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
import { useUIStore } from '@/stores/uiStore';
import { Mic, Square, Upload } from 'lucide-react';
import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
import { isTauri } from '@/lib/tauri';
// Helper function to get audio duration from File
async function getAudioDuration(file: File): Promise<number> {
@@ -101,7 +103,7 @@ export function ProfileForm() {
const addSample = useAddSample();
const transcribe = useTranscription();
const { toast } = useToast();
const [sampleMode, setSampleMode] = useState<'upload' | 'record'>('upload');
const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('upload');
const [audioDuration, setAudioDuration] = useState<number | null>(null);
const [isValidatingAudio, setIsValidatingAudio] = useState(false);
const isCreating = !editingProfileId;
@@ -171,6 +173,28 @@ export function ProfileForm() {
},
});
const {
isRecording: isSystemRecording,
duration: systemDuration,
error: systemRecordingError,
isSupported: isSystemAudioSupported,
startRecording: startSystemRecording,
stopRecording: stopSystemRecording,
cancelRecording: cancelSystemRecording,
} = useSystemAudioCapture({
maxDurationSeconds: 30,
onRecordingComplete: (blob) => {
const file = new File([blob], `system-audio-${Date.now()}.wav`, {
type: blob.type || 'audio/wav',
});
form.setValue('sampleFile', file, { shouldValidate: true });
toast({
title: 'System audio captured',
description: 'Audio has been captured successfully.',
});
},
});
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -182,6 +206,17 @@ export function ProfileForm() {
}
}, [recordingError, toast]);
// Show system audio recording errors
useEffect(() => {
if (systemRecordingError) {
toast({
title: 'System audio capture error',
description: systemRecordingError,
variant: 'destructive',
});
}
}, [systemRecordingError, toast]);
useEffect(() => {
if (editingProfile) {
form.reset({
@@ -234,7 +269,11 @@ export function ProfileForm() {
}
function handleCancelRecording() {
cancelRecording();
if (sampleMode === 'record') {
cancelRecording();
} else if (sampleMode === 'system') {
cancelSystemRecording();
}
form.resetField('sampleFile');
}
@@ -345,6 +384,9 @@ export function ProfileForm() {
if (isRecording) {
cancelRecording();
}
if (isSystemRecording) {
cancelSystemRecording();
}
}
}
@@ -432,9 +474,19 @@ export function ProfileForm() {
<Tabs
value={sampleMode}
onValueChange={(v) => setSampleMode(v as 'upload' | 'record')}
onValueChange={(v) => {
const newMode = v as 'upload' | 'record' | 'system';
// Cancel any active recordings when switching modes
if (isRecording && newMode !== 'record') {
cancelRecording();
}
if (isSystemRecording && newMode !== 'system') {
cancelSystemRecording();
}
setSampleMode(newMode);
}}
>
<TabsList className="grid w-full grid-cols-2">
<TabsList className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}>
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -443,6 +495,12 @@ export function ProfileForm() {
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
{isTauri() && isSystemAudioSupported && (
<TabsTrigger value="system" className="flex items-center gap-2">
<Monitor className="h-4 w-4" />
System Audio
</TabsTrigger>
)}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -611,6 +669,103 @@ export function ProfileForm() {
)}
/>
</TabsContent>
{isTauri() && isSystemAudioSupported && (
<TabsContent value="system" className="space-y-4">
<FormField
control={form.control}
name="sampleFile"
render={() => (
<FormItem>
<FormLabel>Capture System Audio</FormLabel>
<FormControl>
<div className="space-y-4">
{!isSystemRecording && !selectedFile && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-dashed rounded-lg">
<Button
type="button"
onClick={startSystemRecording}
size="lg"
className="flex items-center gap-2"
>
<Monitor className="h-5 w-5" />
Start Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capture audio from your system. Maximum duration: 30 seconds.
</p>
</div>
)}
{isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-destructive rounded-lg bg-destructive/5">
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
<span className="text-lg font-mono font-semibold">
{formatAudioDuration(systemDuration)}
</span>
</div>
</div>
<Button
type="button"
onClick={stopSystemRecording}
variant="destructive"
className="flex items-center gap-2"
>
<Square className="h-4 w-4" />
Stop Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
remaining)
</p>
</div>
)}
{selectedFile && !isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-primary rounded-lg bg-primary/5">
<div className="flex items-center gap-2">
<Monitor className="h-5 w-5 text-primary" />
<span className="font-medium">Capture complete</span>
</div>
<p className="text-sm text-muted-foreground text-center">
File: {selectedFile.name}
</p>
<div className="flex gap-2">
<Button
type="button"
variant="outline"
onClick={handleTranscribe}
disabled={transcribe.isPending}
className="flex items-center gap-2"
>
<Mic className="h-4 w-4" />
{transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
</Button>
<Button
type="button"
variant="outline"
onClick={handleCancelRecording}
className="flex items-center gap-2"
>
Capture Again
</Button>
</div>
</div>
)}
</div>
</FormControl>
<FormDescription>
Capture audio from your system (speakers, applications). Maximum duration is 30
seconds.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
</TabsContent>
)}
</Tabs>
<FormField
@@ -26,8 +26,10 @@ import { useToast } from '@/components/ui/use-toast';
import { useAddSample, useProfile } from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
import { Mic, Square, Upload } from 'lucide-react';
import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
import { isTauri } from '@/lib/tauri';
const sampleSchema = z.object({
file: z.instanceof(File, { message: 'Please select an audio file' }),
@@ -50,7 +52,7 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
const transcribe = useTranscription();
const { data: profile } = useProfile(profileId);
const { toast } = useToast();
const [mode, setMode] = useState<'upload' | 'record'>('upload');
const [mode, setMode] = useState<'upload' | 'record' | 'system'>('upload');
const form = useForm<SampleFormValues>({
resolver: zodResolver(sampleSchema),
@@ -83,6 +85,29 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
},
});
const {
isRecording: isSystemRecording,
duration: systemDuration,
error: systemRecordingError,
isSupported: isSystemAudioSupported,
startRecording: startSystemRecording,
stopRecording: stopSystemRecording,
cancelRecording: cancelSystemRecording,
} = useSystemAudioCapture({
maxDurationSeconds: 30,
onRecordingComplete: (blob) => {
// Convert blob to File object
const file = new File([blob], `system-audio-${Date.now()}.wav`, {
type: blob.type || 'audio/wav',
});
form.setValue('file', file, { shouldValidate: true });
toast({
title: 'System audio captured',
description: 'Audio has been captured successfully.',
});
},
});
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -94,6 +119,17 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
}
}, [recordingError, toast]);
// Show system audio recording errors
useEffect(() => {
if (systemRecordingError) {
toast({
title: 'System audio capture error',
description: systemRecordingError,
variant: 'destructive',
});
}
}, [systemRecordingError, toast]);
async function handleTranscribe() {
const file = form.getValues('file');
if (!file) {
@@ -154,12 +190,19 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
if (isRecording) {
cancelRecording();
}
if (isSystemRecording) {
cancelSystemRecording();
}
}
onOpenChange(newOpen);
}
function handleCancelRecording() {
cancelRecording();
if (mode === 'record') {
cancelRecording();
} else if (mode === 'system') {
cancelSystemRecording();
}
// Reset file field by clearing the input
form.resetField('file');
}
@@ -176,8 +219,13 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<Tabs value={mode} onValueChange={(v) => setMode(v as 'upload' | 'record')}>
<TabsList className="grid w-full grid-cols-2">
<Tabs
value={mode}
onValueChange={(v) => setMode(v as 'upload' | 'record' | 'system')}
>
<TabsList
className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}
>
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -186,6 +234,12 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
{isTauri() && isSystemAudioSupported && (
<TabsTrigger value="system" className="flex items-center gap-2">
<Monitor className="h-4 w-4" />
System Audio
</TabsTrigger>
)}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -325,6 +379,103 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
)}
/>
</TabsContent>
{isTauri() && isSystemAudioSupported && (
<TabsContent value="system" className="space-y-4">
<FormField
control={form.control}
name="file"
render={() => (
<FormItem>
<FormLabel>Capture System Audio</FormLabel>
<FormControl>
<div className="space-y-4">
{!isSystemRecording && !selectedFile && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-dashed rounded-lg">
<Button
type="button"
onClick={startSystemRecording}
size="lg"
className="flex items-center gap-2"
>
<Monitor className="h-5 w-5" />
Start Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capture audio from your system. Maximum duration: 30 seconds.
</p>
</div>
)}
{isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-destructive rounded-lg bg-destructive/5">
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
<span className="text-lg font-mono font-semibold">
{formatAudioDuration(systemDuration)}
</span>
</div>
</div>
<Button
type="button"
onClick={stopSystemRecording}
variant="destructive"
className="flex items-center gap-2"
>
<Square className="h-4 w-4" />
Stop Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
remaining)
</p>
</div>
)}
{selectedFile && !isSystemRecording && mode === 'system' && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-primary rounded-lg bg-primary/5">
<div className="flex items-center gap-2">
<Monitor className="h-5 w-5 text-primary" />
<span className="font-medium">Capture complete</span>
</div>
<p className="text-sm text-muted-foreground">
File: {selectedFile.name}
</p>
<div className="flex gap-2">
<Button
type="button"
variant="outline"
onClick={handleTranscribe}
disabled={transcribe.isPending}
className="flex items-center gap-2"
>
<Mic className="h-4 w-4" />
{transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
</Button>
<Button
type="button"
variant="outline"
onClick={handleCancelRecording}
className="flex items-center gap-2"
>
Capture Again
</Button>
</div>
</div>
)}
</div>
</FormControl>
<FormDescription>
Capture audio from your system (applications, browser tabs, etc.). Works
natively without browser dialogs.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
</TabsContent>
)}
</Tabs>
<FormField
+165
View File
@@ -0,0 +1,165 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { isTauri } from '@/lib/tauri';
interface UseSystemAudioCaptureOptions {
maxDurationSeconds?: number;
onRecordingComplete?: (blob: Blob) => void;
}
/**
* Hook for native system audio capture using Tauri commands.
* Uses ScreenCaptureKit on macOS and WASAPI loopback on Windows.
*/
export function useSystemAudioCapture({
maxDurationSeconds = 30,
onRecordingComplete,
}: UseSystemAudioCaptureOptions = {}) {
const [isRecording, setIsRecording] = useState(false);
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [isSupported, setIsSupported] = useState(false);
const timerRef = useRef<number | null>(null);
const startTimeRef = useRef<number | null>(null);
const stopRecordingRef = useRef<(() => Promise<void>) | null>(null);
// Check if system audio capture is supported
useEffect(() => {
if (!isTauri()) {
setIsSupported(false);
return;
}
invoke<boolean>('is_system_audio_supported')
.then((supported) => {
setIsSupported(supported);
})
.catch(() => {
setIsSupported(false);
});
}, []);
const startRecording = useCallback(async () => {
if (!isTauri()) {
const errorMsg = 'System audio capture is only available in the desktop app.';
setError(errorMsg);
return;
}
if (!isSupported) {
const errorMsg = 'System audio capture is not supported on this platform.';
setError(errorMsg);
return;
}
try {
setError(null);
setDuration(0);
// Start native capture
await invoke('start_system_audio_capture', {
maxDurationSecs: maxDurationSeconds,
});
setIsRecording(true);
startTimeRef.current = Date.now();
// Start timer
timerRef.current = window.setInterval(() => {
if (startTimeRef.current) {
const elapsed = (Date.now() - startTimeRef.current) / 1000;
setDuration(elapsed);
// Auto-stop at max duration
if (elapsed >= maxDurationSeconds && stopRecordingRef.current) {
void stopRecordingRef.current();
}
}
}, 100);
} catch (err) {
const errorMessage =
err instanceof Error
? err.message
: 'Failed to start system audio capture. Please check permissions.';
setError(errorMessage);
setIsRecording(false);
}
}, [maxDurationSeconds, isSupported]);
const stopRecording = useCallback(async () => {
if (!isRecording || !isTauri()) {
return;
}
try {
setIsRecording(false);
if (timerRef.current !== null) {
clearInterval(timerRef.current);
timerRef.current = null;
}
// Stop capture and get base64 WAV data
const base64Data = await invoke<string>('stop_system_audio_capture');
// Convert base64 to Blob
const binaryString = atob(base64Data);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const blob = new Blob([bytes], { type: 'audio/wav' });
onRecordingComplete?.(blob);
} catch (err) {
const errorMessage =
err instanceof Error
? err.message
: 'Failed to stop system audio capture.';
setError(errorMessage);
}
}, [isRecording, onRecordingComplete]);
// Store stopRecording in ref for use in timer
useEffect(() => {
stopRecordingRef.current = stopRecording;
}, [stopRecording]);
const cancelRecording = useCallback(async () => {
if (isRecording) {
await stopRecording();
}
setIsRecording(false);
setDuration(0);
if (timerRef.current !== null) {
clearInterval(timerRef.current);
timerRef.current = null;
}
}, [isRecording, stopRecording]);
// Cleanup on unmount
useEffect(() => {
return () => {
if (timerRef.current !== null) {
clearInterval(timerRef.current);
}
// Cancel recording on unmount if still recording
if (isRecording) {
void cancelRecording();
}
};
// biome-ignore lint/correctness/useExhaustiveDependencies: cancelRecording is stable
}, [isRecording]);
return {
isRecording,
duration,
error,
isSupported,
startRecording,
stopRecording,
cancelRecording,
};
}
+210 -12
View File
@@ -103,6 +103,24 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bindgen"
version = "0.72.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [
"bitflags 2.10.0",
"cexpr",
"clang-sys",
"itertools",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn 2.0.114",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -267,6 +285,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfb"
version = "0.7.3"
@@ -312,6 +339,17 @@ dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "clang-sys"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading 0.8.9",
]
[[package]]
name = "combine"
version = "4.6.7"
@@ -378,6 +416,15 @@ dependencies = [
"libc",
]
[[package]]
name = "coreaudio-sys"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6"
dependencies = [
"bindgen",
]
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -646,6 +693,12 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "embed-resource"
version = "3.0.6"
@@ -1212,6 +1265,12 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hound"
version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
[[package]]
name = "html5ever"
version = "0.29.1"
@@ -1534,6 +1593,15 @@ dependencies = [
"once_cell",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.17"
@@ -1666,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf"
dependencies = [
"gtk-sys",
"libloading",
"libloading 0.7.4",
"once_cell",
]
@@ -1686,6 +1754,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"windows-link 0.2.1",
]
[[package]]
name = "libredox"
version = "0.1.12"
@@ -1736,6 +1814,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "malloc_buf"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
dependencies = [
"libc",
]
[[package]]
name = "markup5ever"
version = "0.14.1"
@@ -1788,6 +1875,12 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "minisign-verify"
version = "0.2.4"
@@ -1878,12 +1971,31 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "num-conv"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
[[package]]
name = "num-integer"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -1915,6 +2027,15 @@ dependencies = [
"syn 2.0.114",
]
[[package]]
name = "objc"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"
dependencies = [
"malloc_buf",
]
[[package]]
name = "objc2"
version = "0.6.3"
@@ -3016,6 +3137,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "screencapturekit"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ccf069cb109cf8e01ebdca0d55dfce45dbbf669e8c56ed5c62150b056d3ec9f"
[[package]]
name = "selectors"
version = "0.24.0"
@@ -3504,7 +3631,7 @@ dependencies = [
"tao-macros",
"unicode-segmentation",
"url",
"windows",
"windows 0.61.3",
"windows-core 0.61.2",
"windows-version",
"x11-dl",
@@ -3586,7 +3713,7 @@ dependencies = [
"webkit2gtk",
"webview2-com",
"window-vibrancy",
"windows",
"windows 0.61.3",
]
[[package]]
@@ -3784,7 +3911,7 @@ dependencies = [
"url",
"webkit2gtk",
"webview2-com",
"windows",
"windows 0.61.3",
]
[[package]]
@@ -3810,7 +3937,7 @@ dependencies = [
"url",
"webkit2gtk",
"webview2-com",
"windows",
"windows 0.61.3",
"wry",
]
@@ -4358,6 +4485,12 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
name = "voicebox"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"core-foundation-sys",
"coreaudio-sys",
"hound",
"objc",
"screencapturekit",
"serde",
"serde_json",
"tauri",
@@ -4367,6 +4500,7 @@ dependencies = [
"tauri-plugin-shell",
"tauri-plugin-updater",
"tokio",
"wasapi",
]
[[package]]
@@ -4408,6 +4542,19 @@ dependencies = [
"try-lock",
]
[[package]]
name = "wasapi"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7834ac561bea8a7413661fdda62180f9e054f99815269cd3572cf7e40d9c3191"
dependencies = [
"log",
"num-integer",
"thiserror 2.0.18",
"windows 0.62.2",
"windows-core 0.62.2",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
@@ -4582,7 +4729,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a"
dependencies = [
"webview2-com-macros",
"webview2-com-sys",
"windows",
"windows 0.61.3",
"windows-core 0.61.2",
"windows-implement",
"windows-interface",
@@ -4606,7 +4753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c"
dependencies = [
"thiserror 2.0.18",
"windows",
"windows 0.61.3",
"windows-core 0.61.2",
]
@@ -4662,11 +4809,23 @@ version = "0.61.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
dependencies = [
"windows-collections",
"windows-collections 0.2.0",
"windows-core 0.61.2",
"windows-future",
"windows-future 0.2.1",
"windows-link 0.1.3",
"windows-numerics",
"windows-numerics 0.2.0",
]
[[package]]
name = "windows"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
dependencies = [
"windows-collections 0.3.2",
"windows-core 0.62.2",
"windows-future 0.3.2",
"windows-numerics 0.3.1",
]
[[package]]
@@ -4678,6 +4837,15 @@ dependencies = [
"windows-core 0.61.2",
]
[[package]]
name = "windows-collections"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
dependencies = [
"windows-core 0.62.2",
]
[[package]]
name = "windows-core"
version = "0.61.2"
@@ -4712,7 +4880,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
dependencies = [
"windows-core 0.61.2",
"windows-link 0.1.3",
"windows-threading",
"windows-threading 0.1.0",
]
[[package]]
name = "windows-future"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
dependencies = [
"windows-core 0.62.2",
"windows-link 0.2.1",
"windows-threading 0.2.1",
]
[[package]]
@@ -4759,6 +4938,16 @@ dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-numerics"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
dependencies = [
"windows-core 0.62.2",
"windows-link 0.2.1",
]
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -4897,6 +5086,15 @@ dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-threading"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "windows-version"
version = "0.1.7"
@@ -5123,7 +5321,7 @@ dependencies = [
"webkit2gtk",
"webkit2gtk-sys",
"webview2-com",
"windows",
"windows 0.61.3",
"windows-core 0.61.2",
"windows-version",
"x11-dl",
+11
View File
@@ -20,6 +20,17 @@ tauri-plugin-shell = "2.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["full"] }
hound = "3.5"
base64 = "0.22"
[target.'cfg(target_os = "macos")'.dependencies]
screencapturekit = { version = "1", features = ["async"] }
coreaudio-sys = "0.2"
objc = "0.2"
core-foundation-sys = "0.8"
[target.'cfg(target_os = "windows")'.dependencies]
wasapi = "0.22"
[target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies]
tauri-plugin-updater = "2.0"
+2
View File
@@ -8,5 +8,7 @@
<string>voicebox</string>
<key>NSMicrophoneUsageDescription</key>
<string>voicebox needs microphone access to record voice samples for voice cloning.</string>
<key>NSScreenCaptureUsageDescription</key>
<string>Voicebox needs screen capture access to record system audio for voice samples.</string>
</dict>
</plist>
+18
View File
@@ -1,6 +1,24 @@
use std::process::Command;
fn main() {
// Link Swift runtime libraries for screencapturekit crate
#[cfg(target_os = "macos")]
{
// Add Swift runtime library paths to RPATH
println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift");
println!("cargo:rustc-link-arg=-L/usr/lib/swift");
// Also try Xcode's Swift libraries
if let Ok(output) = Command::new("xcode-select").arg("-p").output() {
if output.status.success() {
let xcode_path = String::from_utf8_lossy(&output.stdout).trim().to_string();
let swift_lib_path = format!("{}/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx", xcode_path);
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", swift_lib_path);
println!("cargo:rustc-link-arg=-L{}", swift_lib_path);
}
}
}
// Compile macOS Liquid Glass icon
#[cfg(target_os = "macos")]
{
Binary file not shown.
+213
View File
@@ -0,0 +1,213 @@
use crate::audio_capture::AudioCaptureState;
use base64::{engine::general_purpose, Engine as _};
use hound::{WavSpec, WavWriter};
use screencapturekit::{
cm::CMSampleBuffer,
shareable_content::SCShareableContent,
stream::{
configuration::SCStreamConfiguration,
content_filter::SCContentFilter,
output_trait::SCStreamOutputTrait,
output_type::SCStreamOutputType,
sc_stream::SCStream,
},
};
use std::io::Cursor;
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
) -> Result<(), String> {
// Reset previous samples
state.reset();
// Get shareable content
let content = SCShareableContent::get()
.map_err(|e| format!("Failed to get shareable content: {}", e))?;
// Get first display
let displays = content.displays();
if displays.is_empty() {
return Err("No displays available".to_string());
}
let display = &displays[0];
// Create content filter for desktop audio
let filter = SCContentFilter::create()
.with_display(display)
.with_excluding_windows(&[])
.build();
// Create stream configuration - audio only
let mut config = SCStreamConfiguration::default();
config.set_captures_audio(true);
config.set_excludes_current_process_audio(false);
config.set_sample_rate(48000); // Use i32 directly
config.set_channel_count(2); // Use i32 directly
// Create stream using builder
let (tx, mut rx) = mpsc::channel::<()>(1);
*state.stop_tx.lock().unwrap() = Some(tx);
let samples = state.samples.clone();
let sample_rate = state.sample_rate.clone();
let channels = state.channels.clone();
// Set sample rate and channels
*sample_rate.lock().unwrap() = 48000;
*channels.lock().unwrap() = 2;
// Create output handler struct
struct AudioHandler {
samples: Arc<Mutex<Vec<f32>>>,
}
impl SCStreamOutputTrait for AudioHandler {
fn did_output_sample_buffer(
&self,
sample: CMSampleBuffer,
_type: SCStreamOutputType,
) {
if _type == SCStreamOutputType::Audio {
if let Ok(audio_samples) = extract_audio_samples(sample) {
let mut samples_guard = self.samples.lock().unwrap();
samples_guard.extend_from_slice(&audio_samples);
}
}
}
}
let handler = AudioHandler {
samples: samples.clone(),
};
// Create stream
let mut stream = SCStream::new(&filter, &config);
// Add output handler for audio (order: handler, then output_type)
stream.add_output_handler(handler, SCStreamOutputType::Audio);
// Store stream reference
*state.stream.lock().unwrap() = Some(stream.clone());
stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?;
// Spawn task to stop after max duration
let stream_clone = stream.clone();
tokio::spawn(async move {
tokio::select! {
_ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => {
// Timeout reached
}
_ = rx.recv() => {
// Manual stop
}
}
let _ = stream_clone.stop_capture();
});
Ok(())
}
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
// Signal stop
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
let _ = tx.send(());
}
// Stop stream if still active
if let Some(stream) = state.stream.lock().unwrap().take() {
let _ = stream.stop_capture();
}
// Wait a bit for capture to stop
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Get samples
let samples = state.samples.lock().unwrap().clone();
let sample_rate = *state.sample_rate.lock().unwrap();
let channels = *state.channels.lock().unwrap();
if samples.is_empty() {
return Err("No audio samples captured".to_string());
}
// Convert to WAV
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
// Encode to base64
let base64_data = general_purpose::STANDARD.encode(&wav_data);
Ok(base64_data)
}
pub fn is_supported() -> bool {
// ScreenCaptureKit requires macOS 12.3+
// Check if we're on a supported version
#[cfg(target_os = "macos")]
{
// Basic check - ScreenCaptureKit should be available on macOS 12.3+
true
}
#[cfg(not(target_os = "macos"))]
{
false
}
}
fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result<Vec<f32>, String> {
// Use the crate's built-in method to get audio buffer list
let audio_buffer_list = sample_buffer
.audio_buffer_list()
.ok_or_else(|| "Failed to get audio buffer list".to_string())?;
let mut samples = Vec::new();
// Iterate through audio buffers
for buffer in audio_buffer_list.iter() {
// Get raw bytes and interpret as f32 samples
let data_bytes = buffer.data();
let num_samples = data_bytes.len() / std::mem::size_of::<f32>();
if num_samples > 0 {
unsafe {
// Interpret bytes as f32 samples
let data_ptr = data_bytes.as_ptr() as *const f32;
let data = std::slice::from_raw_parts(data_ptr, num_samples);
samples.extend_from_slice(data);
}
}
}
Ok(samples)
}
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
let mut buffer = Vec::new();
let cursor = Cursor::new(&mut buffer);
let spec = WavSpec {
channels,
sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer = WavWriter::new(cursor, spec)
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
// Convert f32 samples to i16
for sample in samples {
let clamped = sample.clamp(-1.0, 1.0);
let i16_sample = (clamped * 32767.0) as i16;
writer.write_sample(i16_sample)
.map_err(|e| format!("Failed to write sample: {}", e))?;
}
writer.finalize()
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
Ok(buffer)
}
+40
View File
@@ -0,0 +1,40 @@
#[cfg(target_os = "macos")]
mod macos;
#[cfg(target_os = "windows")]
mod windows;
#[cfg(target_os = "macos")]
pub use macos::*;
#[cfg(target_os = "windows")]
pub use windows::*;
use std::sync::{Arc, Mutex};
#[cfg(target_os = "macos")]
use screencapturekit::stream::sc_stream::SCStream;
pub struct AudioCaptureState {
pub samples: Arc<Mutex<Vec<f32>>>,
pub sample_rate: Arc<Mutex<u32>>,
pub channels: Arc<Mutex<u16>>,
pub stop_tx: Arc<Mutex<Option<tokio::sync::mpsc::Sender<()>>>>,
#[cfg(target_os = "macos")]
pub stream: Arc<Mutex<Option<SCStream>>>,
}
impl AudioCaptureState {
pub fn new() -> Self {
Self {
samples: Arc::new(Mutex::new(Vec::new())),
sample_rate: Arc::new(Mutex::new(44100)),
channels: Arc::new(Mutex::new(2)),
stop_tx: Arc::new(Mutex::new(None)),
#[cfg(target_os = "macos")]
stream: Arc::new(Mutex::new(None)),
}
}
pub fn reset(&self) {
*self.samples.lock().unwrap() = Vec::new();
}
}
@@ -0,0 +1,181 @@
use crate::audio_capture::AudioCaptureState;
use base64::{engine::general_purpose, Engine as _};
use hound::{WavSpec, WavWriter};
use std::io::Cursor;
use std::sync::{Arc, Mutex};
use tokio::sync::mpsc;
use wasapi::*;
pub async fn start_capture(
state: &AudioCaptureState,
max_duration_secs: u32,
) -> Result<(), String> {
// Reset previous samples
state.reset();
// Get default audio render device for loopback
let device = DeviceEnumerator::new()
.map_err(|e| format!("Failed to create device enumerator: {}", e))?
.get_default_audio_endpoint(&Direction::Render)
.map_err(|e| format!("Failed to get default render device: {}", e))?;
// Create audio client for loopback capture
let audio_client = device
.get_iaudioclient()
.map_err(|e| format!("Failed to get audio client: {}", e))?;
// Get mix format
let mix_format = audio_client
.get_mixformat()
.map_err(|e| format!("Failed to get mix format: {}", e))?;
// Set sample rate and channels
*state.sample_rate.lock().unwrap() = mix_format.get_samples_per_sec();
*state.channels.lock().unwrap() = mix_format.get_nchannels();
// Initialize audio client for loopback
audio_client
.initialize_client(
&mix_format,
0, // Buffer duration (0 = default)
&Direction::Capture,
ShareMode::Shared,
true, // Loopback mode
)
.map_err(|e| format!("Failed to initialize audio client: {}", e))?;
// Get capture client
let capture_client = audio_client
.get_audiocaptureclient()
.map_err(|e| format!("Failed to get capture client: {}", e))?;
// Start capture
audio_client
.start_stream()
.map_err(|e| format!("Failed to start stream: {}", e))?;
let samples = state.samples.clone();
let stop_tx = state.stop_tx.clone();
let (tx, mut rx) = mpsc::channel::<()>(1);
*stop_tx.lock().unwrap() = Some(tx);
// Spawn capture task - move audio_client and capture_client into the task
tokio::spawn(async move {
loop {
tokio::select! {
_ = rx.recv() => {
break;
}
_ = tokio::time::sleep(tokio::time::Duration::from_millis(10)) => {
// Try to get available data
match capture_client.get_available_samples() {
Ok(available) => {
if available > 0 {
match capture_client.get_buffer::<f32>() {
Ok((data, flags)) => {
if flags.contains(&StreamFlags::SILENT) {
// Silent buffer, skip
capture_client.release_buffer(available).ok();
continue;
}
// Convert samples to f32 and store
let mut samples_guard = samples.lock().unwrap();
samples_guard.extend_from_slice(data);
capture_client.release_buffer(available).ok();
}
Err(e) => {
eprintln!("Error getting buffer: {}", e);
}
}
}
}
Err(e) => {
eprintln!("Error getting available samples: {}", e);
}
}
}
}
}
// Stop the stream when done
audio_client.stop_stream().ok();
});
// Spawn timeout task
let stop_tx_clone = state.stop_tx.clone();
tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await;
if let Some(tx) = stop_tx_clone.lock().unwrap().take() {
let _ = tx.send(());
}
});
Ok(())
}
pub async fn stop_capture(state: &AudioCaptureState) -> Result<String, String> {
// Signal stop
if let Some(tx) = state.stop_tx.lock().unwrap().take() {
let _ = tx.send(());
}
// Wait a bit for capture to stop
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Get samples
let samples = state.samples.lock().unwrap().clone();
let sample_rate = *state.sample_rate.lock().unwrap();
let channels = *state.channels.lock().unwrap();
if samples.is_empty() {
return Err("No audio samples captured".to_string());
}
// Convert to WAV
let wav_data = samples_to_wav(&samples, sample_rate, channels)?;
// Encode to base64
let base64_data = general_purpose::STANDARD.encode(&wav_data);
Ok(base64_data)
}
pub fn is_supported() -> bool {
#[cfg(target_os = "windows")]
{
true
}
#[cfg(not(target_os = "windows"))]
{
false
}
}
fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result<Vec<u8>, String> {
let mut buffer = Vec::new();
let cursor = Cursor::new(&mut buffer);
let spec = WavSpec {
channels,
sample_rate,
bits_per_sample: 16,
sample_format: hound::SampleFormat::Int,
};
let mut writer = WavWriter::new(cursor, spec)
.map_err(|e| format!("Failed to create WAV writer: {}", e))?;
// Convert f32 samples to i16
for sample in samples {
let clamped = sample.clamp(-1.0, 1.0);
let i16_sample = (clamped * 32767.0) as i16;
writer.write_sample(i16_sample)
.map_err(|e| format!("Failed to write sample: {}", e))?;
}
writer.finalize()
.map_err(|e| format!("Failed to finalize WAV: {}", e))?;
Ok(buffer)
}
+30 -1
View File
@@ -1,6 +1,8 @@
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
mod audio_capture;
use std::sync::Mutex;
use tauri::{command, State, Manager, WindowEvent, Emitter, Listener};
use tauri_plugin_shell::ShellExt;
@@ -165,6 +167,26 @@ async fn stop_server(state: State<'_, ServerState>) -> Result<(), String> {
Ok(())
}
#[command]
async fn start_system_audio_capture(
state: State<'_, audio_capture::AudioCaptureState>,
max_duration_secs: u32,
) -> Result<(), String> {
audio_capture::start_capture(&state, max_duration_secs).await
}
#[command]
async fn stop_system_audio_capture(
state: State<'_, audio_capture::AudioCaptureState>,
) -> Result<String, String> {
audio_capture::stop_capture(&state).await
}
#[command]
fn is_system_audio_supported() -> bool {
audio_capture::is_supported()
}
#[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() {
tauri::Builder::default()
@@ -174,6 +196,7 @@ pub fn run() {
.manage(ServerState {
child: Mutex::new(None),
})
.manage(audio_capture::AudioCaptureState::new())
.setup(|app| {
#[cfg(desktop)]
app.handle().plugin(tauri_plugin_updater::Builder::new().build())?;
@@ -190,7 +213,13 @@ pub fn run() {
}
Ok(())
})
.invoke_handler(tauri::generate_handler![start_server, stop_server])
.invoke_handler(tauri::generate_handler![
start_server,
stop_server,
start_system_audio_capture,
stop_system_audio_capture,
is_system_audio_supported
])
.on_window_event(|window, event| {
if let WindowEvent::CloseRequested { api, .. } = event {
// Prevent automatic close