macos audio capture for sample creation

This commit is contained in:
Jamie Pine
2026-01-26 16:05:42 -08:00
parent c7c401b98c
commit 446182e16c
13 changed files with 1187 additions and 24 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ import { UpdateNotification } from '@/components/UpdateNotification';
import { Toaster } from '@/components/ui/toaster';
import { ProfileList } from '@/components/VoiceProfiles/ProfileList';
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
import { useRestoreActiveTasks, MODEL_DISPLAY_NAMES } from '@/lib/hooks/useRestoreActiveTasks';
import { MODEL_DISPLAY_NAMES, useRestoreActiveTasks } from '@/lib/hooks/useRestoreActiveTasks';
import { isMacOS, isTauri, setupWindowCloseHandler, startServer } from '@/lib/tauri';
// Track if server is starting to prevent duplicate starts
@@ -39,9 +39,11 @@ import {
} from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
import { useUIStore } from '@/stores/uiStore';
import { Mic, Square, Upload } from 'lucide-react';
import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
import { isTauri } from '@/lib/tauri';
// Helper function to get audio duration from File
async function getAudioDuration(file: File): Promise<number> {
@@ -101,7 +103,7 @@ export function ProfileForm() {
const addSample = useAddSample();
const transcribe = useTranscription();
const { toast } = useToast();
const [sampleMode, setSampleMode] = useState<'upload' | 'record'>('upload');
const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('upload');
const [audioDuration, setAudioDuration] = useState<number | null>(null);
const [isValidatingAudio, setIsValidatingAudio] = useState(false);
const isCreating = !editingProfileId;
@@ -171,6 +173,28 @@ export function ProfileForm() {
},
});
const {
isRecording: isSystemRecording,
duration: systemDuration,
error: systemRecordingError,
isSupported: isSystemAudioSupported,
startRecording: startSystemRecording,
stopRecording: stopSystemRecording,
cancelRecording: cancelSystemRecording,
} = useSystemAudioCapture({
maxDurationSeconds: 30,
onRecordingComplete: (blob) => {
const file = new File([blob], `system-audio-${Date.now()}.wav`, {
type: blob.type || 'audio/wav',
});
form.setValue('sampleFile', file, { shouldValidate: true });
toast({
title: 'System audio captured',
description: 'Audio has been captured successfully.',
});
},
});
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -182,6 +206,17 @@ export function ProfileForm() {
}
}, [recordingError, toast]);
// Show system audio recording errors
useEffect(() => {
if (systemRecordingError) {
toast({
title: 'System audio capture error',
description: systemRecordingError,
variant: 'destructive',
});
}
}, [systemRecordingError, toast]);
useEffect(() => {
if (editingProfile) {
form.reset({
@@ -234,7 +269,11 @@ export function ProfileForm() {
}
function handleCancelRecording() {
cancelRecording();
if (sampleMode === 'record') {
cancelRecording();
} else if (sampleMode === 'system') {
cancelSystemRecording();
}
form.resetField('sampleFile');
}
@@ -345,6 +384,9 @@ export function ProfileForm() {
if (isRecording) {
cancelRecording();
}
if (isSystemRecording) {
cancelSystemRecording();
}
}
}
@@ -432,9 +474,19 @@ export function ProfileForm() {
<Tabs
value={sampleMode}
onValueChange={(v) => setSampleMode(v as 'upload' | 'record')}
onValueChange={(v) => {
const newMode = v as 'upload' | 'record' | 'system';
// Cancel any active recordings when switching modes
if (isRecording && newMode !== 'record') {
cancelRecording();
}
if (isSystemRecording && newMode !== 'system') {
cancelSystemRecording();
}
setSampleMode(newMode);
}}
>
<TabsList className="grid w-full grid-cols-2">
<TabsList className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}>
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -443,6 +495,12 @@ export function ProfileForm() {
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
{isTauri() && isSystemAudioSupported && (
<TabsTrigger value="system" className="flex items-center gap-2">
<Monitor className="h-4 w-4" />
System Audio
</TabsTrigger>
)}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -611,6 +669,103 @@ export function ProfileForm() {
)}
/>
</TabsContent>
{isTauri() && isSystemAudioSupported && (
<TabsContent value="system" className="space-y-4">
<FormField
control={form.control}
name="sampleFile"
render={() => (
<FormItem>
<FormLabel>Capture System Audio</FormLabel>
<FormControl>
<div className="space-y-4">
{!isSystemRecording && !selectedFile && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-dashed rounded-lg">
<Button
type="button"
onClick={startSystemRecording}
size="lg"
className="flex items-center gap-2"
>
<Monitor className="h-5 w-5" />
Start Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capture audio from your system. Maximum duration: 30 seconds.
</p>
</div>
)}
{isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-destructive rounded-lg bg-destructive/5">
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
<span className="text-lg font-mono font-semibold">
{formatAudioDuration(systemDuration)}
</span>
</div>
</div>
<Button
type="button"
onClick={stopSystemRecording}
variant="destructive"
className="flex items-center gap-2"
>
<Square className="h-4 w-4" />
Stop Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
remaining)
</p>
</div>
)}
{selectedFile && !isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-4 border-2 border-primary rounded-lg bg-primary/5">
<div className="flex items-center gap-2">
<Monitor className="h-5 w-5 text-primary" />
<span className="font-medium">Capture complete</span>
</div>
<p className="text-sm text-muted-foreground text-center">
File: {selectedFile.name}
</p>
<div className="flex gap-2">
<Button
type="button"
variant="outline"
onClick={handleTranscribe}
disabled={transcribe.isPending}
className="flex items-center gap-2"
>
<Mic className="h-4 w-4" />
{transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
</Button>
<Button
type="button"
variant="outline"
onClick={handleCancelRecording}
className="flex items-center gap-2"
>
Capture Again
</Button>
</div>
</div>
)}
</div>
</FormControl>
<FormDescription>
Capture audio from your system (speakers, applications). Maximum duration is 30
seconds.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
</TabsContent>
)}
</Tabs>
<FormField
@@ -26,8 +26,10 @@ import { useToast } from '@/components/ui/use-toast';
import { useAddSample, useProfile } from '@/lib/hooks/useProfiles';
import { useTranscription } from '@/lib/hooks/useTranscription';
import { useAudioRecording } from '@/lib/hooks/useAudioRecording';
import { Mic, Square, Upload } from 'lucide-react';
import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture';
import { Mic, Square, Upload, Monitor } from 'lucide-react';
import { formatAudioDuration } from '@/lib/utils/audio';
import { isTauri } from '@/lib/tauri';
const sampleSchema = z.object({
file: z.instanceof(File, { message: 'Please select an audio file' }),
@@ -50,7 +52,7 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
const transcribe = useTranscription();
const { data: profile } = useProfile(profileId);
const { toast } = useToast();
const [mode, setMode] = useState<'upload' | 'record'>('upload');
const [mode, setMode] = useState<'upload' | 'record' | 'system'>('upload');
const form = useForm<SampleFormValues>({
resolver: zodResolver(sampleSchema),
@@ -83,6 +85,29 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
},
});
const {
isRecording: isSystemRecording,
duration: systemDuration,
error: systemRecordingError,
isSupported: isSystemAudioSupported,
startRecording: startSystemRecording,
stopRecording: stopSystemRecording,
cancelRecording: cancelSystemRecording,
} = useSystemAudioCapture({
maxDurationSeconds: 30,
onRecordingComplete: (blob) => {
// Convert blob to File object
const file = new File([blob], `system-audio-${Date.now()}.wav`, {
type: blob.type || 'audio/wav',
});
form.setValue('file', file, { shouldValidate: true });
toast({
title: 'System audio captured',
description: 'Audio has been captured successfully.',
});
},
});
// Show recording errors
useEffect(() => {
if (recordingError) {
@@ -94,6 +119,17 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
}
}, [recordingError, toast]);
// Show system audio recording errors
useEffect(() => {
if (systemRecordingError) {
toast({
title: 'System audio capture error',
description: systemRecordingError,
variant: 'destructive',
});
}
}, [systemRecordingError, toast]);
async function handleTranscribe() {
const file = form.getValues('file');
if (!file) {
@@ -154,12 +190,19 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
if (isRecording) {
cancelRecording();
}
if (isSystemRecording) {
cancelSystemRecording();
}
}
onOpenChange(newOpen);
}
function handleCancelRecording() {
cancelRecording();
if (mode === 'record') {
cancelRecording();
} else if (mode === 'system') {
cancelSystemRecording();
}
// Reset file field by clearing the input
form.resetField('file');
}
@@ -176,8 +219,13 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
<Tabs value={mode} onValueChange={(v) => setMode(v as 'upload' | 'record')}>
<TabsList className="grid w-full grid-cols-2">
<Tabs
value={mode}
onValueChange={(v) => setMode(v as 'upload' | 'record' | 'system')}
>
<TabsList
className={`grid w-full ${isTauri() && isSystemAudioSupported ? 'grid-cols-3' : 'grid-cols-2'}`}
>
<TabsTrigger value="upload" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Upload
@@ -186,6 +234,12 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
<Mic className="h-4 w-4" />
Record
</TabsTrigger>
{isTauri() && isSystemAudioSupported && (
<TabsTrigger value="system" className="flex items-center gap-2">
<Monitor className="h-4 w-4" />
System Audio
</TabsTrigger>
)}
</TabsList>
<TabsContent value="upload" className="space-y-4">
@@ -325,6 +379,103 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
)}
/>
</TabsContent>
{isTauri() && isSystemAudioSupported && (
<TabsContent value="system" className="space-y-4">
<FormField
control={form.control}
name="file"
render={() => (
<FormItem>
<FormLabel>Capture System Audio</FormLabel>
<FormControl>
<div className="space-y-4">
{!isSystemRecording && !selectedFile && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-dashed rounded-lg">
<Button
type="button"
onClick={startSystemRecording}
size="lg"
className="flex items-center gap-2"
>
<Monitor className="h-5 w-5" />
Start Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capture audio from your system. Maximum duration: 30 seconds.
</p>
</div>
)}
{isSystemRecording && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-destructive rounded-lg bg-destructive/5">
<div className="flex items-center gap-4">
<div className="flex items-center gap-2">
<div className="h-3 w-3 rounded-full bg-destructive animate-pulse" />
<span className="text-lg font-mono font-semibold">
{formatAudioDuration(systemDuration)}
</span>
</div>
</div>
<Button
type="button"
onClick={stopSystemRecording}
variant="destructive"
className="flex items-center gap-2"
>
<Square className="h-4 w-4" />
Stop Capture
</Button>
<p className="text-sm text-muted-foreground text-center">
Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '}
remaining)
</p>
</div>
)}
{selectedFile && !isSystemRecording && mode === 'system' && (
<div className="flex flex-col items-center gap-4 p-6 border-2 border-primary rounded-lg bg-primary/5">
<div className="flex items-center gap-2">
<Monitor className="h-5 w-5 text-primary" />
<span className="font-medium">Capture complete</span>
</div>
<p className="text-sm text-muted-foreground">
File: {selectedFile.name}
</p>
<div className="flex gap-2">
<Button
type="button"
variant="outline"
onClick={handleTranscribe}
disabled={transcribe.isPending}
className="flex items-center gap-2"
>
<Mic className="h-4 w-4" />
{transcribe.isPending ? 'Transcribing...' : 'Transcribe'}
</Button>
<Button
type="button"
variant="outline"
onClick={handleCancelRecording}
className="flex items-center gap-2"
>
Capture Again
</Button>
</div>
</div>
)}
</div>
</FormControl>
<FormDescription>
Capture audio from your system (applications, browser tabs, etc.). Works
natively without browser dialogs.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
</TabsContent>
)}
</Tabs>
<FormField
+165
View File
@@ -0,0 +1,165 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { invoke } from '@tauri-apps/api/core';
import { isTauri } from '@/lib/tauri';
interface UseSystemAudioCaptureOptions {
maxDurationSeconds?: number;
onRecordingComplete?: (blob: Blob) => void;
}
/**
* Hook for native system audio capture using Tauri commands.
* Uses ScreenCaptureKit on macOS and WASAPI loopback on Windows.
*/
export function useSystemAudioCapture({
maxDurationSeconds = 30,
onRecordingComplete,
}: UseSystemAudioCaptureOptions = {}) {
const [isRecording, setIsRecording] = useState(false);
const [duration, setDuration] = useState(0);
const [error, setError] = useState<string | null>(null);
const [isSupported, setIsSupported] = useState(false);
const timerRef = useRef<number | null>(null);
const startTimeRef = useRef<number | null>(null);
const stopRecordingRef = useRef<(() => Promise<void>) | null>(null);
// Check if system audio capture is supported
useEffect(() => {
if (!isTauri()) {
setIsSupported(false);
return;
}
invoke<boolean>('is_system_audio_supported')
.then((supported) => {
setIsSupported(supported);
})
.catch(() => {
setIsSupported(false);
});
}, []);
const startRecording = useCallback(async () => {
if (!isTauri()) {
const errorMsg = 'System audio capture is only available in the desktop app.';
setError(errorMsg);
return;
}
if (!isSupported) {
const errorMsg = 'System audio capture is not supported on this platform.';
setError(errorMsg);
return;
}
try {
setError(null);
setDuration(0);
// Start native capture
await invoke('start_system_audio_capture', {
maxDurationSecs: maxDurationSeconds,
});
setIsRecording(true);
startTimeRef.current = Date.now();
// Start timer
timerRef.current = window.setInterval(() => {
if (startTimeRef.current) {
const elapsed = (Date.now() - startTimeRef.current) / 1000;
setDuration(elapsed);
// Auto-stop at max duration
if (elapsed >= maxDurationSeconds && stopRecordingRef.current) {
void stopRecordingRef.current();
}
}
}, 100);
} catch (err) {
const errorMessage =
err instanceof Error
? err.message
: 'Failed to start system audio capture. Please check permissions.';
setError(errorMessage);
setIsRecording(false);
}
}, [maxDurationSeconds, isSupported]);
const stopRecording = useCallback(async () => {
if (!isRecording || !isTauri()) {
return;
}
try {
setIsRecording(false);
if (timerRef.current !== null) {
clearInterval(timerRef.current);
timerRef.current = null;
}
// Stop capture and get base64 WAV data
const base64Data = await invoke<string>('stop_system_audio_capture');
// Convert base64 to Blob
const binaryString = atob(base64Data);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const blob = new Blob([bytes], { type: 'audio/wav' });
onRecordingComplete?.(blob);
} catch (err) {
const errorMessage =
err instanceof Error
? err.message
: 'Failed to stop system audio capture.';
setError(errorMessage);
}
}, [isRecording, onRecordingComplete]);
// Store stopRecording in ref for use in timer
useEffect(() => {
stopRecordingRef.current = stopRecording;
}, [stopRecording]);
const cancelRecording = useCallback(async () => {
if (isRecording) {
await stopRecording();
}
setIsRecording(false);
setDuration(0);
if (timerRef.current !== null) {
clearInterval(timerRef.current);
timerRef.current = null;
}
}, [isRecording, stopRecording]);
// Cleanup on unmount
useEffect(() => {
return () => {
if (timerRef.current !== null) {
clearInterval(timerRef.current);
}
// Cancel recording on unmount if still recording
if (isRecording) {
void cancelRecording();
}
};
// biome-ignore lint/correctness/useExhaustiveDependencies: cancelRecording is stable
}, [isRecording]);
return {
isRecording,
duration,
error,
isSupported,
startRecording,
stopRecording,
cancelRecording,
};
}