diff --git a/app/src/App.tsx b/app/src/App.tsx index 85d51597..623f458a 100644 --- a/app/src/App.tsx +++ b/app/src/App.tsx @@ -14,7 +14,7 @@ import { UpdateNotification } from '@/components/UpdateNotification'; import { Toaster } from '@/components/ui/toaster'; import { ProfileList } from '@/components/VoiceProfiles/ProfileList'; import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast'; -import { useRestoreActiveTasks, MODEL_DISPLAY_NAMES } from '@/lib/hooks/useRestoreActiveTasks'; +import { MODEL_DISPLAY_NAMES, useRestoreActiveTasks } from '@/lib/hooks/useRestoreActiveTasks'; import { isMacOS, isTauri, setupWindowCloseHandler, startServer } from '@/lib/tauri'; // Track if server is starting to prevent duplicate starts diff --git a/app/src/components/VoiceProfiles/ProfileForm.tsx b/app/src/components/VoiceProfiles/ProfileForm.tsx index d01681d6..f67c163f 100644 --- a/app/src/components/VoiceProfiles/ProfileForm.tsx +++ b/app/src/components/VoiceProfiles/ProfileForm.tsx @@ -39,9 +39,11 @@ import { } from '@/lib/hooks/useProfiles'; import { useTranscription } from '@/lib/hooks/useTranscription'; import { useAudioRecording } from '@/lib/hooks/useAudioRecording'; +import { useSystemAudioCapture } from '@/lib/hooks/useSystemAudioCapture'; import { useUIStore } from '@/stores/uiStore'; -import { Mic, Square, Upload } from 'lucide-react'; +import { Mic, Square, Upload, Monitor } from 'lucide-react'; import { formatAudioDuration } from '@/lib/utils/audio'; +import { isTauri } from '@/lib/tauri'; // Helper function to get audio duration from File async function getAudioDuration(file: File): Promise { @@ -101,7 +103,7 @@ export function ProfileForm() { const addSample = useAddSample(); const transcribe = useTranscription(); const { toast } = useToast(); - const [sampleMode, setSampleMode] = useState<'upload' | 'record'>('upload'); + const [sampleMode, setSampleMode] = useState<'upload' | 'record' | 'system'>('upload'); const [audioDuration, setAudioDuration] = useState(null); const [isValidatingAudio, setIsValidatingAudio] = useState(false); const isCreating = !editingProfileId; @@ -171,6 +173,28 @@ export function ProfileForm() { }, }); + const { + isRecording: isSystemRecording, + duration: systemDuration, + error: systemRecordingError, + isSupported: isSystemAudioSupported, + startRecording: startSystemRecording, + stopRecording: stopSystemRecording, + cancelRecording: cancelSystemRecording, + } = useSystemAudioCapture({ + maxDurationSeconds: 30, + onRecordingComplete: (blob) => { + const file = new File([blob], `system-audio-${Date.now()}.wav`, { + type: blob.type || 'audio/wav', + }); + form.setValue('sampleFile', file, { shouldValidate: true }); + toast({ + title: 'System audio captured', + description: 'Audio has been captured successfully.', + }); + }, + }); + // Show recording errors useEffect(() => { if (recordingError) { @@ -182,6 +206,17 @@ export function ProfileForm() { } }, [recordingError, toast]); + // Show system audio recording errors + useEffect(() => { + if (systemRecordingError) { + toast({ + title: 'System audio capture error', + description: systemRecordingError, + variant: 'destructive', + }); + } + }, [systemRecordingError, toast]); + useEffect(() => { if (editingProfile) { form.reset({ @@ -234,7 +269,11 @@ export function ProfileForm() { } function handleCancelRecording() { - cancelRecording(); + if (sampleMode === 'record') { + cancelRecording(); + } else if (sampleMode === 'system') { + cancelSystemRecording(); + } form.resetField('sampleFile'); } @@ -345,6 +384,9 @@ export function ProfileForm() { if (isRecording) { cancelRecording(); } + if (isSystemRecording) { + cancelSystemRecording(); + } } } @@ -432,9 +474,19 @@ export function ProfileForm() { setSampleMode(v as 'upload' | 'record')} + onValueChange={(v) => { + const newMode = v as 'upload' | 'record' | 'system'; + // Cancel any active recordings when switching modes + if (isRecording && newMode !== 'record') { + cancelRecording(); + } + if (isSystemRecording && newMode !== 'system') { + cancelSystemRecording(); + } + setSampleMode(newMode); + }} > - + Upload @@ -443,6 +495,12 @@ export function ProfileForm() { Record + {isTauri() && isSystemAudioSupported && ( + + + System Audio + + )} @@ -611,6 +669,103 @@ export function ProfileForm() { )} /> + + {isTauri() && isSystemAudioSupported && ( + + ( + + Capture System Audio + +
+ {!isSystemRecording && !selectedFile && ( +
+ +

+ Capture audio from your system. Maximum duration: 30 seconds. +

+
+ )} + + {isSystemRecording && ( +
+
+
+
+ + {formatAudioDuration(systemDuration)} + +
+
+ +

+ Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '} + remaining) +

+
+ )} + + {selectedFile && !isSystemRecording && ( +
+
+ + Capture complete +
+

+ File: {selectedFile.name} +

+
+ + +
+
+ )} +
+ + + Capture audio from your system (speakers, applications). Maximum duration is 30 + seconds. + + + + )} + /> + + )} ('upload'); + const [mode, setMode] = useState<'upload' | 'record' | 'system'>('upload'); const form = useForm({ resolver: zodResolver(sampleSchema), @@ -83,6 +85,29 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp }, }); + const { + isRecording: isSystemRecording, + duration: systemDuration, + error: systemRecordingError, + isSupported: isSystemAudioSupported, + startRecording: startSystemRecording, + stopRecording: stopSystemRecording, + cancelRecording: cancelSystemRecording, + } = useSystemAudioCapture({ + maxDurationSeconds: 30, + onRecordingComplete: (blob) => { + // Convert blob to File object + const file = new File([blob], `system-audio-${Date.now()}.wav`, { + type: blob.type || 'audio/wav', + }); + form.setValue('file', file, { shouldValidate: true }); + toast({ + title: 'System audio captured', + description: 'Audio has been captured successfully.', + }); + }, + }); + // Show recording errors useEffect(() => { if (recordingError) { @@ -94,6 +119,17 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp } }, [recordingError, toast]); + // Show system audio recording errors + useEffect(() => { + if (systemRecordingError) { + toast({ + title: 'System audio capture error', + description: systemRecordingError, + variant: 'destructive', + }); + } + }, [systemRecordingError, toast]); + async function handleTranscribe() { const file = form.getValues('file'); if (!file) { @@ -154,12 +190,19 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp if (isRecording) { cancelRecording(); } + if (isSystemRecording) { + cancelSystemRecording(); + } } onOpenChange(newOpen); } function handleCancelRecording() { - cancelRecording(); + if (mode === 'record') { + cancelRecording(); + } else if (mode === 'system') { + cancelSystemRecording(); + } // Reset file field by clearing the input form.resetField('file'); } @@ -176,8 +219,13 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp
- setMode(v as 'upload' | 'record')}> - + setMode(v as 'upload' | 'record' | 'system')} + > + Upload @@ -186,6 +234,12 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp Record + {isTauri() && isSystemAudioSupported && ( + + + System Audio + + )} @@ -325,6 +379,103 @@ export function SampleUpload({ profileId, open, onOpenChange }: SampleUploadProp )} /> + + {isTauri() && isSystemAudioSupported && ( + + ( + + Capture System Audio + +
+ {!isSystemRecording && !selectedFile && ( +
+ +

+ Capture audio from your system. Maximum duration: 30 seconds. +

+
+ )} + + {isSystemRecording && ( +
+
+
+
+ + {formatAudioDuration(systemDuration)} + +
+
+ +

+ Capturing system audio... ({formatAudioDuration(30 - systemDuration)}{' '} + remaining) +

+
+ )} + + {selectedFile && !isSystemRecording && mode === 'system' && ( +
+
+ + Capture complete +
+

+ File: {selectedFile.name} +

+
+ + +
+
+ )} +
+ + + Capture audio from your system (applications, browser tabs, etc.). Works + natively without browser dialogs. + + + + )} + /> + + )} void; +} + +/** + * Hook for native system audio capture using Tauri commands. + * Uses ScreenCaptureKit on macOS and WASAPI loopback on Windows. + */ +export function useSystemAudioCapture({ + maxDurationSeconds = 30, + onRecordingComplete, +}: UseSystemAudioCaptureOptions = {}) { + const [isRecording, setIsRecording] = useState(false); + const [duration, setDuration] = useState(0); + const [error, setError] = useState(null); + const [isSupported, setIsSupported] = useState(false); + const timerRef = useRef(null); + const startTimeRef = useRef(null); + const stopRecordingRef = useRef<(() => Promise) | null>(null); + + // Check if system audio capture is supported + useEffect(() => { + if (!isTauri()) { + setIsSupported(false); + return; + } + + invoke('is_system_audio_supported') + .then((supported) => { + setIsSupported(supported); + }) + .catch(() => { + setIsSupported(false); + }); + }, []); + + const startRecording = useCallback(async () => { + if (!isTauri()) { + const errorMsg = 'System audio capture is only available in the desktop app.'; + setError(errorMsg); + return; + } + + if (!isSupported) { + const errorMsg = 'System audio capture is not supported on this platform.'; + setError(errorMsg); + return; + } + + try { + setError(null); + setDuration(0); + + // Start native capture + await invoke('start_system_audio_capture', { + maxDurationSecs: maxDurationSeconds, + }); + + setIsRecording(true); + startTimeRef.current = Date.now(); + + // Start timer + timerRef.current = window.setInterval(() => { + if (startTimeRef.current) { + const elapsed = (Date.now() - startTimeRef.current) / 1000; + setDuration(elapsed); + + // Auto-stop at max duration + if (elapsed >= maxDurationSeconds && stopRecordingRef.current) { + void stopRecordingRef.current(); + } + } + }, 100); + } catch (err) { + const errorMessage = + err instanceof Error + ? err.message + : 'Failed to start system audio capture. Please check permissions.'; + setError(errorMessage); + setIsRecording(false); + } + }, [maxDurationSeconds, isSupported]); + + const stopRecording = useCallback(async () => { + if (!isRecording || !isTauri()) { + return; + } + + try { + setIsRecording(false); + + if (timerRef.current !== null) { + clearInterval(timerRef.current); + timerRef.current = null; + } + + // Stop capture and get base64 WAV data + const base64Data = await invoke('stop_system_audio_capture'); + + // Convert base64 to Blob + const binaryString = atob(base64Data); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + const blob = new Blob([bytes], { type: 'audio/wav' }); + onRecordingComplete?.(blob); + } catch (err) { + const errorMessage = + err instanceof Error + ? err.message + : 'Failed to stop system audio capture.'; + setError(errorMessage); + } + }, [isRecording, onRecordingComplete]); + + // Store stopRecording in ref for use in timer + useEffect(() => { + stopRecordingRef.current = stopRecording; + }, [stopRecording]); + + const cancelRecording = useCallback(async () => { + if (isRecording) { + await stopRecording(); + } + + setIsRecording(false); + setDuration(0); + + if (timerRef.current !== null) { + clearInterval(timerRef.current); + timerRef.current = null; + } + }, [isRecording, stopRecording]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (timerRef.current !== null) { + clearInterval(timerRef.current); + } + // Cancel recording on unmount if still recording + if (isRecording) { + void cancelRecording(); + } + }; + // biome-ignore lint/correctness/useExhaustiveDependencies: cancelRecording is stable + }, [isRecording]); + + return { + isRecording, + duration, + error, + isSupported, + startRecording, + stopRecording, + cancelRecording, + }; +} diff --git a/tauri/src-tauri/Cargo.lock b/tauri/src-tauri/Cargo.lock index 525a73d3..6773bedf 100644 --- a/tauri/src-tauri/Cargo.lock +++ b/tauri/src-tauri/Cargo.lock @@ -103,6 +103,24 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.114", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -267,6 +285,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfb" version = "0.7.3" @@ -312,6 +339,17 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "combine" version = "4.6.7" @@ -378,6 +416,15 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-sys" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6" +dependencies = [ + "bindgen", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -646,6 +693,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "embed-resource" version = "3.0.6" @@ -1212,6 +1265,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "html5ever" version = "0.29.1" @@ -1534,6 +1593,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1666,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf" dependencies = [ "gtk-sys", - "libloading", + "libloading 0.7.4", "once_cell", ] @@ -1686,6 +1754,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libredox" version = "0.1.12" @@ -1736,6 +1814,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "markup5ever" version = "0.14.1" @@ -1788,6 +1875,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "minisign-verify" version = "0.2.4" @@ -1878,12 +1971,31 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1915,6 +2027,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + [[package]] name = "objc2" version = "0.6.3" @@ -3016,6 +3137,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "screencapturekit" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ccf069cb109cf8e01ebdca0d55dfce45dbbf669e8c56ed5c62150b056d3ec9f" + [[package]] name = "selectors" version = "0.24.0" @@ -3504,7 +3631,7 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", @@ -3586,7 +3713,7 @@ dependencies = [ "webkit2gtk", "webview2-com", "window-vibrancy", - "windows", + "windows 0.61.3", ] [[package]] @@ -3784,7 +3911,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", ] [[package]] @@ -3810,7 +3937,7 @@ dependencies = [ "url", "webkit2gtk", "webview2-com", - "windows", + "windows 0.61.3", "wry", ] @@ -4358,6 +4485,12 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" name = "voicebox" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "core-foundation-sys", + "coreaudio-sys", + "hound", + "objc", + "screencapturekit", "serde", "serde_json", "tauri", @@ -4367,6 +4500,7 @@ dependencies = [ "tauri-plugin-shell", "tauri-plugin-updater", "tokio", + "wasapi", ] [[package]] @@ -4408,6 +4542,19 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasapi" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7834ac561bea8a7413661fdda62180f9e054f99815269cd3572cf7e40d9c3191" +dependencies = [ + "log", + "num-integer", + "thiserror 2.0.18", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -4582,7 +4729,7 @@ checksum = "7130243a7a5b33c54a444e54842e6a9e133de08b5ad7b5861cd8ed9a6a5bc96a" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-implement", "windows-interface", @@ -4606,7 +4753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "381336cfffd772377d291702245447a5251a2ffa5bad679c99e61bc48bacbf9c" dependencies = [ "thiserror 2.0.18", - "windows", + "windows 0.61.3", "windows-core 0.61.2", ] @@ -4662,11 +4809,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -4678,6 +4837,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.61.2" @@ -4712,7 +4880,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -4759,6 +4938,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -4897,6 +5086,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-version" version = "0.1.7" @@ -5123,7 +5321,7 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", + "windows 0.61.3", "windows-core 0.61.2", "windows-version", "x11-dl", diff --git a/tauri/src-tauri/Cargo.toml b/tauri/src-tauri/Cargo.toml index 3b22f7c0..4f484c9d 100644 --- a/tauri/src-tauri/Cargo.toml +++ b/tauri/src-tauri/Cargo.toml @@ -20,6 +20,17 @@ tauri-plugin-shell = "2.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", features = ["full"] } +hound = "3.5" +base64 = "0.22" + +[target.'cfg(target_os = "macos")'.dependencies] +screencapturekit = { version = "1", features = ["async"] } +coreaudio-sys = "0.2" +objc = "0.2" +core-foundation-sys = "0.8" + +[target.'cfg(target_os = "windows")'.dependencies] +wasapi = "0.22" [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] tauri-plugin-updater = "2.0" diff --git a/tauri/src-tauri/Info.plist b/tauri/src-tauri/Info.plist index 63df5b17..4099dc8b 100644 --- a/tauri/src-tauri/Info.plist +++ b/tauri/src-tauri/Info.plist @@ -8,5 +8,7 @@ voicebox NSMicrophoneUsageDescription voicebox needs microphone access to record voice samples for voice cloning. + NSScreenCaptureUsageDescription + Voicebox needs screen capture access to record system audio for voice samples. diff --git a/tauri/src-tauri/build.rs b/tauri/src-tauri/build.rs index d11286b3..69b18c37 100644 --- a/tauri/src-tauri/build.rs +++ b/tauri/src-tauri/build.rs @@ -1,6 +1,24 @@ use std::process::Command; fn main() { + // Link Swift runtime libraries for screencapturekit crate + #[cfg(target_os = "macos")] + { + // Add Swift runtime library paths to RPATH + println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); + println!("cargo:rustc-link-arg=-L/usr/lib/swift"); + + // Also try Xcode's Swift libraries + if let Ok(output) = Command::new("xcode-select").arg("-p").output() { + if output.status.success() { + let xcode_path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let swift_lib_path = format!("{}/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx", xcode_path); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", swift_lib_path); + println!("cargo:rustc-link-arg=-L{}", swift_lib_path); + } + } + } + // Compile macOS Liquid Glass icon #[cfg(target_os = "macos")] { diff --git a/tauri/src-tauri/gen/Assets.car b/tauri/src-tauri/gen/Assets.car index 15d5b61f..9db1fbf0 100644 Binary files a/tauri/src-tauri/gen/Assets.car and b/tauri/src-tauri/gen/Assets.car differ diff --git a/tauri/src-tauri/src/audio_capture/macos.rs b/tauri/src-tauri/src/audio_capture/macos.rs new file mode 100644 index 00000000..d929e65a --- /dev/null +++ b/tauri/src-tauri/src/audio_capture/macos.rs @@ -0,0 +1,213 @@ +use crate::audio_capture::AudioCaptureState; +use base64::{engine::general_purpose, Engine as _}; +use hound::{WavSpec, WavWriter}; +use screencapturekit::{ + cm::CMSampleBuffer, + shareable_content::SCShareableContent, + stream::{ + configuration::SCStreamConfiguration, + content_filter::SCContentFilter, + output_trait::SCStreamOutputTrait, + output_type::SCStreamOutputType, + sc_stream::SCStream, + }, +}; +use std::io::Cursor; +use std::sync::{Arc, Mutex}; +use tokio::sync::mpsc; + +pub async fn start_capture( + state: &AudioCaptureState, + max_duration_secs: u32, +) -> Result<(), String> { + // Reset previous samples + state.reset(); + + // Get shareable content + let content = SCShareableContent::get() + .map_err(|e| format!("Failed to get shareable content: {}", e))?; + + // Get first display + let displays = content.displays(); + if displays.is_empty() { + return Err("No displays available".to_string()); + } + let display = &displays[0]; + + // Create content filter for desktop audio + let filter = SCContentFilter::create() + .with_display(display) + .with_excluding_windows(&[]) + .build(); + + // Create stream configuration - audio only + let mut config = SCStreamConfiguration::default(); + config.set_captures_audio(true); + config.set_excludes_current_process_audio(false); + config.set_sample_rate(48000); // Use i32 directly + config.set_channel_count(2); // Use i32 directly + + // Create stream using builder + let (tx, mut rx) = mpsc::channel::<()>(1); + *state.stop_tx.lock().unwrap() = Some(tx); + + let samples = state.samples.clone(); + let sample_rate = state.sample_rate.clone(); + let channels = state.channels.clone(); + + // Set sample rate and channels + *sample_rate.lock().unwrap() = 48000; + *channels.lock().unwrap() = 2; + + // Create output handler struct + struct AudioHandler { + samples: Arc>>, + } + + impl SCStreamOutputTrait for AudioHandler { + fn did_output_sample_buffer( + &self, + sample: CMSampleBuffer, + _type: SCStreamOutputType, + ) { + if _type == SCStreamOutputType::Audio { + if let Ok(audio_samples) = extract_audio_samples(sample) { + let mut samples_guard = self.samples.lock().unwrap(); + samples_guard.extend_from_slice(&audio_samples); + } + } + } + } + + let handler = AudioHandler { + samples: samples.clone(), + }; + + // Create stream + let mut stream = SCStream::new(&filter, &config); + + // Add output handler for audio (order: handler, then output_type) + stream.add_output_handler(handler, SCStreamOutputType::Audio); + + // Store stream reference + *state.stream.lock().unwrap() = Some(stream.clone()); + + stream.start_capture().map_err(|e| format!("Failed to start capture: {}", e))?; + + // Spawn task to stop after max duration + let stream_clone = stream.clone(); + tokio::spawn(async move { + tokio::select! { + _ = tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)) => { + // Timeout reached + } + _ = rx.recv() => { + // Manual stop + } + } + let _ = stream_clone.stop_capture(); + }); + + Ok(()) +} + +pub async fn stop_capture(state: &AudioCaptureState) -> Result { + // Signal stop + if let Some(tx) = state.stop_tx.lock().unwrap().take() { + let _ = tx.send(()); + } + + // Stop stream if still active + if let Some(stream) = state.stream.lock().unwrap().take() { + let _ = stream.stop_capture(); + } + + // Wait a bit for capture to stop + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + + // Get samples + let samples = state.samples.lock().unwrap().clone(); + let sample_rate = *state.sample_rate.lock().unwrap(); + let channels = *state.channels.lock().unwrap(); + + if samples.is_empty() { + return Err("No audio samples captured".to_string()); + } + + // Convert to WAV + let wav_data = samples_to_wav(&samples, sample_rate, channels)?; + + // Encode to base64 + let base64_data = general_purpose::STANDARD.encode(&wav_data); + + Ok(base64_data) +} + +pub fn is_supported() -> bool { + // ScreenCaptureKit requires macOS 12.3+ + // Check if we're on a supported version + #[cfg(target_os = "macos")] + { + // Basic check - ScreenCaptureKit should be available on macOS 12.3+ + true + } + #[cfg(not(target_os = "macos"))] + { + false + } +} + +fn extract_audio_samples(sample_buffer: CMSampleBuffer) -> Result, String> { + // Use the crate's built-in method to get audio buffer list + let audio_buffer_list = sample_buffer + .audio_buffer_list() + .ok_or_else(|| "Failed to get audio buffer list".to_string())?; + + let mut samples = Vec::new(); + + // Iterate through audio buffers + for buffer in audio_buffer_list.iter() { + // Get raw bytes and interpret as f32 samples + let data_bytes = buffer.data(); + let num_samples = data_bytes.len() / std::mem::size_of::(); + + if num_samples > 0 { + unsafe { + // Interpret bytes as f32 samples + let data_ptr = data_bytes.as_ptr() as *const f32; + let data = std::slice::from_raw_parts(data_ptr, num_samples); + samples.extend_from_slice(data); + } + } + } + + Ok(samples) +} + +fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result, String> { + let mut buffer = Vec::new(); + let cursor = Cursor::new(&mut buffer); + + let spec = WavSpec { + channels, + sample_rate, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + + let mut writer = WavWriter::new(cursor, spec) + .map_err(|e| format!("Failed to create WAV writer: {}", e))?; + + // Convert f32 samples to i16 + for sample in samples { + let clamped = sample.clamp(-1.0, 1.0); + let i16_sample = (clamped * 32767.0) as i16; + writer.write_sample(i16_sample) + .map_err(|e| format!("Failed to write sample: {}", e))?; + } + + writer.finalize() + .map_err(|e| format!("Failed to finalize WAV: {}", e))?; + + Ok(buffer) +} diff --git a/tauri/src-tauri/src/audio_capture/mod.rs b/tauri/src-tauri/src/audio_capture/mod.rs new file mode 100644 index 00000000..54f47856 --- /dev/null +++ b/tauri/src-tauri/src/audio_capture/mod.rs @@ -0,0 +1,40 @@ +#[cfg(target_os = "macos")] +mod macos; +#[cfg(target_os = "windows")] +mod windows; + +#[cfg(target_os = "macos")] +pub use macos::*; +#[cfg(target_os = "windows")] +pub use windows::*; + +use std::sync::{Arc, Mutex}; + +#[cfg(target_os = "macos")] +use screencapturekit::stream::sc_stream::SCStream; + +pub struct AudioCaptureState { + pub samples: Arc>>, + pub sample_rate: Arc>, + pub channels: Arc>, + pub stop_tx: Arc>>>, + #[cfg(target_os = "macos")] + pub stream: Arc>>, +} + +impl AudioCaptureState { + pub fn new() -> Self { + Self { + samples: Arc::new(Mutex::new(Vec::new())), + sample_rate: Arc::new(Mutex::new(44100)), + channels: Arc::new(Mutex::new(2)), + stop_tx: Arc::new(Mutex::new(None)), + #[cfg(target_os = "macos")] + stream: Arc::new(Mutex::new(None)), + } + } + + pub fn reset(&self) { + *self.samples.lock().unwrap() = Vec::new(); + } +} diff --git a/tauri/src-tauri/src/audio_capture/windows.rs b/tauri/src-tauri/src/audio_capture/windows.rs new file mode 100644 index 00000000..57766bff --- /dev/null +++ b/tauri/src-tauri/src/audio_capture/windows.rs @@ -0,0 +1,181 @@ +use crate::audio_capture::AudioCaptureState; +use base64::{engine::general_purpose, Engine as _}; +use hound::{WavSpec, WavWriter}; +use std::io::Cursor; +use std::sync::{Arc, Mutex}; +use tokio::sync::mpsc; +use wasapi::*; + +pub async fn start_capture( + state: &AudioCaptureState, + max_duration_secs: u32, +) -> Result<(), String> { + // Reset previous samples + state.reset(); + + // Get default audio render device for loopback + let device = DeviceEnumerator::new() + .map_err(|e| format!("Failed to create device enumerator: {}", e))? + .get_default_audio_endpoint(&Direction::Render) + .map_err(|e| format!("Failed to get default render device: {}", e))?; + + // Create audio client for loopback capture + let audio_client = device + .get_iaudioclient() + .map_err(|e| format!("Failed to get audio client: {}", e))?; + + // Get mix format + let mix_format = audio_client + .get_mixformat() + .map_err(|e| format!("Failed to get mix format: {}", e))?; + + // Set sample rate and channels + *state.sample_rate.lock().unwrap() = mix_format.get_samples_per_sec(); + *state.channels.lock().unwrap() = mix_format.get_nchannels(); + + // Initialize audio client for loopback + audio_client + .initialize_client( + &mix_format, + 0, // Buffer duration (0 = default) + &Direction::Capture, + ShareMode::Shared, + true, // Loopback mode + ) + .map_err(|e| format!("Failed to initialize audio client: {}", e))?; + + // Get capture client + let capture_client = audio_client + .get_audiocaptureclient() + .map_err(|e| format!("Failed to get capture client: {}", e))?; + + // Start capture + audio_client + .start_stream() + .map_err(|e| format!("Failed to start stream: {}", e))?; + + let samples = state.samples.clone(); + let stop_tx = state.stop_tx.clone(); + let (tx, mut rx) = mpsc::channel::<()>(1); + *stop_tx.lock().unwrap() = Some(tx); + + // Spawn capture task - move audio_client and capture_client into the task + tokio::spawn(async move { + loop { + tokio::select! { + _ = rx.recv() => { + break; + } + _ = tokio::time::sleep(tokio::time::Duration::from_millis(10)) => { + // Try to get available data + match capture_client.get_available_samples() { + Ok(available) => { + if available > 0 { + match capture_client.get_buffer::() { + Ok((data, flags)) => { + if flags.contains(&StreamFlags::SILENT) { + // Silent buffer, skip + capture_client.release_buffer(available).ok(); + continue; + } + + // Convert samples to f32 and store + let mut samples_guard = samples.lock().unwrap(); + samples_guard.extend_from_slice(data); + capture_client.release_buffer(available).ok(); + } + Err(e) => { + eprintln!("Error getting buffer: {}", e); + } + } + } + } + Err(e) => { + eprintln!("Error getting available samples: {}", e); + } + } + } + } + } + + // Stop the stream when done + audio_client.stop_stream().ok(); + }); + + // Spawn timeout task + let stop_tx_clone = state.stop_tx.clone(); + tokio::spawn(async move { + tokio::time::sleep(tokio::time::Duration::from_secs(max_duration_secs as u64)).await; + if let Some(tx) = stop_tx_clone.lock().unwrap().take() { + let _ = tx.send(()); + } + }); + + Ok(()) +} + +pub async fn stop_capture(state: &AudioCaptureState) -> Result { + // Signal stop + if let Some(tx) = state.stop_tx.lock().unwrap().take() { + let _ = tx.send(()); + } + + // Wait a bit for capture to stop + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + + // Get samples + let samples = state.samples.lock().unwrap().clone(); + let sample_rate = *state.sample_rate.lock().unwrap(); + let channels = *state.channels.lock().unwrap(); + + if samples.is_empty() { + return Err("No audio samples captured".to_string()); + } + + // Convert to WAV + let wav_data = samples_to_wav(&samples, sample_rate, channels)?; + + // Encode to base64 + let base64_data = general_purpose::STANDARD.encode(&wav_data); + + Ok(base64_data) +} + +pub fn is_supported() -> bool { + #[cfg(target_os = "windows")] + { + true + } + #[cfg(not(target_os = "windows"))] + { + false + } +} + +fn samples_to_wav(samples: &[f32], sample_rate: u32, channels: u16) -> Result, String> { + let mut buffer = Vec::new(); + let cursor = Cursor::new(&mut buffer); + + let spec = WavSpec { + channels, + sample_rate, + bits_per_sample: 16, + sample_format: hound::SampleFormat::Int, + }; + + let mut writer = WavWriter::new(cursor, spec) + .map_err(|e| format!("Failed to create WAV writer: {}", e))?; + + // Convert f32 samples to i16 + for sample in samples { + let clamped = sample.clamp(-1.0, 1.0); + let i16_sample = (clamped * 32767.0) as i16; + writer.write_sample(i16_sample) + .map_err(|e| format!("Failed to write sample: {}", e))?; + } + + writer.finalize() + .map_err(|e| format!("Failed to finalize WAV: {}", e))?; + + Ok(buffer) +} diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index 4af341b8..667ee952 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -1,6 +1,8 @@ // Prevents additional console window on Windows in release, DO NOT REMOVE!! #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] +mod audio_capture; + use std::sync::Mutex; use tauri::{command, State, Manager, WindowEvent, Emitter, Listener}; use tauri_plugin_shell::ShellExt; @@ -165,6 +167,26 @@ async fn stop_server(state: State<'_, ServerState>) -> Result<(), String> { Ok(()) } +#[command] +async fn start_system_audio_capture( + state: State<'_, audio_capture::AudioCaptureState>, + max_duration_secs: u32, +) -> Result<(), String> { + audio_capture::start_capture(&state, max_duration_secs).await +} + +#[command] +async fn stop_system_audio_capture( + state: State<'_, audio_capture::AudioCaptureState>, +) -> Result { + audio_capture::stop_capture(&state).await +} + +#[command] +fn is_system_audio_supported() -> bool { + audio_capture::is_supported() +} + #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { tauri::Builder::default() @@ -174,6 +196,7 @@ pub fn run() { .manage(ServerState { child: Mutex::new(None), }) + .manage(audio_capture::AudioCaptureState::new()) .setup(|app| { #[cfg(desktop)] app.handle().plugin(tauri_plugin_updater::Builder::new().build())?; @@ -190,7 +213,13 @@ pub fn run() { } Ok(()) }) - .invoke_handler(tauri::generate_handler![start_server, stop_server]) + .invoke_handler(tauri::generate_handler![ + start_server, + stop_server, + start_system_audio_capture, + stop_system_audio_capture, + is_system_audio_supported + ]) .on_window_event(|window, event| { if let WindowEvent::CloseRequested { api, .. } = event { // Prevent automatic close