From f1963740b4dc74f24de1e1cc5c7b8d95364d7dc8 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Sun, 15 Mar 2026 04:04:56 -0700 Subject: [PATCH] Fix server binary build, watchdog logging, pedalboard import, window close loop --- backend/build_binary.py | 83 +++++++++++++++++++++++++++++++++++-- backend/server.py | 55 ++++++++++++++++++++---- justfile | 2 + tauri/src-tauri/src/main.rs | 26 ++++++------ 4 files changed, 141 insertions(+), 25 deletions(-) diff --git a/backend/build_binary.py b/backend/build_binary.py index 9ce7db87..223170aa 100644 --- a/backend/build_binary.py +++ b/backend/build_binary.py @@ -10,6 +10,7 @@ import PyInstaller.__main__ import argparse import os import platform +import sys from pathlib import Path @@ -62,6 +63,10 @@ def build_server(cuda=False): '--hidden-import', 'backend.utils.hf_progress', '--hidden-import', 'backend.utils.validation', '--hidden-import', 'backend.cuda_download', + '--hidden-import', 'backend.effects', + '--hidden-import', 'backend.utils.effects', + '--hidden-import', 'backend.versions', + '--hidden-import', 'pedalboard', '--hidden-import', 'torch', '--hidden-import', 'transformers', '--hidden-import', 'fastapi', @@ -91,9 +96,10 @@ def build_server(cuda=False): '--hidden-import', 'torch.backends.cudnn', ]) else: - # Exclude NVIDIA CUDA packages from CPU-only builds to keep binary under 4GB. - # On Linux, pip may pull CUDA-enabled PyTorch by default which includes ~3GB - # of NVIDIA shared libraries that PyInstaller would bundle. + # Exclude NVIDIA CUDA packages from CPU-only builds to keep binary small. + # When building from a venv with CUDA torch installed, PyInstaller would + # bundle ~3GB of NVIDIA shared libraries. We exclude both the Python + # modules and the binary DLLs. nvidia_packages = [ 'nvidia', 'nvidia.cublas', 'nvidia.cuda_cupti', 'nvidia.cuda_nvrtc', 'nvidia.cuda_runtime', 'nvidia.cudnn', 'nvidia.cufft', 'nvidia.curand', @@ -127,7 +133,12 @@ def build_server(cuda=False): elif not cuda: print("Building for non-Apple Silicon platform - PyTorch only") + dist_dir = str(backend_dir / 'dist') + build_dir = str(backend_dir / 'build') + args.extend([ + '--distpath', dist_dir, + '--workpath', build_dir, '--noconfirm', '--clean', ]) @@ -135,12 +146,78 @@ def build_server(cuda=False): # Change to backend directory os.chdir(backend_dir) + # For CPU builds on Windows, ensure we're using CPU-only torch. + # If CUDA torch is installed (local dev), swap to CPU torch before building, + # then restore CUDA torch after. This prevents PyInstaller from bundling + # ~3GB of CUDA DLLs into the CPU binary. + restore_cuda = False + if not cuda and platform.system() == "Windows": + import subprocess + result = subprocess.run( + [sys.executable, "-c", "import torch; print(torch.version.cuda or '')"], + capture_output=True, text=True + ) + has_cuda_torch = bool(result.stdout.strip()) + if has_cuda_torch: + print("CUDA torch detected — installing CPU torch for CPU build...") + subprocess.run( + [sys.executable, "-m", "pip", "install", "torch", "torchvision", "torchaudio", + "--index-url", "https://download.pytorch.org/whl/cpu", "--force-reinstall", "-q"], + check=True + ) + restore_cuda = True + # Run PyInstaller PyInstaller.__main__.run(args) + + # Restore CUDA torch if we swapped it out + if restore_cuda: + print("Restoring CUDA torch...") + import subprocess + subprocess.run( + [sys.executable, "-m", "pip", "install", "torch", "torchvision", "torchaudio", + "--index-url", "https://download.pytorch.org/whl/cu126", "--force-reinstall", "-q"], + check=True + ) print(f"Binary built in {backend_dir / 'dist' / binary_name}") +def _get_cuda_dll_excludes(): + """Get list of CUDA DLL filenames to exclude from CPU builds. + + When building locally with CUDA torch installed, PyInstaller bundles ~3GB of + CUDA DLLs from torch/lib/. Returns a list of DLL filenames to exclude. + """ + try: + import torch + torch_lib = Path(torch.__file__).parent / 'lib' + except ImportError: + return [] + + cuda_prefixes = ( + 'torch_cuda', 'cublas', 'cublasLt', 'cudnn', 'cusparse', 'cufft', + 'cusolver', 'cusolverMg', 'curand', 'nvrtc', 'nvJitLink', 'nccl', + 'nvperf', 'nvrtc-builtins', + ) + + exclude_dlls = [] + if torch_lib.exists(): + for f in torch_lib.iterdir(): + if f.suffix == '.dll' and any(f.name.startswith(p) for p in cuda_prefixes): + exclude_dlls.append(f.name) + + if exclude_dlls: + total_mb = sum( + (torch_lib / dll).stat().st_size + for dll in exclude_dlls + if (torch_lib / dll).exists() + ) / 1024 / 1024 + print(f"CPU build: will exclude {len(exclude_dlls)} CUDA DLLs ({total_mb:.0f} MB)") + + return exclude_dlls + + if __name__ == '__main__': parser = argparse.ArgumentParser(description="Build voicebox-server binary") parser.add_argument( diff --git a/backend/server.py b/backend/server.py index dc2b0da1..2cf6d212 100644 --- a/backend/server.py +++ b/backend/server.py @@ -51,7 +51,7 @@ except Exception as e: logger.error(f"Failed to import required modules: {e}", exc_info=True) sys.exit(1) -def _start_parent_watchdog(parent_pid): +def _start_parent_watchdog(parent_pid, data_dir=None): """Monitor parent process and exit if it dies. This is the clean shutdown mechanism: instead of the Tauri app trying to @@ -63,17 +63,44 @@ def _start_parent_watchdog(parent_pid): import threading import time + # Set up a file logger so we can debug in production + watchdog_logger = logging.getLogger("watchdog") + if data_dir: + try: + log_dir = os.path.join(data_dir, "logs") + os.makedirs(log_dir, exist_ok=True) + fh = logging.FileHandler(os.path.join(log_dir, "watchdog.log")) + fh.setFormatter(logging.Formatter('%(asctime)s - %(message)s')) + watchdog_logger.addHandler(fh) + except Exception: + pass + watchdog_logger.setLevel(logging.INFO) + def _is_pid_alive(pid): """Check if a process with the given PID exists (cross-platform).""" try: if sys.platform == "win32": import ctypes kernel32 = ctypes.windll.kernel32 - SYNCHRONIZE = 0x00100000 - handle = kernel32.OpenProcess(SYNCHRONIZE, False, pid) + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) if handle: + # Check if process has actually exited + STILL_ACTIVE = 259 + exit_code = ctypes.c_ulong() + result = kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) kernel32.CloseHandle(handle) - return True + if result and exit_code.value == STILL_ACTIVE: + return True + watchdog_logger.info(f"PID {pid}: exited with code {exit_code.value}") + return False + # OpenProcess failed — check if it's an access error (process exists + # but we can't open it) vs process not found + error = ctypes.GetLastError() + ACCESS_DENIED = 5 + if error == ACCESS_DENIED: + return True # process exists, we just can't open it + watchdog_logger.info(f"PID {pid}: OpenProcess failed, error={error}") return False else: os.kill(pid, 0) @@ -82,13 +109,19 @@ def _start_parent_watchdog(parent_pid): return False def _watch(): - logger.info(f"Parent watchdog started, monitoring PID {parent_pid}") + watchdog_logger.info(f"Parent watchdog started, monitoring PID {parent_pid}, server PID {os.getpid()}") + # Verify parent is alive before starting the loop + alive = _is_pid_alive(parent_pid) + watchdog_logger.info(f"Parent PID {parent_pid} initial check: alive={alive}") + if not alive: + watchdog_logger.warning(f"Parent PID {parent_pid} not found on first check — disabling watchdog") + return while True: if not _is_pid_alive(parent_pid): - logger.info(f"Parent process {parent_pid} no longer exists, shutting down...") + watchdog_logger.info(f"Parent process {parent_pid} gone, shutting down server...") os.kill(os.getpid(), signal.SIGTERM) return - time.sleep(1) + time.sleep(2) t = threading.Thread(target=_watch, daemon=True) t.start() @@ -139,9 +172,13 @@ if __name__ == "__main__": os.environ["VOICEBOX_BACKEND_VARIANT"] = "cpu" logger.info("Backend variant: CPU") - # Start parent process watchdog if requested + # Register parent watchdog to start after server is fully ready if args.parent_pid is not None: - _start_parent_watchdog(args.parent_pid) + _parent_pid = args.parent_pid + _data_dir = args.data_dir + @app.on_event("startup") + async def _on_startup(): + _start_parent_watchdog(_parent_pid, _data_dir) logger.info(f"Parsed arguments: host={args.host}, port={args.port}, data_dir={args.data_dir}") diff --git a/justfile b/justfile index 108becc8..9788123a 100644 --- a/justfile +++ b/justfile @@ -189,8 +189,10 @@ build-server: _ensure-venv [windows] build-server: _ensure-venv + $ErrorActionPreference = "Stop"; \ $env:PATH = "{{ venv_bin }};$env:PATH"; \ & "{{ python }}" backend/build_binary.py; \ + if ($LASTEXITCODE -ne 0) { throw "build_binary.py failed with exit code $LASTEXITCODE" }; \ $triple = (rustc --print host-tuple); \ Copy-Item "backend/dist/voicebox-server.exe" "{{ tauri_dir }}/src-tauri/binaries/voicebox-server-$triple.exe" -Force; \ Write-Host "Copied sidecar: voicebox-server-$triple.exe" diff --git a/tauri/src-tauri/src/main.rs b/tauri/src-tauri/src/main.rs index ad1bb126..939bb962 100644 --- a/tauri/src-tauri/src/main.rs +++ b/tauri/src-tauri/src/main.rs @@ -708,9 +708,17 @@ pub fn run() { play_audio_to_devices, stop_audio_playback ]) - .on_window_event(|window, event| { + .on_window_event({ + let closing = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + move |window, event| { if let WindowEvent::CloseRequested { api, .. } = event { - // Prevent automatic close + // If we're already in the close flow, let it proceed + if closing.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + closing.store(true, std::sync::atomic::Ordering::SeqCst); + + // Prevent automatic close so frontend can clean up api.prevent_close(); // Emit event to frontend to check setting and stop server if needed @@ -718,42 +726,34 @@ pub fn run() { if let Err(e) = app_handle.emit("window-close-requested", ()) { eprintln!("Failed to emit window-close-requested event: {}", e); - // If event emission fails, allow close anyway window.close().ok(); return; } // Set up listener for frontend response let window_for_close = window.clone(); + let closing_for_timeout = closing.clone(); let (tx, mut rx) = mpsc::unbounded_channel::<()>(); - // Listen for response from frontend using window's listen method let listener_id = window.listen("window-close-allowed", move |_| { - // Frontend has checked setting and stopped server if needed - // Signal that we can close let _ = tx.send(()); }); - // Wait for frontend response or timeout - // Use tauri::async_runtime::spawn instead of tokio::spawn to avoid - // panics when the Tokio runtime is being dropped during app shutdown tauri::async_runtime::spawn(async move { tokio::select! { _ = rx.recv() => { - // Frontend responded, close window window_for_close.close().ok(); } _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => { - // Timeout - close anyway eprintln!("Window close timeout, closing anyway"); window_for_close.close().ok(); } } - // Clean up listener window_for_close.unlisten(listener_id); + closing_for_timeout.store(false, std::sync::atomic::Ordering::SeqCst); }); } - }) + }}) .build(tauri::generate_context!()) .expect("error while building tauri application") .run(|app, event| {