From f4d21504e37c3219a6d2a50b643dbeb0a3a57b2d Mon Sep 17 00:00:00 2001 From: James Pine Date: Sat, 25 Apr 2026 17:09:32 -0700 Subject: [PATCH] Mobile companion app + paired-device backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New iOS-first companion (Expo SDK 54 + NativeWind v4) with three tabs: Captures (the hero — floating gold mic, live mic-meter waveform, expand-row playback), Generate (profile picker + speak + autoplay + recent), and Voices (searchable profile list). Pairing (V0): backend mints a one-time token, mobile scans/pastes the voicebox:// URL, server returns a long-lived bearer it stores only as a SHA-256 hash. Bearer-or-loopback auth applied to every user-data router so binding 0.0.0.0 doesn't leak existing endpoints. Loopback callers (the desktop app) keep their friction-free access. Desktop Settings → Mobile pane: live host picker (LAN / Tailscale auto- detected via the App-bundle binary path on macOS), QR rendering, 5-minute expiry countdown, copyable URL fallback, paired-device list with revoke. Auto-closes when a new device pairs. just dev now binds the backend to 0.0.0.0 so paired phones can reach it — and just setup-python pins mlx-audio==0.4.1 + mlx-lm so fresh Apple Silicon worktrees get a working STT path on first install. --- app/package.json | 1 + app/src/components/ServerTab/MobilePage.tsx | 219 ++ .../components/ServerTab/PairDeviceDialog.tsx | 240 +++ app/src/components/ServerTab/ServerTab.tsx | 4 + app/src/lib/api/client.ts | 23 + app/src/lib/api/types.ts | 24 + app/src/lib/hooks/usePairedDevices.ts | 57 + app/src/router.tsx | 8 + backend/database/__init__.py | 4 + backend/database/models.py | 34 + backend/models.py | 58 + backend/routes/__init__.py | 61 +- backend/routes/pairing.py | 92 + backend/services/pairing.py | 201 ++ backend/utils/auth.py | 103 + bun.lock | 13 +- justfile | 32 +- mobile/.gitignore | 41 + mobile/PLAN.md | 139 ++ mobile/app.json | 44 + mobile/app/(tabs)/_layout.tsx | 56 + mobile/app/(tabs)/generate.tsx | 323 +++ mobile/app/(tabs)/index.tsx | 394 ++++ mobile/app/(tabs)/voices.tsx | 178 ++ mobile/app/_layout.tsx | 46 + mobile/app/pair.tsx | 164 ++ mobile/app/welcome.tsx | 7 + mobile/assets/adaptive-icon.png | Bin 0 -> 318876 bytes mobile/assets/favicon.png | Bin 0 -> 1466 bytes mobile/assets/icon.png | Bin 0 -> 318876 bytes mobile/assets/splash-icon.png | Bin 0 -> 318876 bytes mobile/babel.config.js | 9 + mobile/bun.lock | 1763 +++++++++++++++++ mobile/global.css | 3 + mobile/metro.config.js | 6 + mobile/nativewind-env.d.ts | 1 + mobile/package.json | 39 + mobile/src/components/ui/Button.tsx | 108 + mobile/src/components/ui/LiveWaveform.tsx | 78 + mobile/src/lib/api.ts | 262 +++ mobile/src/lib/colors.ts | 20 + mobile/src/lib/dictation.ts | 105 + mobile/src/lib/hooks.ts | 110 + mobile/src/lib/pairing.ts | 20 + mobile/src/lib/session.ts | 33 + mobile/src/lib/storage.ts | 30 + mobile/src/screens/WelcomeScreen.tsx | 49 + mobile/tailwind.config.js | 46 + mobile/tsconfig.json | 11 + 49 files changed, 5229 insertions(+), 30 deletions(-) create mode 100644 app/src/components/ServerTab/MobilePage.tsx create mode 100644 app/src/components/ServerTab/PairDeviceDialog.tsx create mode 100644 app/src/lib/hooks/usePairedDevices.ts create mode 100644 backend/routes/pairing.py create mode 100644 backend/services/pairing.py create mode 100644 backend/utils/auth.py create mode 100644 mobile/.gitignore create mode 100644 mobile/PLAN.md create mode 100644 mobile/app.json create mode 100644 mobile/app/(tabs)/_layout.tsx create mode 100644 mobile/app/(tabs)/generate.tsx create mode 100644 mobile/app/(tabs)/index.tsx create mode 100644 mobile/app/(tabs)/voices.tsx create mode 100644 mobile/app/_layout.tsx create mode 100644 mobile/app/pair.tsx create mode 100644 mobile/app/welcome.tsx create mode 100644 mobile/assets/adaptive-icon.png create mode 100644 mobile/assets/favicon.png create mode 100644 mobile/assets/icon.png create mode 100644 mobile/assets/splash-icon.png create mode 100644 mobile/babel.config.js create mode 100644 mobile/bun.lock create mode 100644 mobile/global.css create mode 100644 mobile/metro.config.js create mode 100644 mobile/nativewind-env.d.ts create mode 100644 mobile/package.json create mode 100644 mobile/src/components/ui/Button.tsx create mode 100644 mobile/src/components/ui/LiveWaveform.tsx create mode 100644 mobile/src/lib/api.ts create mode 100644 mobile/src/lib/colors.ts create mode 100644 mobile/src/lib/dictation.ts create mode 100644 mobile/src/lib/hooks.ts create mode 100644 mobile/src/lib/pairing.ts create mode 100644 mobile/src/lib/session.ts create mode 100644 mobile/src/lib/storage.ts create mode 100644 mobile/src/screens/WelcomeScreen.tsx create mode 100644 mobile/tailwind.config.js create mode 100644 mobile/tsconfig.json diff --git a/app/package.json b/app/package.json index 56bf162a..39271fa9 100644 --- a/app/package.json +++ b/app/package.json @@ -52,6 +52,7 @@ "react-dom": "^18.3.0", "react-hook-form": "^7.53.0", "react-i18next": "^17.0.4", + "react-qr-code": "^2.0.18", "react-sound-visualizer": "^1.4.0", "tailwind-merge": "^2.5.4", "wavesurfer.js": "^7.0.0", diff --git a/app/src/components/ServerTab/MobilePage.tsx b/app/src/components/ServerTab/MobilePage.tsx new file mode 100644 index 00000000..d0249bac --- /dev/null +++ b/app/src/components/ServerTab/MobilePage.tsx @@ -0,0 +1,219 @@ +import { Lock, MoreHorizontal, Plus, Smartphone, WifiOff } from 'lucide-react'; +import { useState } from 'react'; +import { Button } from '@/components/ui/button'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; +import { useToast } from '@/components/ui/use-toast'; +import { + usePairedDevices, + useRevokePairedDevice, +} from '@/lib/hooks/usePairedDevices'; +import type { PairedDeviceResponse } from '@/lib/api/types'; +import { PairDeviceDialog } from './PairDeviceDialog'; +import { SettingRow, SettingSection } from './SettingRow'; + +function formatRelative(iso: string | null): string { + if (!iso) return 'never'; + const then = new Date(iso).getTime(); + const diffSec = Math.floor((Date.now() - then) / 1000); + if (diffSec < 60) return 'just now'; + if (diffSec < 3600) return `${Math.floor(diffSec / 60)}m ago`; + if (diffSec < 86400) return `${Math.floor(diffSec / 3600)}h ago`; + if (diffSec < 86400 * 30) return `${Math.floor(diffSec / 86400)}d ago`; + return new Date(iso).toLocaleDateString(); +} + +export function MobilePage() { + const [pairOpen, setPairOpen] = useState(false); + const devices = usePairedDevices(); + const revoke = useRevokePairedDevice(); + const { toast } = useToast(); + + const active = (devices.data ?? []).filter((d) => !d.revoked); + const revoked = (devices.data ?? []).filter((d) => d.revoked); + + function handleRevoke(d: PairedDeviceResponse) { + revoke.mutate(d.id, { + onSuccess: () => { + toast({ + title: 'Device revoked', + description: `${d.name} can no longer reach this Voicebox.`, + }); + }, + onError: (e) => { + toast({ + title: 'Revoke failed', + description: e instanceof Error ? e.message : String(e), + variant: 'destructive', + }); + }, + }); + } + + return ( +
+
+ + 0 ? `, ${revoked.length} revoked` : ''}` + } + action={ + + } + /> + + {devices.data && devices.data.length > 0 ? ( +
+ {[...active, ...revoked].map((d) => ( + handleRevoke(d)} + revoking={revoke.isPending && revoke.variables === d.id} + /> + ))} +
+ ) : devices.isLoading ? ( +
Loading devices…
+ ) : ( + setPairOpen(true)} /> + )} + +
+ + + + +
+ ); +} + +function DeviceRow({ + device, + onRevoke, + revoking, +}: { + device: PairedDeviceResponse; + onRevoke: () => void; + revoking: boolean; +}) { + return ( +
+
+
+ +
+
+
+ {device.name} + {device.revoked ? ( + + revoked + + ) : null} +
+
+ Last seen {formatRelative(device.last_seen_at)} · paired{' '} + {formatRelative(device.created_at)} +
+
+
+ {!device.revoked ? ( + + + + + + + Revoke + + + + ) : null} +
+ ); +} + +function EmptyState({ onPair }: { onPair: () => void }) { + return ( +
+
+ +
+
+

No paired devices

+

+ Pair your phone to dictate captures, queue generations, and play back voices on the go. +

+
+ +
+ ); +} diff --git a/app/src/components/ServerTab/PairDeviceDialog.tsx b/app/src/components/ServerTab/PairDeviceDialog.tsx new file mode 100644 index 00000000..f3757fc0 --- /dev/null +++ b/app/src/components/ServerTab/PairDeviceDialog.tsx @@ -0,0 +1,240 @@ +import { Check, Copy, Loader2, RefreshCw, Smartphone } from 'lucide-react'; +import { useEffect, useMemo, useState } from 'react'; +import QRCode from 'react-qr-code'; +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { useToast } from '@/components/ui/use-toast'; +import { + PAIRED_DEVICES_KEY, + useInitPairing, + usePairedDevices, + usePairHostCandidates, +} from '@/lib/hooks/usePairedDevices'; +import { useQueryClient } from '@tanstack/react-query'; + +type Props = { + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +function formatRemaining(ms: number): string { + if (ms <= 0) return 'expired'; + const total = Math.floor(ms / 1000); + const m = Math.floor(total / 60); + const s = total % 60; + return `${m}:${s.toString().padStart(2, '0')}`; +} + +export function PairDeviceDialog({ open, onOpenChange }: Props) { + const { toast } = useToast(); + const qc = useQueryClient(); + const candidates = usePairHostCandidates(open); + const initPairing = useInitPairing(); + const devices = usePairedDevices({ polling: open }); + + const [selectedHost, setSelectedHost] = useState(null); + const [copied, setCopied] = useState(false); + const [now, setNow] = useState(() => Date.now()); + + // The IDs that existed when the dialog opened — anything new is a fresh + // pairing we should celebrate. + const [baselineDeviceIds, setBaselineDeviceIds] = useState | null>(null); + + // On open: snapshot baseline devices, default-select the first non-loopback + // candidate, and mint the first token. + useEffect(() => { + if (!open) { + setBaselineDeviceIds(null); + setSelectedHost(null); + initPairing.reset(); + return; + } + if (devices.data && baselineDeviceIds === null) { + setBaselineDeviceIds(new Set(devices.data.map((d) => d.id))); + } + if (candidates.data && candidates.data.length > 0 && selectedHost === null) { + const preferred = + candidates.data.find((c) => c.kind !== 'loopback') ?? candidates.data[0]; + setSelectedHost(preferred.address); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [open, candidates.data, devices.data]); + + // Re-mint the token whenever the host selection changes (the URL embeds + // the host, so a new selection means a new QR). + useEffect(() => { + if (!open || !selectedHost) return; + initPairing.mutate(selectedHost); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [open, selectedHost]); + + // Wall-clock tick for the countdown. + useEffect(() => { + if (!open) return; + const id = window.setInterval(() => setNow(Date.now()), 1000); + return () => window.clearInterval(id); + }, [open]); + + // Detect a freshly paired device and close + toast. + useEffect(() => { + if (!open || !devices.data || baselineDeviceIds === null) return; + const newDevice = devices.data.find( + (d) => !baselineDeviceIds.has(d.id) && !d.revoked, + ); + if (newDevice) { + toast({ + title: 'Device paired', + description: `${newDevice.name} is now connected.`, + }); + qc.invalidateQueries({ queryKey: PAIRED_DEVICES_KEY }); + onOpenChange(false); + } + }, [open, devices.data, baselineDeviceIds, toast, qc, onOpenChange]); + + const pairing = initPairing.data; + const expiresAtMs = pairing ? new Date(pairing.expires_at).getTime() : 0; + const remainingMs = expiresAtMs - now; + const expired = pairing != null && remainingMs <= 0; + + const qrValue = useMemo(() => pairing?.pairing_url ?? '', [pairing]); + + async function handleCopy() { + if (!pairing) return; + try { + await navigator.clipboard.writeText(pairing.pairing_url); + setCopied(true); + window.setTimeout(() => setCopied(false), 1500); + } catch (e) { + toast({ + title: 'Copy failed', + description: e instanceof Error ? e.message : 'Could not access clipboard', + variant: 'destructive', + }); + } + } + + function handleRegenerate() { + if (!selectedHost) return; + initPairing.mutate(selectedHost); + } + + return ( + + + + + + Pair a new device + + + Open Voicebox on your phone, tap Get started → Pair, + then point its camera at this QR. + + + +
+ {/* Host picker */} +
+ + + {candidates.isError ? ( +

+ {(candidates.error as Error)?.message ?? + 'Failed to fetch /pair/host-candidates — is the backend up to date?'} +

+ ) : null} +
+ + {/* QR */} +
+ {initPairing.isPending && !pairing ? ( + + ) : initPairing.isError ? ( +

+ {(initPairing.error as Error)?.message ?? 'Failed to mint token'} +

+ ) : qrValue ? ( + + ) : ( +

No host selected

+ )} +
+ + {/* Countdown + regenerate */} + {pairing ? ( +
+ + {expired + ? 'QR expired — regenerate to continue' + : `Expires in ${formatRemaining(remainingMs)}`} + + +
+ ) : null} + + {/* Copyable URL */} + {pairing ? ( +
+ +
+ {pairing.pairing_url} + +
+
+ ) : null} + +

+ The token is single-use and expires in 5 minutes. Once paired, your phone holds a + long-lived bearer that only it knows — Voicebox stores just a hash. Revoke any time. +

+
+ + + ); +} diff --git a/app/src/components/ServerTab/ServerTab.tsx b/app/src/components/ServerTab/ServerTab.tsx index ec8502d0..5ffd2691 100644 --- a/app/src/components/ServerTab/ServerTab.tsx +++ b/app/src/components/ServerTab/ServerTab.tsx @@ -13,6 +13,7 @@ interface SettingsTab { | '/settings/generation' | '/settings/captures' | '/settings/mcp' + | '/settings/mobile' | '/settings/gpu' | '/settings/logs' | '/settings/changelog' @@ -25,6 +26,9 @@ const tabs: SettingsTab[] = [ { labelKey: 'settings.tabs.generation', path: '/settings/generation' }, { labelKey: 'settings.tabs.captures', path: '/settings/captures' }, { labelKey: 'settings.tabs.mcp', path: '/settings/mcp' }, + // Plain-string label for V0 — translation keys come when mobile graduates + // out of "experimental" status. + { label: 'Mobile', path: '/settings/mobile' }, { labelKey: 'settings.tabs.gpu', path: '/settings/gpu', tauriOnly: true }, { labelKey: 'settings.tabs.logs', path: '/settings/logs', tauriOnly: true }, { labelKey: 'settings.tabs.changelog', path: '/settings/changelog' }, diff --git a/app/src/lib/api/client.ts b/app/src/lib/api/client.ts index a8d030af..c28b293d 100644 --- a/app/src/lib/api/client.ts +++ b/app/src/lib/api/client.ts @@ -50,6 +50,9 @@ import type { MCPClientBinding, MCPClientBindingListResponse, MCPClientBindingUpsert, + HostCandidate, + PairInitResponse, + PairedDeviceResponse, } from './types'; function formatErrorDetail(detail: unknown, fallback: string): string { @@ -920,6 +923,26 @@ class ApiClient { return response.blob(); } + + // Mobile pairing + async getPairHostCandidates(): Promise { + return this.request('/pair/host-candidates'); + } + + async initPairing(host: string): Promise { + return this.request( + `/pair/init?host=${encodeURIComponent(host)}`, + { method: 'POST' }, + ); + } + + async listPairedDevices(): Promise { + return this.request('/devices'); + } + + async revokePairedDevice(deviceId: string): Promise { + await this.request(`/devices/${deviceId}`, { method: 'DELETE' }); + } } export const apiClient = new ApiClient(); diff --git a/app/src/lib/api/types.ts b/app/src/lib/api/types.ts index 37ca4667..17f1560c 100644 --- a/app/src/lib/api/types.ts +++ b/app/src/lib/api/types.ts @@ -521,3 +521,27 @@ export interface MCPClientBindingUpsert { export interface MCPClientBindingListResponse { items: MCPClientBinding[]; } + +/* ─── Mobile pairing (V0) ────────────────────────────────────────────── */ + +export type HostCandidateKind = 'lan' | 'tailscale' | 'loopback'; + +export interface HostCandidate { + address: string; // host:port + label: string; // human-friendly name + kind: HostCandidateKind; +} + +export interface PairInitResponse { + token: string; + expires_at: string; + pairing_url: string; // voicebox://pair?host=…&token=… +} + +export interface PairedDeviceResponse { + id: string; + name: string; + revoked: boolean; + created_at: string; + last_seen_at: string | null; +} diff --git a/app/src/lib/hooks/usePairedDevices.ts b/app/src/lib/hooks/usePairedDevices.ts new file mode 100644 index 00000000..9cd3c29b --- /dev/null +++ b/app/src/lib/hooks/usePairedDevices.ts @@ -0,0 +1,57 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { apiClient } from '@/lib/api/client'; + +export const PAIRED_DEVICES_KEY = ['paired-devices'] as const; +export const PAIR_HOST_CANDIDATES_KEY = ['pair-host-candidates'] as const; + +/** + * List all paired (and revoked) devices. Pass ``polling: true`` while the + * pair dialog is open so the device list refreshes when the user finishes + * scanning on their phone — this is how the desktop UI detects success + * without needing an SSE stream. + */ +export function usePairedDevices({ polling = false }: { polling?: boolean } = {}) { + return useQuery({ + queryKey: PAIRED_DEVICES_KEY, + queryFn: () => apiClient.listPairedDevices(), + refetchInterval: polling ? 2000 : false, + }); +} + +/** + * The candidate addresses the desktop can embed in the QR (LAN, Tailscale, + * loopback). Cached for the lifetime of the dialog — interfaces don't + * change often enough to be worth re-polling. + */ +export function usePairHostCandidates(enabled: boolean) { + return useQuery({ + queryKey: PAIR_HOST_CANDIDATES_KEY, + queryFn: () => apiClient.getPairHostCandidates(), + enabled, + staleTime: Infinity, + }); +} + +/** + * Mint a fresh pairing token for a chosen host. The result is short-lived + * (5 min); the dialog should re-mint when expiry is hit. + */ +export function useInitPairing() { + return useMutation({ + mutationFn: (host: string) => apiClient.initPairing(host), + }); +} + +/** + * Revoke a paired device. Invalidates the device list so the row disappears + * on success. + */ +export function useRevokePairedDevice() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (deviceId: string) => apiClient.revokePairedDevice(deviceId), + onSuccess: () => { + qc.invalidateQueries({ queryKey: PAIRED_DEVICES_KEY }); + }, + }); +} diff --git a/app/src/router.tsx b/app/src/router.tsx index 940e7c52..04699f34 100644 --- a/app/src/router.tsx +++ b/app/src/router.tsx @@ -18,6 +18,7 @@ import { GenerationPage } from '@/components/ServerTab/GenerationPage'; import { GpuPage } from '@/components/ServerTab/GpuPage'; import { LogsPage } from '@/components/ServerTab/LogsPage'; import { MCPPage } from '@/components/ServerTab/MCPPage'; +import { MobilePage } from '@/components/ServerTab/MobilePage'; import { SettingsLayout } from '@/components/ServerTab/ServerTab'; import { Sidebar } from '@/components/Sidebar'; import { StoriesTab } from '@/components/StoriesTab/StoriesTab'; @@ -166,6 +167,12 @@ const settingsMCPRoute = createRoute({ component: MCPPage, }); +const settingsMobileRoute = createRoute({ + getParentRoute: () => settingsRoute, + path: '/mobile', + component: MobilePage, +}); + const settingsGpuRoute = createRoute({ getParentRoute: () => settingsRoute, path: '/gpu', @@ -212,6 +219,7 @@ const routeTree = rootRoute.addChildren([ settingsGenerationRoute, settingsCapturesRoute, settingsMCPRoute, + settingsMobileRoute, settingsGpuRoute, settingsLogsRoute, settingsChangelogRoute, diff --git a/backend/database/__init__.py b/backend/database/__init__.py index bfb4b124..1fffc6e3 100644 --- a/backend/database/__init__.py +++ b/backend/database/__init__.py @@ -16,6 +16,8 @@ from .models import ( GenerationSettings, GenerationVersion, MCPClientBinding, + PairedDevice, + PairingToken, ProfileChannelMapping, ProfileSample, Project, @@ -37,6 +39,8 @@ __all__ = [ "GenerationSettings", "GenerationVersion", "MCPClientBinding", + "PairedDevice", + "PairingToken", "ProfileChannelMapping", "ProfileSample", "Project", diff --git a/backend/database/models.py b/backend/database/models.py index 6ef2213e..d1c9af82 100644 --- a/backend/database/models.py +++ b/backend/database/models.py @@ -279,3 +279,37 @@ class Capture(Base): llm_model = Column(String, nullable=True) refinement_flags = Column(Text, nullable=True) # JSON blob created_at = Column(DateTime, default=datetime.utcnow) + + +class PairedDevice(Base): + """A mobile device paired with this Voicebox install (V0 pair flow). + + Stores only the SHA-256 of the bearer token; the bearer plaintext is + returned to the device once at pairing time and never persisted + server-side. If the device loses its bearer the user must re-pair. + """ + + __tablename__ = "paired_devices" + + id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4())) + name = Column(String, nullable=False) + bearer_hash = Column(String, nullable=False, unique=True, index=True) + revoked = Column(Boolean, default=False, nullable=False) + created_at = Column(DateTime, default=datetime.utcnow) + last_seen_at = Column(DateTime, nullable=True) + + +class PairingToken(Base): + """One-time token used to complete a device pairing. + + Minted by the desktop UI via /pair/init, redeemed by the mobile + device via /pair/complete in exchange for a long-lived bearer. + Single-use; expires after ~5 minutes. + """ + + __tablename__ = "pairing_tokens" + + token = Column(String, primary_key=True) + expires_at = Column(DateTime, nullable=False) + used_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=datetime.utcnow) diff --git a/backend/models.py b/backend/models.py index 06f321ac..e6bda09e 100644 --- a/backend/models.py +++ b/backend/models.py @@ -793,3 +793,61 @@ class AvailableEffectsResponse(BaseModel): """Response listing all available effect types.""" effects: List[AvailableEffect] + + +# --- Mobile pairing (V0) ----------------------------------------------------- + +class HostCandidate(BaseModel): + """A reachable address the desktop can embed in the pair QR.""" + + address: str # ``host:port``, e.g. "192.168.1.5:17493" + label: str # human-friendly name shown in the desktop host picker + kind: str # "lan" | "tailscale" | "loopback" + + +class PairInitResponse(BaseModel): + """Response for POST /pair/init — desktop renders ``pairing_url`` as a QR.""" + + token: str + expires_at: datetime + pairing_url: str # full voicebox://pair?host=…&token=… URL + + +class PairCompleteRequest(BaseModel): + """Mobile-side request body for POST /pair/complete.""" + + token: str = Field(..., min_length=1, max_length=128) + device_name: str = Field(..., min_length=1, max_length=80) + + +class PairCompleteResponse(BaseModel): + """One-time response after successful pairing. + + ``bearer`` is returned in plaintext exactly once and is never persisted + server-side; the device must save it (e.g. iOS SecureStore) immediately. + """ + + device_id: str + bearer: str + device_name: str + + +class PairedDeviceResponse(BaseModel): + """A row in the desktop's Settings → Mobile device list.""" + + id: str + name: str + revoked: bool + created_at: datetime + last_seen_at: Optional[datetime] = None + + class Config: + from_attributes = True + + +class MeResponse(BaseModel): + """Identity of the bearer-authenticated caller.""" + + device_id: str + device_name: str + last_seen_at: Optional[datetime] = None diff --git a/backend/routes/__init__.py b/backend/routes/__init__.py index 35563aaa..206217d0 100644 --- a/backend/routes/__init__.py +++ b/backend/routes/__init__.py @@ -1,6 +1,23 @@ -"""Route registration for the voicebox API.""" +"""Route registration for the voicebox API. -from fastapi import FastAPI +Authentication model +-------------------- +Two router groups: + +* **Open** — ``health`` (status checks anyone on the LAN may probe) and + ``pairing`` (pre-pair endpoints + admin endpoints with their own + loopback-only or token-only gates). + +* **Protected** — everything else, gated by ``require_bearer_or_loopback``: + loopback callers (the desktop app over 127.0.0.1) pass without auth as + before; LAN/Tailscale callers must present a valid paired-device bearer. + This is what lets ``just dev`` bind to 0.0.0.0 without exposing user + data to anyone on the same network. +""" + +from fastapi import Depends, FastAPI + +from ..utils.auth import require_bearer_or_loopback def register_routers(app: FastAPI) -> None: @@ -23,22 +40,28 @@ def register_routers(app: FastAPI) -> None: from .speak import router as speak_router from .mcp_bindings import router as mcp_bindings_router from .events import router as events_router + from .pairing import router as pairing_router + # Open — health probes and the pre-pair / admin pairing endpoints. app.include_router(health_router) - app.include_router(profiles_router) - app.include_router(channels_router) - app.include_router(generations_router) - app.include_router(history_router) - app.include_router(transcription_router) - app.include_router(llm_router) - app.include_router(captures_router) - app.include_router(stories_router) - app.include_router(effects_router) - app.include_router(audio_router) - app.include_router(models_router) - app.include_router(settings_router) - app.include_router(tasks_router) - app.include_router(cuda_router) - app.include_router(speak_router) - app.include_router(mcp_bindings_router) - app.include_router(events_router) + app.include_router(pairing_router) + + # Protected — loopback callers pass through; LAN callers need a paired bearer. + protected = [Depends(require_bearer_or_loopback)] + app.include_router(profiles_router, dependencies=protected) + app.include_router(channels_router, dependencies=protected) + app.include_router(generations_router, dependencies=protected) + app.include_router(history_router, dependencies=protected) + app.include_router(transcription_router, dependencies=protected) + app.include_router(llm_router, dependencies=protected) + app.include_router(captures_router, dependencies=protected) + app.include_router(stories_router, dependencies=protected) + app.include_router(effects_router, dependencies=protected) + app.include_router(audio_router, dependencies=protected) + app.include_router(models_router, dependencies=protected) + app.include_router(settings_router, dependencies=protected) + app.include_router(tasks_router, dependencies=protected) + app.include_router(cuda_router, dependencies=protected) + app.include_router(speak_router, dependencies=protected) + app.include_router(mcp_bindings_router, dependencies=protected) + app.include_router(events_router, dependencies=protected) diff --git a/backend/routes/pairing.py b/backend/routes/pairing.py new file mode 100644 index 00000000..e3bb17cd --- /dev/null +++ b/backend/routes/pairing.py @@ -0,0 +1,92 @@ +"""Mobile device pairing endpoints (V0 — bearer auth).""" + +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy.orm import Session + +from .. import models +from ..database import PairedDevice, get_db +from ..services import pairing +from ..utils.auth import require_loopback, require_paired_device + +router = APIRouter() + + +@router.post( + "/pair/init", + response_model=models.PairInitResponse, + dependencies=[Depends(require_loopback)], +) +async def pair_init(request: Request, db: Session = Depends(get_db)): + """Mint a one-time pairing token (loopback callers only). + + Optional ``?host=`` query param overrides what's embedded in the QR's + pairing URL. The desktop UI should pass whatever address is reachable + from the mobile device — LAN IP, Tailscale 100.x address, or MagicDNS + name. Defaults to the request Host header for curl-driven local testing. + """ + host = request.query_params.get("host") or ( + request.headers.get("host") or "127.0.0.1:17493" + ) + return pairing.init_pairing_token(db, host=host) + + +@router.post("/pair/complete", response_model=models.PairCompleteResponse) +async def pair_complete( + body: models.PairCompleteRequest, + db: Session = Depends(get_db), +): + """Exchange a pairing token for a long-lived bearer. + + Open endpoint — possession of the (one-time, short-TTL) token is + itself the proof of authorization. + """ + try: + return pairing.complete_pairing(db, body.token, body.device_name) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.get( + "/devices", + response_model=list[models.PairedDeviceResponse], + dependencies=[Depends(require_loopback)], +) +async def list_paired_devices(db: Session = Depends(get_db)): + """List paired devices for the desktop Settings → Mobile pane.""" + return pairing.list_devices(db) + + +@router.delete( + "/devices/{device_id}", + status_code=204, + dependencies=[Depends(require_loopback)], +) +async def revoke_paired_device(device_id: str, db: Session = Depends(get_db)): + """Revoke a paired device's bearer.""" + try: + pairing.revoke_device(db, device_id) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + + +@router.get("/me", response_model=models.MeResponse) +async def me(device: PairedDevice = Depends(require_paired_device)): + """Identity of the bearer-authenticated caller — used by mobile to + confirm pairing succeeded and the bearer round-trips. + """ + return models.MeResponse( + device_id=device.id, + device_name=device.name, + last_seen_at=device.last_seen_at, + ) + + +@router.get( + "/pair/host-candidates", + response_model=list[models.HostCandidate], + dependencies=[Depends(require_loopback)], +) +async def host_candidates(request: Request): + """Suggested host strings (LAN IP, Tailscale, loopback) for the QR.""" + port = request.url.port or 17493 + return pairing.list_host_candidates(port=port) diff --git a/backend/services/pairing.py b/backend/services/pairing.py new file mode 100644 index 00000000..6f70cca7 --- /dev/null +++ b/backend/services/pairing.py @@ -0,0 +1,201 @@ +"""Mobile device pairing service (V0 — bearer auth, no E2E payload encryption yet). + +Flow: + 1. Desktop UI (loopback) calls POST /pair/init → mints a single-use token + with a 5-minute TTL and returns a ``voicebox://pair?...`` URL. + 2. Mobile scans the QR (or pastes the URL) and POSTs /pair/complete with + the token + a human-readable device name. + 3. Server validates the token, mints a long-lived bearer, and stores + only SHA-256(bearer). The bearer plaintext is returned exactly once. + 4. Mobile saves the bearer in SecureStore. Subsequent calls carry + ``Authorization: Bearer ``. + +The bearer is returned exactly once. Server has no path to recover it; if +the device loses its key the user must re-pair (and revoke the old device +from Settings → Mobile if they want to be tidy). + +Phase 2 will layer XChaCha20-Poly1305 payload encryption + HKDF session +keys on top — see ``mobile/PLAN.md`` § Pairing & transport. The bearer +established here is the foundation either way. +""" + +import hashlib +import logging +import secrets +import socket +import subprocess +import uuid +from datetime import datetime, timedelta +from typing import Optional + +from sqlalchemy.orm import Session + +from ..database import PairedDevice, PairingToken +from ..models import ( + HostCandidate, + PairCompleteResponse, + PairInitResponse, + PairedDeviceResponse, +) + +logger = logging.getLogger(__name__) + +PAIRING_TOKEN_TTL = timedelta(minutes=5) +TOKEN_BYTES = 32 # urlsafe-b64 encoded → ~44 chars + + +def _hash_bearer(bearer: str) -> str: + return hashlib.sha256(bearer.encode("utf-8")).hexdigest() + + +def _build_pairing_url(token: str, host: str) -> str: + # ``host`` should be reachable from the mobile device — LAN IP, Tailscale + # 100.x address, MagicDNS hostname (e.g. ``mac.tail-xxxx.ts.net:17493``). + # The desktop UI is responsible for picking the right host; loopback is + # only useful for curl-driven local testing. + return f"voicebox://pair?host={host}&token={token}" + + +def init_pairing_token(db: Session, host: str) -> PairInitResponse: + """Mint a one-time pairing token. Caller must already be authorized as loopback.""" + token = secrets.token_urlsafe(TOKEN_BYTES) + expires_at = datetime.utcnow() + PAIRING_TOKEN_TTL + db.add(PairingToken(token=token, expires_at=expires_at)) + db.commit() + return PairInitResponse( + token=token, + expires_at=expires_at, + pairing_url=_build_pairing_url(token, host), + ) + + +def complete_pairing(db: Session, token: str, device_name: str) -> PairCompleteResponse: + """Exchange a pairing token for a long-lived device bearer. + + Raises ValueError on invalid / expired / already-used token. + """ + row = db.query(PairingToken).filter(PairingToken.token == token).first() + if row is None: + raise ValueError("Invalid pairing token") + if row.used_at is not None: + raise ValueError("Pairing token already used") + if row.expires_at < datetime.utcnow(): + raise ValueError("Pairing token expired") + + row.used_at = datetime.utcnow() + + bearer = secrets.token_urlsafe(TOKEN_BYTES) + device = PairedDevice( + id=str(uuid.uuid4()), + name=device_name.strip(), + bearer_hash=_hash_bearer(bearer), + ) + db.add(device) + db.commit() + + logger.info("Paired new device id=%s name=%s", device.id, device.name) + + return PairCompleteResponse( + device_id=device.id, + bearer=bearer, + device_name=device.name, + ) + + +def authenticate_bearer(db: Session, bearer: str) -> Optional[PairedDevice]: + """Look up a paired device by bearer. Bumps last_seen_at on success.""" + if not bearer: + return None + bearer_hash = _hash_bearer(bearer) + device = ( + db.query(PairedDevice) + .filter(PairedDevice.bearer_hash == bearer_hash, PairedDevice.revoked.is_(False)) + .first() + ) + if device is not None: + device.last_seen_at = datetime.utcnow() + db.commit() + return device + + +def list_devices(db: Session) -> list[PairedDeviceResponse]: + """Return all paired devices (revoked included) for the desktop UI.""" + rows = db.query(PairedDevice).order_by(PairedDevice.created_at.desc()).all() + return [PairedDeviceResponse.model_validate(r) for r in rows] + + +def revoke_device(db: Session, device_id: str) -> None: + """Mark a device as revoked. Idempotent on repeat calls.""" + device = db.query(PairedDevice).filter(PairedDevice.id == device_id).first() + if device is None: + raise ValueError("Device not found") + device.revoked = True + db.commit() + logger.info("Revoked device id=%s name=%s", device.id, device.name) + + +# --- Host discovery --------------------------------------------------------- + +def _get_lan_ip() -> Optional[str]: + """Best-effort outbound IPv4 of the host. Uses the UDP-connect trick — + no packets are actually sent; the kernel just picks the source IP it + would use to reach 8.8.8.8. + """ + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect(("8.8.8.8", 80)) + return s.getsockname()[0] + except OSError: + return None + + +def _get_tailscale_ip() -> Optional[str]: + """Return the host's Tailscale 100.x address if Tailscale is installed + and reports one. Returns ``None`` on any failure — Tailscale is optional. + + Tries ``tailscale`` on PATH first, then falls back to the Mac App Store + install location (the macOS App bundle doesn't symlink onto PATH by + default, only into the user's shell init via an alias subprocess can't see). + """ + candidate_binaries = [ + "tailscale", + "/Applications/Tailscale.app/Contents/MacOS/Tailscale", + ] + for binary in candidate_binaries: + try: + result = subprocess.run( + [binary, "ip", "--4"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + continue + if result.returncode != 0: + continue + ip = result.stdout.strip().splitlines()[0].strip() if result.stdout else "" + if ip: + return ip + return None + + +def list_host_candidates(port: int) -> list[HostCandidate]: + """Return suggested host strings the desktop can embed in the QR. + + Order matters — the UI defaults to the first non-loopback entry. + """ + candidates: list[HostCandidate] = [] + lan_ip = _get_lan_ip() + if lan_ip and not lan_ip.startswith("127."): + candidates.append( + HostCandidate(address=f"{lan_ip}:{port}", label="Local network", kind="lan") + ) + tailscale_ip = _get_tailscale_ip() + if tailscale_ip and tailscale_ip != lan_ip: + candidates.append( + HostCandidate(address=f"{tailscale_ip}:{port}", label="Tailscale", kind="tailscale") + ) + candidates.append( + HostCandidate(address=f"127.0.0.1:{port}", label="Loopback (testing only)", kind="loopback") + ) + return candidates diff --git a/backend/utils/auth.py b/backend/utils/auth.py new file mode 100644 index 00000000..78c0cbb8 --- /dev/null +++ b/backend/utils/auth.py @@ -0,0 +1,103 @@ +"""FastAPI dependencies for the V0 mobile-pair auth model. + +Two dependencies are exposed: + +* ``require_loopback`` — reject calls that don't originate from a loopback + address. Used to gate desktop-only admin endpoints (pair init, devices + list, revoke). Loopback callers stay unauthenticated everywhere else + too — the desktop app talks to its own backend over 127.0.0.1. + +* ``require_paired_device`` — validate ``Authorization: Bearer `` + against the ``paired_devices`` table. Used to identify paired mobile + callers and bumps ``last_seen_at`` on success. + +Phase 2 will layer XChaCha20-Poly1305 payload encryption on top of the +bearer (see ``mobile/PLAN.md``); the bearer stays the identity primitive. +""" + +from typing import Optional + +from fastapi import Depends, HTTPException, Request, status +from sqlalchemy.orm import Session + +from ..database import PairedDevice, get_db +from ..services import pairing as pairing_service + +LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +def require_loopback(request: Request) -> None: + """Reject calls from non-loopback addresses.""" + client = request.client + host = client.host if client else None + if host not in LOOPBACK_HOSTS: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Loopback only", + ) + + +def _extract_bearer(request: Request) -> Optional[str]: + auth = request.headers.get("Authorization") or request.headers.get("authorization") + if not auth: + return None + parts = auth.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer": + return None + return parts[1].strip() + + +def require_paired_device( + request: Request, + db: Session = Depends(get_db), +) -> PairedDevice: + """Resolve the PairedDevice authenticated by the request bearer.""" + bearer = _extract_bearer(request) + if not bearer: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + device = pairing_service.authenticate_bearer(db, bearer) + if device is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or revoked bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + return device + + +def require_bearer_or_loopback( + request: Request, + db: Session = Depends(get_db), +) -> None: + """Loopback callers pass without auth; everyone else needs a paired bearer. + + Applied as a router-level dependency on user-data endpoints so the + desktop UI (which talks over 127.0.0.1) keeps its current friction-free + access while LAN-reachable callers must be a paired mobile device. + + The pre-pair endpoints (``POST /pair/complete``) and the desktop-only + admin endpoints (``POST /pair/init``, ``GET /devices``) intentionally + stay outside this gate — they have their own dependencies. + """ + client = request.client + host = client.host if client else None + if host in LOOPBACK_HOSTS: + return + bearer = _extract_bearer(request) + if not bearer: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + device = pairing_service.authenticate_bearer(db, bearer) + if device is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or revoked bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) diff --git a/bun.lock b/bun.lock index 157138af..b284ee3c 100644 --- a/bun.lock +++ b/bun.lock @@ -17,7 +17,7 @@ }, "app": { "name": "@voicebox/app", - "version": "0.4.2", + "version": "0.5.0", "dependencies": { "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", @@ -57,6 +57,7 @@ "react-dom": "^18.3.0", "react-hook-form": "^7.53.0", "react-i18next": "^17.0.4", + "react-qr-code": "^2.0.18", "react-sound-visualizer": "^1.4.0", "tailwind-merge": "^2.5.4", "wavesurfer.js": "^7.0.0", @@ -75,7 +76,7 @@ }, "landing": { "name": "@voicebox/landing", - "version": "0.4.2", + "version": "0.5.0", "dependencies": { "@fontsource/space-grotesk": "^5.2.10", "@icons-pack/react-simple-icons": "^13.13.0", @@ -104,7 +105,7 @@ }, "tauri": { "name": "@voicebox/tauri", - "version": "0.4.2", + "version": "0.5.0", "dependencies": { "@tauri-apps/api": "^2.0.0", "@tauri-apps/plugin-dialog": "^2.0.0", @@ -127,7 +128,7 @@ }, "web": { "name": "@voicebox/web", - "version": "0.4.2", + "version": "0.5.0", "dependencies": { "@tanstack/react-query": "^5.0.0", "react": "^18.3.0", @@ -991,6 +992,8 @@ "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "qr.js": ["qr.js@0.0.0", "", {}, "sha512-c4iYnWb+k2E+vYpRimHqSu575b1/wKl4XFeJGpFmrJQz5I88v9aY2czh7s0w36srfCM1sXgC/xpoJz5dJfq+OQ=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], "react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="], @@ -1005,6 +1008,8 @@ "react-loaders": ["react-loaders@3.0.1", "", { "dependencies": { "classnames": "^2.2.3" }, "peerDependencies": { "prop-types": ">=15.6.0", "react": ">=15" } }, "sha512-4igMNqs9Fb3d4Z+0UHIGQNJsw/37gX0nUO8QxupnEKRn1dtyYC1LGwk5GuaoDciMQCQc/MmPwb4Fn6ZfdoX1FQ=="], + "react-qr-code": ["react-qr-code@2.0.18", "", { "dependencies": { "prop-types": "^15.8.1", "qr.js": "0.0.0" }, "peerDependencies": { "react": "*" } }, "sha512-v1Jqz7urLMhkO6jkgJuBYhnqvXagzceg3qJUWayuCK/c6LTIonpWbwxR1f1APGd4xrW/QcQEovNrAojbUz65Tg=="], + "react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="], "react-remove-scroll": ["react-remove-scroll@2.7.2", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q=="], diff --git a/justfile b/justfile index d178d166..95da71f2 100644 --- a/justfile +++ b/justfile @@ -52,6 +52,10 @@ setup-python: if [ "$(uname -m)" = "arm64" ] && [ "$(uname)" = "Darwin" ]; then echo "Detected Apple Silicon — installing MLX dependencies..." {{ pip }} install -r {{ backend_dir }}/requirements-mlx.txt + # mlx-audio + mlx-lm are intentionally --no-deps (their transformers>=5 + # pin conflicts with our pinned 4.x). The runtime API surface we use + # works fine on transformers 4.57.x. See requirements-mlx.txt notes. + {{ pip }} install --no-deps mlx-audio==0.4.1 mlx-lm fi {{ pip }} install git+https://github.com/QwenLM/Qwen3-TTS.git {{ pip }} install pyinstaller ruff pytest pytest-asyncio -q @@ -100,6 +104,11 @@ setup-js: # ─── Development ────────────────────────────────────────────────────── # Start backend (if not already running) + frontend for development +# Binds the backend to 0.0.0.0 so paired mobile devices can reach it over the +# LAN/Tailscale. This exposes existing unauth routes (/generate, /transcribe, +# /captures, /profiles) to anyone on the same network — fine for a trusted +# home network, NOT fine on public Wi-Fi. Bearer-auth middleware on those +# routes is on the roadmap; until then, treat dev as LAN-trusted. [unix] dev: _ensure-venv _ensure-sidecar #!/usr/bin/env bash @@ -109,8 +118,8 @@ dev: _ensure-venv _ensure-sidecar if curl -sf http://127.0.0.1:17493/health > /dev/null 2>&1; then echo "Backend already running on http://localhost:17493" else - echo "Starting backend on http://localhost:17493 ..." - {{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 & + echo "Starting backend on http://0.0.0.0:17493 (LAN-reachable) ..." + {{ venv_bin }}/uvicorn backend.main:app --reload --host 0.0.0.0 --port 17493 & backend_pid=$! sleep 2 fi @@ -124,21 +133,21 @@ dev: _ensure-venv _ensure-sidecar dev: _ensure-venv _ensure-sidecar $backendJob = $null; \ try { $null = Invoke-WebRequest -Uri "http://127.0.0.1:17493/health" -UseBasicParsing -TimeoutSec 2 -ErrorAction Stop; Write-Host "Backend already running on http://localhost:17493" } catch { \ - Write-Host "Starting backend on http://localhost:17493 ..."; \ - $backendJob = Start-Process -PassThru -NoNewWindow -FilePath "{{ python }}" -ArgumentList "-m","uvicorn","backend.main:app","--reload","--port","17493"; \ + Write-Host "Starting backend on http://0.0.0.0:17493 (LAN-reachable) ..."; \ + $backendJob = Start-Process -PassThru -NoNewWindow -FilePath "{{ python }}" -ArgumentList "-m","uvicorn","backend.main:app","--reload","--host","0.0.0.0","--port","17493"; \ Start-Sleep -Seconds 2; \ }; \ Write-Host "Starting Tauri desktop app..."; \ try { Set-Location "{{ tauri_dir }}"; bun run tauri dev } finally { if ($backendJob) { taskkill /PID $backendJob.Id /T /F 2>$null | Out-Null } } -# Start backend only +# Start backend only — bound to 0.0.0.0 for mobile pairing (see `dev` above) [unix] dev-backend: _ensure-venv - {{ venv_bin }}/uvicorn backend.main:app --reload --port 17493 + {{ venv_bin }}/uvicorn backend.main:app --reload --host 0.0.0.0 --port 17493 [windows] dev-backend: _ensure-venv - & "{{ python }}" -m uvicorn backend.main:app --reload --port 17493 + & "{{ python }}" -m uvicorn backend.main:app --reload --host 0.0.0.0 --port 17493 # Start Tauri desktop app only (backend must be running separately) [unix] @@ -180,6 +189,15 @@ dev-web: _ensure-venv Write-Host "Starting web app..."; \ try { Set-Location "{{ web_dir }}"; bun run dev } finally { if ($backendJob) { taskkill /PID $backendJob.Id /T /F 2>$null | Out-Null } } +# Start Expo dev server for the mobile companion app (run `cd mobile && bun install` once first) +[unix] +dev-mobile: + cd mobile && bunx expo start + +[windows] +dev-mobile: + Set-Location "mobile"; bunx expo start + # Kill all dev processes [unix] kill: diff --git a/mobile/.gitignore b/mobile/.gitignore new file mode 100644 index 00000000..d914c328 --- /dev/null +++ b/mobile/.gitignore @@ -0,0 +1,41 @@ +# Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files + +# dependencies +node_modules/ + +# Expo +.expo/ +dist/ +web-build/ +expo-env.d.ts + +# Native +.kotlin/ +*.orig.* +*.jks +*.p8 +*.p12 +*.key +*.mobileprovision + +# Metro +.metro-health-check* + +# debug +npm-debug.* +yarn-debug.* +yarn-error.* + +# macOS +.DS_Store +*.pem + +# local env files +.env*.local + +# typescript +*.tsbuildinfo + +# generated native folders +/ios +/android diff --git a/mobile/PLAN.md b/mobile/PLAN.md new file mode 100644 index 00000000..176a1ffe --- /dev/null +++ b/mobile/PLAN.md @@ -0,0 +1,139 @@ +# Voicebox Mobile — V1 plan + +A companion app for the Voicebox desktop. iPhone-first, dictate-anywhere, Captures as the hero. Talks to a paired desktop over Tailscale or LAN with end-to-end encryption. + +V1 is entirely local — no cloud, no account. The device key minted during pairing is the root of the user's lifetime encryption identity and gets reused by the cloud phases that follow; see [`docs/plans/CLOUD_ROADMAP.md`](../docs/plans/CLOUD_ROADMAP.md) for the post-mobile arc (backup & sync → private inference → marketplace). + +--- + +## Repo layout + +- New `mobile/` at repo root, sibling to `app/`, `backend/`, `tauri/`, `landing/`, `web/` +- Standalone Expo project — not a Bun workspace member (avoids React/Tauri/Bun version drag) +- Type-sharing via OpenAPI: generate `mobile/src/api/types.ts` from backend's `/openapi.json`, commit it, regenerate on demand +- Branch off `main` once 0.5.0 ships → `feat/mobile-app` + +## Stack (verified 2026-04-25) + +### The SDK 54 vs SDK 55 fork + +Expo SDK 55 is the current `latest` (`expo@55.0.17`, React Native 0.83, React 19.2, New Architecture only — Legacy Architecture was dropped in 55). It also ships Expo Router v7, Hermes v1 with bytecode diffing, and `expo-brownfield`. But: **NativeWind v5 is the only NativeWind line that targets SDK 55, and v5 is still pre-release** (`5.0.0-preview.3`, with explicit "not intended for production use" warning in the v5 docs). NativeWind v4 stable (`4.2.3`) is paired with SDK 54. + +Two real options: + +- **Option A — Ship-fast (recommended for V1):** Expo SDK 54 + NativeWind v4.2.3. Both stable, official pairing, well-documented. Loses SDK 55's bytecode-diff updates and Expo Router v7 sugar but everything works. +- **Option B — Cutting-edge:** Expo SDK 55.0.17 + NativeWind 5.0.0-preview.3. Latest everything, but the NativeWind v5 maintainer explicitly says it's for experimentation. More breakage during dev, brittle CI. + +Recommendation: **Option A**. We're trying to ship a companion app, not stress-test pre-release styling layers. We can bump to SDK 55 + NativeWind v5 once both stabilize (Expo targets stable SDK 55 mid-2026). + +### Pinned versions (assuming Option A) + +| Package | Version | Notes | +| --- | --- | --- | +| `expo` | `~54` (latest 54.x) | New Architecture default since SDK 51 | +| `expo-router` | bundled with SDK | file-based, typed routes | +| `nativewind` | `4.2.3` | Tailwind class parity with `app/` | +| `expo-audio` | `~54` (bundled with SDK) | stable in SDK 54+, replaces `expo-av` recording | +| `@shopify/react-native-skia` | `2.6.2` | live waveform; WaveSurfer is DOM-only | +| `expo-camera` | `~54` (bundled) | QR scan (barcode scanning built in) | +| `expo-secure-store` | `~54` (bundled) | paired device key | +| `react-native-reanimated` | `4.3.0` | rewritten for new arch | +| `react-native-gesture-handler` | `2.31.1` | | +| `zustand` | `5.0.12` | mirrors desktop | +| `@tanstack/react-query` | `5.100.5` | mirrors desktop | + +(Versions for `expo-*` packages are managed by `npx expo install`, which picks the patch that matches the SDK — don't pin them by hand.) + +### Other stack decisions + +- **TypeScript strict** +- Theme tokens copied straight from `app/`'s shadcn theme (`hsl(43 60% 50%)` for the gold accent, dark surfaces match) +- **EAS Build** + **Dev Client** from day one — Skia and SecureStore push us off Expo Go + +## Pairing & transport — Tailscale-friendly + +1. Desktop: new **Settings → Mobile** with "Pair device" → renders QR + 6-digit fallback +2. QR payload: `voicebox://pair?host=&secret=&fp=` + - `host` = whatever address the user can reach: LAN IP (`192.168.x.x:17493`), Tailscale 100.x address, or MagicDNS name (`mac.tail-xxxx.ts.net:17493`) — Tailscale Just Works with zero extra code + - `secret` = one-time pairing token; mobile exchanges it for a long-lived device key on first request + - `fp` = self-signed cert fingerprint we mint at pair time, pinned on mobile +3. Auth: bearer token + XChaCha20-Poly1305 payload encryption with HKDF per-session keys — E2E layer above HTTP, survives any future cloud relay swap + +## Backend additions (desktop, separate PR before mobile work) + +- `POST /pair/init` — mint pairing token, return QR payload +- `POST /pair/complete` — exchange token for device-bound long-lived key, persist `paired_devices` row +- `GET/DELETE /devices` — Settings → Mobile lists & revokes paired devices, with `last_seen_at` +- Bearer middleware on `/generate`, `/transcribe`, `/profiles`, `/captures`, `/speak` (loopback callers stay unauthenticated as today; paired-device callers use the bearer) +- `/captures` upload accepts `m4a` (expo-audio's iOS default) in addition to existing formats + +## Screens + +### First-run + +Pair flow: scan QR → confirm desktop name + fingerprint → store creds in SecureStore. + +### Tab 1 — Generate + +- Profile cards horizontal scroll (top) +- Recent generations list (middle) — tap to play, long-press for version picker / regenerate / share +- Floating generate box (bottom): text input + engine indicator + speak button + +### Tab 2 — Voices + +- List of profiles (cloned + presets), grouped by engine compatibility +- Tap to inspect: samples (Skia waveform thumbs), language, last used +- **No** profile creation in V1 + +### Tab 3 — Captures (the hero) + +- **Big gold mic button** bottom-front-and-center — same accent as the sponsor CTA +- Tap-to-toggle in V1 (push-to-talk fights iOS gestures, defer) +- Live mic waveform via Skia, amplitude polled at 30 Hz, scrolling buffer +- Transcript text area above the waveform — empty during recording, populated on stop +- State pill at top: `recording → uploading → transcribing → refining → done` (mirrors desktop pill semantics) +- Captures list below: each row has a Skia mini-waveform, tap to expand, scrub, edit transcript inline, "Play as voice profile" +- **Audio preserved + downloadable** — visually obvious in the UI; this is the USP +- Schema/UI built so resumable capture can land later without re-architecting (no "one capture = one continuous take" assumptions baked in) + +## Out of scope for V1 (deliberately) + +Stories editor · Voice profile creation / sample recording · Effects editor · Personality LLM controls · Streaming transcription (lands with resumable capture in V2) · Settings beyond pairing · Android (pipeline supports it; QA focus iOS first) + +## V2+ candidates + +- **Resumable capture** — pause/resume that actually appends audio AND transcript (Apple Voice Notes drops the second-half transcript on resume; Voicebox shouldn't) +- **Document mode** — refinement LLM writes/edits a markdown document live as you speak, including dictated edits ("change the second bullet to…") +- **Streaming transcription** — partial transcripts during recording +- **Voice profile creation on mobile** — record samples directly +- **Android** +- **Cloud relay** — once the Voicebox platform exists, the same E2E layer rides over a relay so pairing isn't tied to Tailscale/LAN + +## Build/dev workflow + +```bash +cd mobile +bun install +bunx expo prebuild # generate native projects (Skia + SecureStore need it) +bunx expo run:ios # device on the same Tailnet +eas build --profile development # distributable Dev Client for TestFlight +``` + +Bundle ID: `sh.voicebox.mobile` (need to register in App Store Connect). + +## Order of attack + +1. Backend: pairing endpoints + bearer middleware (desktop PR, can land before mobile) +2. Mobile: Expo scaffold + NativeWind + theme tokens + Pair screen +3. Mobile: Captures tab end-to-end — validates transport, E2E layer, audio upload, waveform, pill states all in one flow +4. Mobile: Generate tab +5. Mobile: Voices tab +6. EAS dev build + TestFlight internal track + +## Open questions + +1. **iOS only V1?** Default: yes. +2. **NativeWind or hand-rolled styles?** Default: NativeWind — class parity with `app/` is worth the small bundler tax. +3. **Crib patterns from the Spacedrive mobile app first**, or start clean from current Expo docs? +4. **Bundle ID + display name** — confirm `sh.voicebox.mobile` / "Voicebox" before EAS setup. +5. **Pair screen UX** — QR-only, or always offer the 6-digit code as an a11y/fallback? diff --git a/mobile/app.json b/mobile/app.json new file mode 100644 index 00000000..0b627946 --- /dev/null +++ b/mobile/app.json @@ -0,0 +1,44 @@ +{ + "expo": { + "name": "Voicebox", + "slug": "voicebox", + "scheme": "voicebox", + "version": "0.1.0", + "orientation": "portrait", + "icon": "./assets/icon.png", + "userInterfaceStyle": "dark", + "newArchEnabled": true, + "splash": { + "image": "./assets/splash-icon.png", + "resizeMode": "contain", + "backgroundColor": "#0F0F0F" + }, + "ios": { + "supportsTablet": true, + "bundleIdentifier": "sh.voicebox.mobile" + }, + "android": { + "package": "sh.voicebox.mobile", + "adaptiveIcon": { + "foregroundImage": "./assets/adaptive-icon.png", + "backgroundColor": "#0F0F0F" + }, + "edgeToEdgeEnabled": true, + "predictiveBackGestureEnabled": false + }, + "web": { + "favicon": "./assets/favicon.png" + }, + "plugins": [ + "expo-router", + "expo-secure-store", + [ + "expo-camera", + { + "cameraPermission": "Voicebox uses the camera to scan the pairing QR shown by your desktop." + } + ], + "expo-audio" + ] + } +} diff --git a/mobile/app/(tabs)/_layout.tsx b/mobile/app/(tabs)/_layout.tsx new file mode 100644 index 00000000..7110f18c --- /dev/null +++ b/mobile/app/(tabs)/_layout.tsx @@ -0,0 +1,56 @@ +import { Redirect, Tabs } from 'expo-router'; +import { Mic, PenTool, Users } from 'lucide-react-native'; +import { View } from 'react-native'; +import { colors } from '@/lib/colors'; +import { useSession } from '@/lib/session'; + +export default function TabsLayout() { + const { session, hydrated } = useSession(); + + // Wait for SecureStore hydration before deciding where to send the user. + // Without this we'd briefly render the tabs against a null session and + // every authenticated query would fire with no bearer. + if (!hydrated) { + return ; + } + if (!session) return ; + + return ( + + , + }} + /> + , + }} + /> + , + }} + /> + + ); +} diff --git a/mobile/app/(tabs)/generate.tsx b/mobile/app/(tabs)/generate.tsx new file mode 100644 index 00000000..6839032d --- /dev/null +++ b/mobile/app/(tabs)/generate.tsx @@ -0,0 +1,323 @@ +import { useAudioPlayer, useAudioPlayerStatus } from 'expo-audio'; +import { Loader2, Pause, Play, Sparkles, Volume2 } from 'lucide-react-native'; +import { useEffect, useMemo, useState } from 'react'; +import { + ActivityIndicator, + FlatList, + Pressable, + RefreshControl, + ScrollView, + Text, + TextInput, + View, +} from 'react-native'; +import { SafeAreaView } from 'react-native-safe-area-context'; +import { Button } from '@/components/ui/Button'; +import { + authHeaders, + buildGenerationAudioUrl, + type HistoryResponse, + type VoiceProfileResponse, +} from '@/lib/api'; +import { colors } from '@/lib/colors'; +import { useGenerate, useGenerationPolling, useHistory, useProfiles } from '@/lib/hooks'; +import { useSession } from '@/lib/session'; + +export default function GenerateTab() { + const session = useSession((s) => s.session); + const profiles = useProfiles(); + const history = useHistory(20); + const generateMutation = useGenerate(); + + const [selectedProfileId, setSelectedProfileId] = useState(null); + const [text, setText] = useState(''); + const [pendingId, setPendingId] = useState(null); + const [nowPlayingId, setNowPlayingId] = useState(null); + + const polling = useGenerationPolling(pendingId); + const playingSource = useMemo(() => { + if (!session || !nowPlayingId) return null; + return { + uri: buildGenerationAudioUrl(session, nowPlayingId), + headers: authHeaders(session), + }; + }, [session, nowPlayingId]); + const player = useAudioPlayer(playingSource ?? null); + const playerStatus = useAudioPlayerStatus(player); + + // Default-select first profile when the list arrives. + useEffect(() => { + if (selectedProfileId === null && profiles.data && profiles.data.length > 0) { + setSelectedProfileId(profiles.data[0].id); + } + }, [profiles.data, selectedProfileId]); + + // When the polled generation completes, auto-play it and clear pending. + useEffect(() => { + const data = polling.data; + if (!data) return; + if (data.status === 'completed') { + setNowPlayingId(data.id); + setPendingId(null); + } else if (data.status === 'failed') { + setPendingId(null); + } + }, [polling.data]); + + // Auto-play when a new source comes in. + useEffect(() => { + if (!playingSource) return; + const t = window.setTimeout(() => { + try { + player.play(); + } catch { + // ignore — player may not be ready yet + } + }, 100); + return () => window.clearTimeout(t); + }, [playingSource, player]); + + const selectedProfile = profiles.data?.find((p) => p.id === selectedProfileId) ?? null; + const isGenerating = + generateMutation.isPending || + (pendingId !== null && polling.data?.status !== 'completed'); + + function handleGenerate() { + if (!selectedProfile || !text.trim() || isGenerating) return; + generateMutation.mutate( + { + profile_id: selectedProfile.id, + text: text.trim(), + language: selectedProfile.language, + engine: selectedProfile.default_engine ?? undefined, + }, + { + onSuccess: (gen) => { + setPendingId(gen.id); + setText(''); + }, + }, + ); + } + + function handlePlayHistory(item: HistoryResponse) { + if (item.status !== 'completed') return; + if (nowPlayingId === item.id) { + // Toggle play/pause + if (playerStatus.playing) player.pause(); + else player.play(); + return; + } + setNowPlayingId(item.id); + } + + return ( + + {/* Header */} + + Generate + + + {/* Profile picker */} + + {profiles.isLoading ? ( + + + + ) : profiles.data && profiles.data.length > 0 ? ( + + {profiles.data.map((p) => ( + setSelectedProfileId(p.id)} + /> + ))} + + ) : ( + + + No profiles found on this desktop. Create one in Voicebox first. + + + )} + + + {/* Text input */} + + + + + + {text.length > 0 ? `${text.length} chars` : ' '} + +