Mobile companion app + paired-device backend

New iOS-first companion (Expo SDK 54 + NativeWind v4) with three tabs:
Captures (the hero — floating gold mic, live mic-meter waveform,
expand-row playback), Generate (profile picker + speak + autoplay +
recent), and Voices (searchable profile list).

Pairing (V0): backend mints a one-time token, mobile scans/pastes the
voicebox:// URL, server returns a long-lived bearer it stores only as a
SHA-256 hash. Bearer-or-loopback auth applied to every user-data router
so binding 0.0.0.0 doesn't leak existing endpoints. Loopback callers
(the desktop app) keep their friction-free access.

Desktop Settings → Mobile pane: live host picker (LAN / Tailscale auto-
detected via the App-bundle binary path on macOS), QR rendering,
5-minute expiry countdown, copyable URL fallback, paired-device list
with revoke. Auto-closes when a new device pairs.

just dev now binds the backend to 0.0.0.0 so paired phones can reach
it — and just setup-python pins mlx-audio==0.4.1 + mlx-lm so fresh
Apple Silicon worktrees get a working STT path on first install.
This commit is contained in:
James Pine
2026-04-25 17:09:32 -07:00
parent 2bcb98d1a8
commit f4d21504e3
49 changed files with 5229 additions and 30 deletions
+108
View File
@@ -0,0 +1,108 @@
import { forwardRef } from 'react';
import {
ActivityIndicator,
Pressable,
type PressableProps,
Text,
View,
} from 'react-native';
import { colors } from '@/lib/colors';
type Variant = 'primary' | 'secondary' | 'ghost';
type Size = 'sm' | 'md' | 'lg';
type Props = Omit<PressableProps, 'children'> & {
label: string;
variant?: Variant;
size?: Size;
loading?: boolean;
leftSlot?: React.ReactNode;
rightSlot?: React.ReactNode;
};
const containerBase =
'flex-row items-center justify-center rounded-full active:opacity-90';
const variantContainer: Record<Variant, string> = {
primary: 'bg-accent',
secondary: 'bg-card border border-border',
ghost: 'bg-transparent',
};
const variantLabel: Record<Variant, string> = {
primary: 'text-accent-foreground',
secondary: 'text-foreground',
ghost: 'text-muted-foreground',
};
const sizeContainer: Record<Size, string> = {
sm: 'px-4 py-2 gap-1.5',
md: 'px-6 py-3 gap-2',
lg: 'px-8 py-4 gap-2.5',
};
const sizeLabel: Record<Size, string> = {
sm: 'text-xs',
md: 'text-sm',
lg: 'text-base',
};
// Mirrors the desktop CTA in landing/src/components/SponsorPromo.tsx —
// rounded-full, gold accent, uppercase semibold label with wide tracking.
export const Button = forwardRef<View, Props>(function Button(
{
label,
variant = 'primary',
size = 'md',
loading = false,
leftSlot,
rightSlot,
disabled,
style,
...rest
},
ref,
) {
const isPrimary = variant === 'primary';
const isDisabled = disabled || loading;
return (
<Pressable
ref={ref}
accessibilityRole="button"
accessibilityState={{ disabled: isDisabled, busy: loading }}
disabled={isDisabled}
style={[
isPrimary && {
shadowColor: colors.accent,
shadowOffset: { width: 0, height: 4 },
shadowOpacity: 0.35,
shadowRadius: 16,
elevation: 6,
},
isDisabled && { opacity: 0.5 },
style as object,
]}
className={`${containerBase} ${variantContainer[variant]} ${sizeContainer[size]}`}
{...rest}
>
{loading ? (
<ActivityIndicator
size="small"
color={isPrimary ? colors.accentForeground : colors.foreground}
/>
) : (
<>
{leftSlot}
<Text
className={`font-semibold uppercase ${variantLabel[variant]} ${sizeLabel[size]}`}
style={{ letterSpacing: 1.2 }}
>
{label}
</Text>
{rightSlot}
</>
)}
</Pressable>
);
});
+78
View File
@@ -0,0 +1,78 @@
import { useEffect, useState } from 'react';
import { View } from 'react-native';
import { colors } from '@/lib/colors';
const BAR_COUNT = 36;
const MIN_BAR = 0.1;
/**
* Scrolling-buffer waveform driven by mic metering. Each metering tick
* pushes a new bar onto the right and drops the oldest from the left, so
* the waveform reads as a 36-bar sliding window of the last few seconds
* of audio. Pure React Native — no Skia, no native modules.
*
* ``levelDb`` should be the value out of ``RecorderState.metering`` (a
* dB-scale instantaneous level, typically -160 to 0).
*/
export function LiveWaveform({
active,
levelDb,
}: {
active: boolean;
levelDb: number | undefined;
}) {
const [buffer, setBuffer] = useState<number[]>(() => Array(BAR_COUNT).fill(MIN_BAR));
// Push a new bar each time the metering value changes (which happens
// every poll interval — set to ~80ms in dictation.ts).
useEffect(() => {
if (!active) return;
const norm = normalize(levelDb);
setBuffer((prev) => [...prev.slice(1), norm]);
}, [levelDb, active]);
// Decay back to flat when not recording.
useEffect(() => {
if (active) return;
setBuffer(Array(BAR_COUNT).fill(MIN_BAR));
}, [active]);
return (
<View
className="flex-row items-center justify-center"
style={{ height: 48, gap: 3 }}
>
{buffer.map((h, i) => (
<View
key={i}
style={{
width: 3,
height: `${Math.round(h * 100)}%`,
backgroundColor: active ? colors.destructive : colors.mutedForeground,
borderRadius: 1.5,
opacity: active ? 0.85 : 0.4,
}}
/>
))}
</View>
);
}
/**
* Map dB (-160 → 0) to display height (0.1 → 1.0).
*
* Speech typically lives in the -40 dB → -10 dB range; we compress that
* into the visible part of the bar so quiet speech still moves the bars
* visibly. Returns MIN_BAR for null/undefined so a missing metering field
* just shows a flat baseline rather than breaking the waveform.
*/
function normalize(db: number | undefined): number {
if (db == null || !Number.isFinite(db)) return MIN_BAR;
// Clamp to a useful range and normalize.
const clamped = Math.max(-60, Math.min(0, db));
// Linear in dB (close enough — psychoacoustic-correct curves are V2).
const linear = (clamped + 60) / 60;
// Power curve to favor mid-range — keeps the bars feeling responsive.
const shaped = Math.pow(linear, 1.6);
return Math.max(MIN_BAR, Math.min(1, shaped));
}
+262
View File
@@ -0,0 +1,262 @@
import type { StoredSession } from './storage';
// Voicebox HTTP client. V0 transport is plain HTTP-over-LAN/Tailscale with a
// bearer header. Phase 2 will wrap payloads in XChaCha20-Poly1305 — this
// module is the single chokepoint for that future change.
export class ApiError extends Error {
constructor(public status: number, message: string) {
super(message);
this.name = 'ApiError';
}
}
export type PairCompleteResponse = {
device_id: string;
bearer: string;
device_name: string;
};
export type MeResponse = {
device_id: string;
device_name: string;
last_seen_at: string | null;
};
function buildUrl(host: string, path: string): string {
// host may include a port (e.g. "192.168.1.5:17494"); just slap http:// on.
// Phase 2 should require https or self-signed-with-pinned-fp; for V0 plain
// HTTP over a trusted LAN/Tailnet is the deal we made.
return `http://${host}${path}`;
}
async function parseError(res: Response): Promise<string> {
try {
const body = await res.json();
if (typeof body?.detail === 'string') return body.detail;
return JSON.stringify(body);
} catch {
return res.statusText || `HTTP ${res.status}`;
}
}
// --- Unauthenticated (pre-pair) ---------------------------------------------
export async function completePair(
host: string,
token: string,
deviceName: string,
): Promise<PairCompleteResponse> {
const res = await fetch(buildUrl(host, '/pair/complete'), {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ token, device_name: deviceName }),
});
if (!res.ok) throw new ApiError(res.status, await parseError(res));
return (await res.json()) as PairCompleteResponse;
}
// --- Authenticated (post-pair) ----------------------------------------------
async function authedFetch(
session: StoredSession,
path: string,
init: RequestInit = {},
): Promise<Response> {
const headers = new Headers(init.headers);
headers.set('Authorization', `Bearer ${session.bearer}`);
if (init.body && !headers.has('Content-Type') && !(init.body instanceof FormData)) {
headers.set('Content-Type', 'application/json');
}
const res = await fetch(buildUrl(session.host, path), { ...init, headers });
if (!res.ok) throw new ApiError(res.status, await parseError(res));
return res;
}
export async function getMe(session: StoredSession): Promise<MeResponse> {
const res = await authedFetch(session, '/me');
return (await res.json()) as MeResponse;
}
// --- Captures ---------------------------------------------------------------
export type CaptureSource = 'dictation' | 'recording' | 'file';
export type CaptureResponse = {
id: string;
audio_path: string;
source: CaptureSource;
language: string | null;
duration_ms: number | null;
transcript_raw: string;
transcript_refined: string | null;
stt_model: string | null;
llm_model: string | null;
created_at: string;
};
export type CaptureListResponse = {
items: CaptureResponse[];
total: number;
};
export type CaptureCreateResponse = CaptureResponse & {
auto_refine: boolean;
allow_auto_paste: boolean;
};
export async function listCaptures(
session: StoredSession,
opts: { limit?: number; offset?: number } = {},
): Promise<CaptureListResponse> {
const params = new URLSearchParams();
if (opts.limit != null) params.set('limit', String(opts.limit));
if (opts.offset != null) params.set('offset', String(opts.offset));
const qs = params.toString();
const res = await authedFetch(session, `/captures${qs ? `?${qs}` : ''}`);
return (await res.json()) as CaptureListResponse;
}
export async function uploadCapture(
session: StoredSession,
audioUri: string,
opts: { filename?: string; mimeType?: string; source?: CaptureSource } = {},
): Promise<CaptureCreateResponse> {
const filename = opts.filename ?? 'capture.m4a';
const mimeType = opts.mimeType ?? 'audio/m4a';
const source = opts.source ?? 'dictation';
const form = new FormData();
// React Native's FormData accepts {uri, name, type} for file fields; the
// typing pretends it doesn't, so we cast.
form.append('file', {
uri: audioUri,
name: filename,
type: mimeType,
} as unknown as Blob);
form.append('source', source);
const res = await authedFetch(session, '/captures', {
method: 'POST',
body: form,
});
return (await res.json()) as CaptureCreateResponse;
}
export async function deleteCapture(
session: StoredSession,
captureId: string,
): Promise<void> {
await authedFetch(session, `/captures/${captureId}`, { method: 'DELETE' });
}
export function buildCaptureAudioUrl(session: StoredSession, captureId: string): string {
return buildUrl(session.host, `/captures/${captureId}/audio`);
}
// --- Voice profiles ---------------------------------------------------------
export type VoiceProfileResponse = {
id: string;
name: string;
description: string | null;
language: string;
avatar_path: string | null;
voice_type: 'cloned' | 'preset' | 'designed';
preset_engine: string | null;
preset_voice_id: string | null;
default_engine: string | null;
generation_count: number;
sample_count: number;
created_at: string;
updated_at: string;
};
export async function listProfiles(session: StoredSession): Promise<VoiceProfileResponse[]> {
const res = await authedFetch(session, '/profiles');
return (await res.json()) as VoiceProfileResponse[];
}
// --- Generations ------------------------------------------------------------
export type GenerationStatus = 'pending' | 'completed' | 'failed';
export type GenerationResponse = {
id: string;
profile_id: string;
text: string;
language: string;
audio_path: string | null;
duration: number | null;
seed: number | null;
engine: string | null;
status: GenerationStatus | string;
error: string | null;
is_favorited: boolean;
created_at: string;
};
export type HistoryResponse = GenerationResponse & {
profile_name: string;
};
export type HistoryListResponse = {
items: HistoryResponse[];
total: number;
};
export type GenerateRequest = {
profile_id: string;
text: string;
language?: string;
engine?: string;
seed?: number;
personality?: boolean;
};
export async function generate(
session: StoredSession,
req: GenerateRequest,
): Promise<GenerationResponse> {
const res = await authedFetch(session, '/generate', {
method: 'POST',
body: JSON.stringify(req),
});
return (await res.json()) as GenerationResponse;
}
export async function getHistoryItem(
session: StoredSession,
generationId: string,
): Promise<HistoryResponse> {
const res = await authedFetch(session, `/history/${generationId}`);
return (await res.json()) as HistoryResponse;
}
export async function listHistory(
session: StoredSession,
opts: { limit?: number; offset?: number; profile_id?: string } = {},
): Promise<HistoryListResponse> {
const params = new URLSearchParams();
if (opts.limit != null) params.set('limit', String(opts.limit));
if (opts.offset != null) params.set('offset', String(opts.offset));
if (opts.profile_id) params.set('profile_id', opts.profile_id);
const qs = params.toString();
const res = await authedFetch(session, `/history${qs ? `?${qs}` : ''}`);
return (await res.json()) as HistoryListResponse;
}
export function buildGenerationAudioUrl(
session: StoredSession,
generationId: string,
): string {
return buildUrl(session.host, `/audio/${generationId}`);
}
/**
* Authentication header for streaming sources where a separate fetch isn't
* possible (expo-audio's source object accepts headers directly).
*/
export function authHeaders(session: StoredSession): Record<string, string> {
return { Authorization: `Bearer ${session.bearer}` };
}
+20
View File
@@ -0,0 +1,20 @@
// Color tokens mirroring the desktop dark theme (`app/src/index.css` `.dark`).
// Use NativeWind classes (`bg-accent`, `text-foreground`, …) in components;
// this constants file is for places NativeWind doesn't reach — StatusBar tint,
// native module props, Skia, animated colors.
export const colors = {
background: '#0F0F0F', // hsl(0 0% 6%)
foreground: '#F2F2F2', // hsl(0 0% 95%)
card: '#141414', // hsl(0 0% 8%)
border: '#1F1F1F', // hsl(0 0% 12%)
muted: '#1F1F1F',
mutedForeground: '#999999', // hsl(0 0% 60%)
accent: '#AC8C39', // hsl(43 50% 45%) — the gold
accentFaint: '#916F2D', // hsl(43 50% 38%)
accentForeground: '#F2F2F2',
destructive: '#CD3535', // hsl(0 62.8% 50%)
ring: '#666666', // hsl(0 0% 40%)
} as const;
export type ColorKey = keyof typeof colors;
+105
View File
@@ -0,0 +1,105 @@
import {
AudioModule,
RecordingPresets,
setAudioModeAsync,
useAudioRecorder,
useAudioRecorderState,
} from 'expo-audio';
import { useCallback, useState } from 'react';
import { uploadCapture, type CaptureCreateResponse } from './api';
import { useSession } from './session';
export type DictationPhase = 'idle' | 'recording' | 'processing' | 'error';
export type DictationFlow = {
phase: DictationPhase;
durationSec: number;
/** Instantaneous mic level in dB (typically -160 → 0). undefined when not recording or metering disabled. */
meteringDb: number | undefined;
error: string | null;
start: () => Promise<void>;
stop: () => Promise<CaptureCreateResponse | null>;
isBusy: boolean;
};
// HIGH_QUALITY + metering enabled so the screen can render a live waveform
// while recording. Metering is per-poll-interval, so we ask for ~80ms ticks.
const RECORDING_OPTIONS = {
...RecordingPresets.HIGH_QUALITY,
isMeteringEnabled: true,
};
const RECORDER_POLL_INTERVAL_MS = 80;
/**
* Wraps expo-audio's recorder + the /captures upload into a single
* lifecycle: tap-to-start → tap-to-stop → upload → transcribed capture.
*
* Synchronous from the caller's POV: ``stop()`` resolves with the new
* capture (transcript and all) or null on failure.
*/
export function useDictation(): DictationFlow {
const session = useSession((s) => s.session);
const recorder = useAudioRecorder(RECORDING_OPTIONS);
const recState = useAudioRecorderState(recorder, RECORDER_POLL_INTERVAL_MS);
const [phase, setPhase] = useState<DictationPhase>('idle');
const [error, setError] = useState<string | null>(null);
const start = useCallback(async () => {
if (!session) {
setError('Not paired');
setPhase('error');
return;
}
setError(null);
try {
const perm = await AudioModule.requestRecordingPermissionsAsync();
if (!perm.granted) {
setError('Microphone permission denied');
setPhase('error');
return;
}
await setAudioModeAsync({ allowsRecording: true, playsInSilentMode: true });
await recorder.prepareToRecordAsync();
recorder.record();
setPhase('recording');
} catch (e) {
setError(e instanceof Error ? e.message : String(e));
setPhase('error');
}
}, [recorder, session]);
const stop = useCallback(async (): Promise<CaptureCreateResponse | null> => {
if (!session) return null;
setPhase('processing');
try {
await recorder.stop();
const uri = recorder.uri;
if (!uri) throw new Error('No audio captured');
const result = await uploadCapture(session, uri, {
filename: `dictation-${Date.now()}.m4a`,
mimeType: 'audio/m4a',
source: 'dictation',
});
setPhase('idle');
return result;
} catch (e) {
setError(
e instanceof Error
? e.message
: 'Upload failed — is the desktop reachable?',
);
setPhase('error');
return null;
}
}, [recorder, session]);
return {
phase,
durationSec: Math.floor((recState.durationMillis ?? 0) / 1000),
meteringDb: recState.metering,
error,
start,
stop,
isBusy: phase === 'processing',
};
}
+110
View File
@@ -0,0 +1,110 @@
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import {
generate,
getHistoryItem,
listCaptures,
listHistory,
listProfiles,
type CaptureListResponse,
type GenerateRequest,
type GenerationResponse,
type HistoryListResponse,
type HistoryResponse,
type VoiceProfileResponse,
} from './api';
import { useSession } from './session';
export const CAPTURES_KEY = ['captures'] as const;
export const PROFILES_KEY = ['profiles'] as const;
export const HISTORY_KEY = ['history'] as const;
/**
* Paginated capture list. V0 just fetches the first 50 and refetches on
* window focus / explicit invalidation. Pagination + virtualized list
* land when the user actually accumulates enough captures to need them.
*/
export function useCaptures() {
const session = useSession((s) => s.session);
return useQuery({
queryKey: CAPTURES_KEY,
queryFn: async (): Promise<CaptureListResponse> => {
if (!session) return { items: [], total: 0 };
return listCaptures(session, { limit: 50 });
},
enabled: !!session,
});
}
export function useInvalidateCaptures() {
const qc = useQueryClient();
return () => qc.invalidateQueries({ queryKey: CAPTURES_KEY });
}
/** Voice profiles (cloned + preset). */
export function useProfiles() {
const session = useSession((s) => s.session);
return useQuery({
queryKey: PROFILES_KEY,
queryFn: async (): Promise<VoiceProfileResponse[]> => {
if (!session) return [];
return listProfiles(session);
},
enabled: !!session,
staleTime: 30_000,
});
}
/** Recent generations (most recent first). */
export function useHistory(limit = 20) {
const session = useSession((s) => s.session);
return useQuery({
queryKey: [...HISTORY_KEY, limit] as const,
queryFn: async (): Promise<HistoryListResponse> => {
if (!session) return { items: [], total: 0 };
return listHistory(session, { limit });
},
enabled: !!session,
});
}
/**
* Trigger a new generation. The mutation returns the generation row in its
* initial state — callers should follow up with ``useGenerationPolling``
* to await completion.
*/
export function useGenerate() {
const session = useSession((s) => s.session);
const qc = useQueryClient();
return useMutation({
mutationFn: async (req: GenerateRequest): Promise<GenerationResponse> => {
if (!session) throw new Error('Not paired');
return generate(session, req);
},
onSuccess: () => {
qc.invalidateQueries({ queryKey: HISTORY_KEY });
},
});
}
/**
* Poll a single generation row until it completes or fails. Stops polling
* automatically once a terminal state is reached, so it's cheap to leave
* mounted with a null id.
*/
export function useGenerationPolling(generationId: string | null) {
const session = useSession((s) => s.session);
return useQuery({
queryKey: ['generation', generationId] as const,
queryFn: async (): Promise<HistoryResponse | null> => {
if (!session || !generationId) return null;
return getHistoryItem(session, generationId);
},
enabled: !!session && !!generationId,
refetchInterval: (query) => {
const data = query.state.data;
if (!data) return 1500;
if (data.status === 'completed' || data.status === 'failed') return false;
return 1500;
},
});
}
+20
View File
@@ -0,0 +1,20 @@
// Parsers for the voicebox:// pair URL scheme. The desktop UI generates
// these as QR codes; the mobile app accepts them via QR scan or manual paste.
export type PairPayload = {
host: string;
token: string;
};
const SCHEME_RE = /^voicebox:\/\/pair\?(.+)$/;
export function parsePairUrl(input: string): PairPayload | null {
const trimmed = input.trim();
const match = trimmed.match(SCHEME_RE);
if (!match) return null;
const params = new URLSearchParams(match[1]);
const host = params.get('host');
const token = params.get('token');
if (!host || !token) return null;
return { host, token };
}
+33
View File
@@ -0,0 +1,33 @@
import { create } from 'zustand';
import {
clearStoredSession,
loadSession,
saveSession,
type StoredSession,
} from './storage';
type SessionState = {
session: StoredSession | null;
hydrated: boolean; // true after the first SecureStore read completes
hydrate: () => Promise<void>;
setSession: (s: StoredSession) => Promise<void>;
clear: () => Promise<void>;
};
export const useSession = create<SessionState>((set, get) => ({
session: null,
hydrated: false,
async hydrate() {
if (get().hydrated) return;
const session = await loadSession();
set({ session, hydrated: true });
},
async setSession(s) {
await saveSession(s);
set({ session: s });
},
async clear() {
await clearStoredSession();
set({ session: null });
},
}));
+30
View File
@@ -0,0 +1,30 @@
import * as SecureStore from 'expo-secure-store';
const SESSION_KEY = 'voicebox.session.v1';
export type StoredSession = {
host: string; // e.g. "192.168.1.5:17494" or "mac.tail-xxxx.ts.net:17493"
bearer: string; // long-lived bearer returned from POST /pair/complete
deviceId: string;
deviceName: string;
};
export async function loadSession(): Promise<StoredSession | null> {
const raw = await SecureStore.getItemAsync(SESSION_KEY);
if (!raw) return null;
try {
return JSON.parse(raw) as StoredSession;
} catch {
// Stored blob is corrupt — wipe so the user re-pairs cleanly
await SecureStore.deleteItemAsync(SESSION_KEY);
return null;
}
}
export async function saveSession(session: StoredSession): Promise<void> {
await SecureStore.setItemAsync(SESSION_KEY, JSON.stringify(session));
}
export async function clearStoredSession(): Promise<void> {
await SecureStore.deleteItemAsync(SESSION_KEY);
}
+49
View File
@@ -0,0 +1,49 @@
import { Image, Text, View } from 'react-native';
import { SafeAreaView } from 'react-native-safe-area-context';
import { Button } from '@/components/ui/Button';
type Props = {
onGetStarted: () => void;
};
export function WelcomeScreen({ onGetStarted }: Props) {
return (
<SafeAreaView className="flex-1 bg-background">
<View className="flex-1 px-8 items-center justify-between py-12">
<View className="flex-1 items-center justify-center gap-8">
<View
className="items-center justify-center"
style={{
shadowColor: '#AC8C39',
shadowOpacity: 0.45,
shadowRadius: 40,
shadowOffset: { width: 0, height: 0 },
}}
>
<Image
source={require('../../assets/icon.png')}
style={{ width: 120, height: 120, borderRadius: 28 }}
/>
</View>
<View className="items-center gap-3">
<Text className="text-foreground text-4xl font-bold tracking-tight text-center">
Welcome to Voicebox
</Text>
<Text className="text-muted-foreground text-base text-center max-w-[280px] leading-snug">
The open-source AI voice studio, in your pocket.
Clone voices, dictate anywhere, and talk to agents in voices you own.
</Text>
</View>
</View>
<View className="w-full items-center gap-4">
<Button label="Get started" size="lg" onPress={onGetStarted} />
<Text className="text-muted-foreground text-xs text-center">
You'll pair with your Voicebox desktop in the next step.
</Text>
</View>
</View>
</SafeAreaView>
);
}