mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-15 04:40:40 -07:00
feat(mcp): Rust-owned speaking pill with self-contained audio playback
The pill window now surfaces for agent-initiated speech without main-window
involvement. Rust subscribes to /events/speak via a tokio task + reqwest
streaming body (speak_monitor.rs), shows the pill, and forwards events to
the dictate webview over Tauri's event bus. The pill plays audio via a
plain HTMLAudioElement and emits dictate:hide when playback ends. The
pill stays hidden through the ~1 s generation wait and only surfaces when
audio actually starts, with the counter armed at that moment.
Fixes a shared-dict mutation in mcp_server/events.publish() that caused
the second subscriber (Rust speak_monitor) to receive `event: message`
instead of named speak-start/speak-end frames. Also teaches the speak_monitor
parser to handle CRLF framing (sse-starlette default). Main-window
AudioPlayer now skips autoplay for source in {mcp, rest} to avoid
double-play when both windows are alive.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
0cef2c9fe1
commit
6b75e097e1
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"voicebox": {
|
||||
"type": "http",
|
||||
"url": "http://127.0.0.1:17493/mcp",
|
||||
"headers": {
|
||||
"X-Voicebox-Client-Id": "claude-code"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,25 @@
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import { emit, listen, type UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { CapturePill } from '@/components/CapturePill/CapturePill';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { FocusSnapshot } from '@/lib/api/types';
|
||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||
import { useSpeakEvents } from '@/lib/hooks/useSpeakEvents';
|
||||
|
||||
/**
|
||||
* Floating dictate surface shown in a separate transparent Tauri window.
|
||||
* Mounted when the URL contains ``?view=dictate``. The main window bypasses
|
||||
* this branch and renders the full app shell.
|
||||
*
|
||||
* The pill is driven entirely by the global chord / toggle shortcut — there
|
||||
* is no fallback button here because the window is only visible while a
|
||||
* capture cycle is in flight.
|
||||
* The pill surfaces for two independent cycles:
|
||||
* 1. User dictation — driven by ``dictate:start`` / ``dictate:stop``
|
||||
* from the Rust hotkey monitor.
|
||||
* 2. Agent speech — driven by ``dictate:speak-start`` / ``dictate:speak-end``
|
||||
* from the Rust ``speak_monitor`` (which owns the backend SSE stream).
|
||||
* On speak-start we subscribe to this single generation's status SSE,
|
||||
* then play ``/audio/{id}`` via a plain ``HTMLAudioElement`` when it
|
||||
* lands. When the audio element's ``ended`` fires, we emit
|
||||
* ``dictate:hide`` so Rust tucks the window away.
|
||||
*/
|
||||
export function DictateWindow() {
|
||||
// Force the host document chrome to be transparent so the Tauri window
|
||||
@@ -83,27 +89,163 @@ export function DictateWindow() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Subscribe to agent-initiated speak events so the pill surfaces while
|
||||
// voicebox.speak (MCP) or POST /speak is producing audio. We ask Rust
|
||||
// to show the pill window by emitting `dictate:show` — the existing
|
||||
// `dictate:hide` happens on cycle end below.
|
||||
const speaking = useSpeakEvents();
|
||||
const prevSpeakingIdRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
const id = speaking?.generationId ?? null;
|
||||
if (id && id !== prevSpeakingIdRef.current) {
|
||||
emit('dictate:show').catch(() => {});
|
||||
}
|
||||
prevSpeakingIdRef.current = id;
|
||||
}, [speaking?.generationId]);
|
||||
// --- Agent-speak cycle ---------------------------------------------------
|
||||
|
||||
const [speaking, setSpeaking] = useState<{
|
||||
generationId: string;
|
||||
// Null while the backend is still generating audio; set to the
|
||||
// wall-clock timestamp when audio playback actually begins, so the
|
||||
// pill's elapsed counter only ticks while sound is coming out.
|
||||
startedAt: number | null;
|
||||
} | null>(null);
|
||||
const [speakElapsed, setSpeakElapsed] = useState(0);
|
||||
|
||||
// Refs so handlers inside long-lived `listen()` callbacks can read the
|
||||
// latest state without re-subscribing on every render.
|
||||
const speakingRef = useRef<typeof speaking>(null);
|
||||
speakingRef.current = speaking;
|
||||
const statusSourceRef = useRef<EventSource | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
|
||||
const dismissSpeak = (id?: string) => {
|
||||
// Guard against a late dismiss targeting a stale cycle (a new speak
|
||||
// already started by the time audio.ended from the previous one fired).
|
||||
if (id && speakingRef.current && speakingRef.current.generationId !== id) return;
|
||||
statusSourceRef.current?.close();
|
||||
statusSourceRef.current = null;
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
audioRef.current.src = '';
|
||||
audioRef.current = null;
|
||||
}
|
||||
setSpeaking(null);
|
||||
};
|
||||
|
||||
const startSpeakPlayback = (generationId: string) => {
|
||||
const audio = new Audio(apiClient.getAudioUrl(generationId));
|
||||
audio.onended = () => dismissSpeak(generationId);
|
||||
audio.onerror = () => dismissSpeak(generationId);
|
||||
// The pill window stays hidden through the ~1 s generation wait so the
|
||||
// user doesn't see a silent pill. We surface it the moment audio
|
||||
// actually starts playing, and that's also when the elapsed counter
|
||||
// arms.
|
||||
audio.onplaying = () => {
|
||||
emit('dictate:show').catch(() => {});
|
||||
setSpeaking((prev) =>
|
||||
prev && prev.generationId === generationId
|
||||
? { ...prev, startedAt: Date.now() }
|
||||
: prev,
|
||||
);
|
||||
setSpeakElapsed(0);
|
||||
};
|
||||
audioRef.current = audio;
|
||||
audio.play().catch((err) => {
|
||||
console.warn('[dictate] audio.play failed:', err);
|
||||
dismissSpeak(generationId);
|
||||
});
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const unlistens: Promise<UnlistenFn>[] = [];
|
||||
|
||||
// Rust emits the SSE payload as a JSON *string* (not a parsed object);
|
||||
// the payload shape for speak-start is
|
||||
// {generation_id, profile_name, source, client_id}.
|
||||
unlistens.push(
|
||||
listen<string>('dictate:speak-start', (event) => {
|
||||
let parsed: { generation_id?: string } = {};
|
||||
try {
|
||||
parsed = typeof event.payload === 'string' ? JSON.parse(event.payload) : {};
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const id = parsed.generation_id;
|
||||
if (!id) return;
|
||||
|
||||
// Tear down any previous cycle — last speak wins.
|
||||
dismissSpeak();
|
||||
|
||||
setSpeaking({ generationId: id, startedAt: null });
|
||||
setSpeakElapsed(0);
|
||||
|
||||
// Subscribe to this one generation's status. When it completes, the
|
||||
// `/audio/{id}` endpoint will serve the WAV we need to play.
|
||||
const source = new EventSource(apiClient.getGenerationStatusUrl(id));
|
||||
statusSourceRef.current = source;
|
||||
source.onmessage = (msg) => {
|
||||
try {
|
||||
const data = JSON.parse(msg.data) as { status?: string };
|
||||
if (data.status === 'completed') {
|
||||
source.close();
|
||||
if (statusSourceRef.current === source) statusSourceRef.current = null;
|
||||
startSpeakPlayback(id);
|
||||
} else if (data.status === 'failed' || data.status === 'not_found') {
|
||||
source.close();
|
||||
dismissSpeak(id);
|
||||
}
|
||||
} catch {
|
||||
// heartbeats / junk — ignore.
|
||||
}
|
||||
};
|
||||
source.onerror = () => {
|
||||
// Let browser retry a few times; if it gives up, force-dismiss.
|
||||
// We don't close here because EventSource auto-reconnects on
|
||||
// transient drops and we'd like to keep trying.
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// Speak-end from the backend is advisory: the authoritative dismiss is
|
||||
// `audio.ended`. But if generation failed or nothing ever triggered
|
||||
// playback, a short grace window followed by forced dismiss avoids a
|
||||
// stuck-visible pill.
|
||||
unlistens.push(
|
||||
listen<string>('dictate:speak-end', (event) => {
|
||||
let parsed: { generation_id?: string; status?: string } = {};
|
||||
try {
|
||||
parsed = typeof event.payload === 'string' ? JSON.parse(event.payload) : {};
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (parsed.status && parsed.status !== 'completed') {
|
||||
// Failed / cancelled — dismiss immediately.
|
||||
if (parsed.generation_id) dismissSpeak(parsed.generation_id);
|
||||
return;
|
||||
}
|
||||
// Completed: if audio never started (shouldn't happen, but guard),
|
||||
// auto-dismiss after 15 s so the pill never stays forever.
|
||||
const id = parsed.generation_id;
|
||||
window.setTimeout(() => {
|
||||
if (speakingRef.current?.generationId === id && !audioRef.current) {
|
||||
dismissSpeak(id);
|
||||
}
|
||||
}, 15_000);
|
||||
}),
|
||||
);
|
||||
|
||||
return () => {
|
||||
for (const p of unlistens) p.then((fn) => fn()).catch(() => {});
|
||||
dismissSpeak();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Advance the pill's elapsed-time label while audio is playing. Paused
|
||||
// during the pre-playback generation window (startedAt is null) so the
|
||||
// counter stays at 0:00 until sound actually starts.
|
||||
useEffect(() => {
|
||||
if (!speaking?.startedAt) return;
|
||||
const anchor = speaking.startedAt;
|
||||
const iv = window.setInterval(() => {
|
||||
setSpeakElapsed(Date.now() - anchor);
|
||||
}, 250);
|
||||
return () => window.clearInterval(iv);
|
||||
}, [speaking?.generationId, speaking?.startedAt]);
|
||||
|
||||
// --- Effective pill state -----------------------------------------------
|
||||
|
||||
// Compose the effective pill state: speak events override the capture
|
||||
// session when both would render, because agent speech is always
|
||||
// category-mattering (the user can't hear two pills). Elapsed time
|
||||
// restarts for speaking so the timer reflects playback length.
|
||||
const isSpeaking = Boolean(speaking);
|
||||
const effectiveState = isSpeaking ? 'speaking' : session.pillState;
|
||||
const effectiveElapsed = isSpeaking ? speaking!.elapsedMs : session.pillElapsedMs;
|
||||
const effectiveElapsed = isSpeaking ? speakElapsed : session.pillElapsedMs;
|
||||
|
||||
// When the pill cycle ends (no capture AND no speak), tell Rust to tuck
|
||||
// the window away. Rust owns the hide + park-off-screen + click-through
|
||||
|
||||
@@ -11,8 +11,13 @@ interface GenerationStatusEvent {
|
||||
status: 'loading_model' | 'generating' | 'completed' | 'failed' | 'not_found';
|
||||
duration?: number;
|
||||
error?: string;
|
||||
source?: string;
|
||||
}
|
||||
|
||||
// Agent-initiated generations are played by the floating pill, not the
|
||||
// main-window AudioPlayer. Skip autoplay here to avoid double-playback.
|
||||
const AGENT_SOURCES = new Set(['mcp', 'rest']);
|
||||
|
||||
/**
|
||||
* Subscribes to SSE for all pending generations. When a generation completes,
|
||||
* invalidates the history query, removes it from pending, and auto-plays
|
||||
@@ -110,8 +115,11 @@ export function useGenerationProgress() {
|
||||
// });
|
||||
}
|
||||
|
||||
// Auto-play if enabled and nothing is currently playing
|
||||
if (autoplayRef.current && !isPlayingRef.current) {
|
||||
// Auto-play if enabled and nothing is currently playing.
|
||||
// Skip agent-initiated sources — the floating pill window
|
||||
// plays those itself.
|
||||
const isAgentSpeak = data.source ? AGENT_SOURCES.has(data.source) : false;
|
||||
if (autoplayRef.current && !isPlayingRef.current && !isAgentSpeak) {
|
||||
const genAudioUrl = apiClient.getAudioUrl(id);
|
||||
setAudioWithAutoPlay(genAudioUrl, id, '', '');
|
||||
}
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { useServerStore } from '@/stores/serverStore';
|
||||
|
||||
/** Payload for a speak-start SSE event broadcast by the backend. */
|
||||
export interface ActiveSpeak {
|
||||
generationId: string;
|
||||
profileName: string;
|
||||
source: 'mcp' | 'rest' | string;
|
||||
clientId: string | null;
|
||||
startedAt: number;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribes to `/events/speak` and reports whichever agent-initiated
|
||||
* speak is currently producing audio. Returns ``null`` when nothing is
|
||||
* speaking.
|
||||
*
|
||||
* Multiple concurrent speaks are rare (the model can only really do one
|
||||
* at a time) and we don't bother stacking them — newest wins, the old
|
||||
* one's speak-end will clear when it fires.
|
||||
*/
|
||||
export function useSpeakEvents(): ActiveSpeak | null {
|
||||
const [active, setActive] = useState<ActiveSpeak | null>(null);
|
||||
const activeRef = useRef<ActiveSpeak | null>(null);
|
||||
activeRef.current = active;
|
||||
|
||||
// Keep a live timer so the pill's elapsed label advances smoothly
|
||||
// without re-opening the SSE stream.
|
||||
const [, setTick] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!active) return;
|
||||
const iv = window.setInterval(() => setTick((t) => t + 1), 250);
|
||||
return () => window.clearInterval(iv);
|
||||
}, [active]);
|
||||
|
||||
useEffect(() => {
|
||||
const baseUrl = useServerStore.getState().serverUrl;
|
||||
if (!baseUrl) return;
|
||||
|
||||
let cancelled = false;
|
||||
let source: EventSource | null = null;
|
||||
|
||||
const connect = () => {
|
||||
if (cancelled) return;
|
||||
source = new EventSource(`${baseUrl}/events/speak`);
|
||||
|
||||
source.addEventListener('speak-start', (event: MessageEvent) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
const now = Date.now();
|
||||
setActive({
|
||||
generationId: String(data.generation_id ?? ''),
|
||||
profileName: String(data.profile_name ?? ''),
|
||||
source: String(data.source ?? 'mcp'),
|
||||
clientId: data.client_id ?? null,
|
||||
startedAt: now,
|
||||
elapsedMs: 0,
|
||||
});
|
||||
} catch {
|
||||
// malformed payload — ignore, don't crash the stream
|
||||
}
|
||||
});
|
||||
|
||||
source.addEventListener('speak-end', (event: MessageEvent) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
const endedId = String(data.generation_id ?? '');
|
||||
// Only clear if this end matches the currently-active id; late
|
||||
// ends from previous sessions are ignored.
|
||||
if (activeRef.current?.generationId === endedId) setActive(null);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
|
||||
source.onerror = () => {
|
||||
// EventSource auto-reconnects, but if the browser gives up we
|
||||
// manually retry with backoff.
|
||||
source?.close();
|
||||
if (!cancelled) window.setTimeout(connect, 2000);
|
||||
};
|
||||
};
|
||||
|
||||
connect();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
source?.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!active) return null;
|
||||
return { ...active, elapsedMs: Date.now() - active.startedAt };
|
||||
}
|
||||
@@ -25,9 +25,15 @@ def unsubscribe(queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
|
||||
|
||||
def publish(kind: str, payload: dict[str, Any]) -> None:
|
||||
"""Fan out to all current subscribers. Non-blocking; drops on full queue."""
|
||||
event = {"kind": kind, **payload}
|
||||
"""Fan out to all current subscribers. Non-blocking; drops on full queue.
|
||||
|
||||
Each subscriber gets its own dict copy — the SSE consumer calls
|
||||
``event.pop("kind", ...)``, so sharing a single dict between queues
|
||||
would mean the first consumer to drain its queue strips ``kind`` from
|
||||
the object the next consumer later reads.
|
||||
"""
|
||||
for queue in list(_subscribers):
|
||||
event = {"kind": kind, **payload}
|
||||
try:
|
||||
queue.put_nowait(event)
|
||||
except asyncio.QueueFull:
|
||||
|
||||
@@ -237,6 +237,9 @@ async def get_generation_status(generation_id: str, db: Session = Depends(get_db
|
||||
"status": gen.status or "completed",
|
||||
"duration": gen.duration,
|
||||
"error": gen.error,
|
||||
# Agent-originated sources ("mcp", "rest") skip main-window
|
||||
# autoplay — the floating pill plays those directly.
|
||||
"source": gen.source,
|
||||
}
|
||||
yield f"data: {json.dumps(payload)}\n\n"
|
||||
|
||||
|
||||
@@ -60,12 +60,11 @@ export default function CapturePage() {
|
||||
<div className="grid md:grid-cols-3 gap-6">
|
||||
<div className="rounded-xl border border-border bg-card/40 backdrop-blur-sm p-6">
|
||||
<h3 className="text-[15px] font-semibold text-foreground mb-2">
|
||||
Four STT engines, one picker
|
||||
Whisper, sized for every machine
|
||||
</h3>
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">
|
||||
Whisper, Whisper Turbo, Parakeet v3, Qwen3-ASR. Pick per-capture — broad
|
||||
multilingual, speed, non-English quality, or cross-platform coverage. All local,
|
||||
all downloadable from inside the app.
|
||||
Base, Small, Medium, Large, and Turbo. Pick per-capture — 99 languages at every
|
||||
tier, all local, all downloadable from inside the app.
|
||||
</p>
|
||||
</div>
|
||||
<div className="rounded-xl border border-border bg-card/40 backdrop-blur-sm p-6">
|
||||
|
||||
@@ -147,15 +147,16 @@ export function DictationHero() {
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Card: Multi-Engine STT ─────────────────────────────────────────────────
|
||||
// ─── Card: Whisper, sized for every machine ────────────────────────────────
|
||||
|
||||
type EngineRow = { name: string; size: string; langs: string };
|
||||
|
||||
const STT_ENGINES: EngineRow[] = [
|
||||
{ name: 'Whisper', size: '1.5B', langs: '99 langs' },
|
||||
{ name: 'Whisper Base', size: '74M', langs: '99 langs' },
|
||||
{ name: 'Whisper Small', size: '244M', langs: '99 langs' },
|
||||
{ name: 'Whisper Medium', size: '769M', langs: '99 langs' },
|
||||
{ name: 'Whisper Large', size: '1.5B', langs: '99 langs' },
|
||||
{ name: 'Whisper Turbo', size: '809M', langs: '99 langs' },
|
||||
{ name: 'Parakeet v3', size: '600M', langs: '25 langs' },
|
||||
{ name: 'Qwen3-ASR', size: '600M', langs: '50+ langs' },
|
||||
];
|
||||
|
||||
function MultiEngineSTTAnimation() {
|
||||
@@ -400,9 +401,9 @@ function AgentVoiceAnimation() {
|
||||
|
||||
const CAPTURE_FEATURES = [
|
||||
{
|
||||
title: 'Multi-Engine STT',
|
||||
title: 'Whisper, sized for every machine',
|
||||
description:
|
||||
'Whisper, Whisper Turbo, Parakeet v3, Qwen3-ASR. Pick the model that fits your accent, language, or speed — all running on your hardware.',
|
||||
'Base, Small, Medium, Large, and Turbo. Pick the size that fits your hardware and quality bar — 99 languages across every tier, all running locally.',
|
||||
icon: Mic2,
|
||||
animation: MultiEngineSTTAnimation,
|
||||
},
|
||||
|
||||
@@ -157,16 +157,16 @@ const CAPTURES: Capture[] = [
|
||||
ago: '22 min ago',
|
||||
createdAtLabel: 'Apr 22, 3:29 PM',
|
||||
source: 'dictation',
|
||||
sttModel: 'parakeet-v3',
|
||||
sttModel: 'turbo',
|
||||
language: 'en',
|
||||
},
|
||||
{
|
||||
id: 'c3',
|
||||
seed: 37,
|
||||
transcriptRaw:
|
||||
"tech overview for the readme seven TTS engines qwen3 kokoro chatterbox luxtts customvoice tada and chatterbox turbo four STT whisper whisper turbo parakeet v3 qwen3 ASR one local LLM qwen 3.5 shared runtime across all of them one model directory one GPU story no fragmented caches pick the right model per job speed on CPU laptops quality on an M series mac all switchable per generation",
|
||||
"tech overview for the readme seven TTS engines qwen3 kokoro chatterbox luxtts customvoice tada and chatterbox turbo whisper for STT in five sizes from base up to large and a turbo variant one local LLM qwen 3.5 shared runtime across all of them one model directory one GPU story no fragmented caches pick the right model per job speed on CPU laptops quality on an M series mac all switchable per generation",
|
||||
transcriptRefined:
|
||||
"Tech overview for the README: seven TTS engines — Qwen3, Kokoro, Chatterbox, LuxTTS, CustomVoice, TADA, and Chatterbox Turbo. Four STT — Whisper, Whisper Turbo, Parakeet v3, Qwen3-ASR. One local LLM, Qwen 3.5, with a shared runtime across all of them. One model directory, one GPU story, no fragmented caches. Pick the right model per job — speed on CPU laptops, quality on an M-series Mac, switchable per-generation.",
|
||||
"Tech overview for the README: seven TTS engines — Qwen3, Kokoro, Chatterbox, LuxTTS, CustomVoice, TADA, and Chatterbox Turbo. Whisper for STT, in five sizes from Base up to Large, plus a Turbo variant. One local LLM, Qwen 3.5, with a shared runtime across all of them. One model directory, one GPU story, no fragmented caches. Pick the right model per job — speed on CPU laptops, quality on an M-series Mac, switchable per-generation.",
|
||||
durationMs: 34000,
|
||||
ago: '1 hr ago',
|
||||
createdAtLabel: 'Apr 22, 2:51 PM',
|
||||
|
||||
@@ -131,25 +131,6 @@ const MODEL_GROUPS: ModelGroup[] = [
|
||||
{ icon: Zap, label: '8x faster' },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Parakeet v3',
|
||||
author: 'NVIDIA',
|
||||
sizes: ['600M'],
|
||||
description:
|
||||
'Current quality leader for non-English local STT. Very fast, with strong accuracy on European and Asian languages.',
|
||||
tags: [
|
||||
{ icon: Languages, label: '25 langs' },
|
||||
{ icon: Zap, label: 'Fast' },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'Qwen3-ASR',
|
||||
author: 'Alibaba',
|
||||
sizes: ['600M'],
|
||||
description:
|
||||
'int8 quantized for cross-platform use. Highest multilingual coverage of any engine — 50+ languages with strong accuracy.',
|
||||
tags: [{ icon: Languages, label: '50+ langs' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -20,7 +20,7 @@ tauri-plugin-shell = "2.0"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
reqwest = { version = "0.12", features = ["blocking", "json"] }
|
||||
reqwest = { version = "0.12", features = ["blocking", "json", "stream"] }
|
||||
hound = "3.5"
|
||||
base64 = "0.22"
|
||||
cpal = "0.15"
|
||||
|
||||
@@ -172,7 +172,6 @@ pub struct HotkeyMonitor {
|
||||
|
||||
impl HotkeyMonitor {
|
||||
pub fn spawn(app: AppHandle, bindings: Bindings) -> Self {
|
||||
eprintln!("[HotkeyMonitor] spawn() called with {} bindings", bindings.len());
|
||||
let chord = Arc::new(Mutex::new(Chord::new(bindings)));
|
||||
let chord_for_thread = chord.clone();
|
||||
let app_for_thread = app.clone();
|
||||
@@ -184,9 +183,7 @@ impl HotkeyMonitor {
|
||||
#[cfg(target_os = "macos")]
|
||||
rdev::set_is_main_thread(false);
|
||||
|
||||
eprintln!("[HotkeyMonitor] background thread entering rdev::listen");
|
||||
let result = listen(move |event| {
|
||||
eprintln!("[HotkeyMonitor] rdev event: {:?}", event.event_type);
|
||||
let input = match event.event_type {
|
||||
EventType::KeyPress(k) => KeyEvent::Down(k),
|
||||
EventType::KeyRelease(k) => KeyEvent::Up(k),
|
||||
@@ -198,17 +195,11 @@ impl HotkeyMonitor {
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
if !effects.is_empty() {
|
||||
eprintln!("[HotkeyMonitor] chord matched, effects: {:?}", effects);
|
||||
}
|
||||
|
||||
for effect in effects {
|
||||
apply_effect(&app_for_thread, effect);
|
||||
}
|
||||
});
|
||||
|
||||
// listen() blocks forever on success; reaching here means it errored.
|
||||
eprintln!("[HotkeyMonitor] rdev::listen returned (this only happens on error): {:?}", result);
|
||||
if let Err(err) = result {
|
||||
eprintln!(
|
||||
"HotkeyMonitor: rdev::listen failed ({:?}). Global chord detection is disabled. On macOS, grant Input Monitoring in System Settings → Privacy & Security → Input Monitoring and relaunch.",
|
||||
|
||||
+53
-28
@@ -11,6 +11,7 @@ mod hotkey_monitor;
|
||||
mod input_monitoring;
|
||||
#[cfg(desktop)]
|
||||
mod key_codes;
|
||||
mod speak_monitor;
|
||||
mod synthetic_keys;
|
||||
|
||||
use std::sync::Mutex;
|
||||
@@ -66,33 +67,56 @@ fn build_dictate_window(app: &tauri::AppHandle) -> tauri::Result<tauri::WebviewW
|
||||
/// `Effect::StartRecording` path runs, minus the focus snapshot — this is
|
||||
/// for agent-initiated speech, not dictation, so there's no focused text
|
||||
/// field to paste into.
|
||||
/// Build the pill webview if it doesn't exist yet. Idempotent — used by
|
||||
/// agent-speech to prime the webview on speak-start so its listeners can
|
||||
/// register before the actual show arrives from `audio.onplaying`.
|
||||
#[cfg(desktop)]
|
||||
pub fn show_dictate_window(app: &tauri::AppHandle) {
|
||||
if let Some(window) = app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
// current_monitor() returns None when the window has been parked
|
||||
// off any display by the hide path; fall back to the primary.
|
||||
let monitor = window
|
||||
.current_monitor()
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| window.primary_monitor().ok().flatten());
|
||||
if let Some(monitor) = monitor {
|
||||
let monitor_pos = monitor.position();
|
||||
let monitor_size = monitor.size();
|
||||
if let Ok(win_size) = window.outer_size() {
|
||||
let x = monitor_pos.x
|
||||
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||
let _ = window.set_position(PhysicalPosition::new(x, y));
|
||||
}
|
||||
pub fn ensure_dictate_window(app: &tauri::AppHandle) {
|
||||
if app.get_webview_window(DICTATE_WINDOW_LABEL).is_none() {
|
||||
if let Err(e) = build_dictate_window(app) {
|
||||
eprintln!("ensure_dictate_window: failed to build pill: {e}");
|
||||
}
|
||||
let _ = window.set_ignore_cursor_events(false);
|
||||
let _ = window.show();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(desktop)]
|
||||
pub fn show_dictate_window(app: &tauri::AppHandle) {
|
||||
// Build on demand so agent-initiated speech works before the user has
|
||||
// enabled the global hotkey (the hotkey path is the other place this
|
||||
// window gets built, see `enable_hotkey`).
|
||||
let window = match app.get_webview_window(DICTATE_WINDOW_LABEL) {
|
||||
Some(w) => w,
|
||||
None => match build_dictate_window(app) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
eprintln!("show_dictate_window: failed to build pill window: {e}");
|
||||
return;
|
||||
}
|
||||
},
|
||||
};
|
||||
// current_monitor() returns None when the window has been parked
|
||||
// off any display by the hide path; fall back to the primary.
|
||||
let monitor = window
|
||||
.current_monitor()
|
||||
.ok()
|
||||
.flatten()
|
||||
.or_else(|| window.primary_monitor().ok().flatten());
|
||||
if let Some(monitor) = monitor {
|
||||
let monitor_pos = monitor.position();
|
||||
let monitor_size = monitor.size();
|
||||
if let Ok(win_size) = window.outer_size() {
|
||||
let x = monitor_pos.x
|
||||
+ (monitor_size.width as i32 - win_size.width as i32) / 2;
|
||||
let y = monitor_pos.y + (monitor_size.height as f64 * 0.04) as i32;
|
||||
let _ = window.set_position(PhysicalPosition::new(x, y));
|
||||
}
|
||||
}
|
||||
let _ = window.set_ignore_cursor_events(false);
|
||||
let _ = window.show();
|
||||
}
|
||||
|
||||
const LEGACY_PORT: u16 = 8000;
|
||||
const SERVER_PORT: u16 = 17493;
|
||||
pub(crate) const SERVER_PORT: u16 = 17493;
|
||||
|
||||
/// Find a voicebox-server process listening on a given port (Windows only).
|
||||
///
|
||||
@@ -895,7 +919,6 @@ fn enable_hotkey(
|
||||
push_to_talk: Vec<String>,
|
||||
toggle_to_talk: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
eprintln!("[enable_hotkey] called: push={:?}, toggle={:?}", push_to_talk, toggle_to_talk);
|
||||
let bindings = build_chord_bindings(&push_to_talk, &toggle_to_talk)?;
|
||||
|
||||
// Fire the Input Monitoring TCC prompt explicitly from the user's
|
||||
@@ -908,9 +931,7 @@ fn enable_hotkey(
|
||||
// The call returns the current grant state; we ignore it because
|
||||
// rdev::listen will surface its own error via stderr, and the
|
||||
// settings UI polls `check_input_monitoring_permission` separately.
|
||||
let granted = input_monitoring::request();
|
||||
eprintln!("[enable_hotkey] IOHIDRequestAccess returned granted={}", granted);
|
||||
eprintln!("[enable_hotkey] IOHIDCheckAccess says trusted={}", input_monitoring::is_trusted());
|
||||
let _ = input_monitoring::request();
|
||||
|
||||
// The dictate pill webview must exist before the first chord fires so it
|
||||
// can subscribe to `dictate:start`. Build it here (idempotent — Tauri
|
||||
@@ -1225,13 +1246,17 @@ pub fn run() {
|
||||
|
||||
// Agent-initiated speech (voicebox.speak over MCP or POST /speak)
|
||||
// pops the pill up so the user can see what's coming out of their
|
||||
// machine. The DictateWindow subscribes to /events/speak via SSE
|
||||
// and emits `dictate:show` on speak-start; we repeat the same
|
||||
// position+show dance the hotkey path uses.
|
||||
// machine. The `dictate:show` listener is kept for any frontend
|
||||
// caller that wants to force-surface the pill directly, but the
|
||||
// primary source is `speak_monitor` below — Rust subscribes to
|
||||
// the backend /events/speak SSE stream so the pill surfaces even
|
||||
// when no JS window is active.
|
||||
let handle_for_show = app.handle().clone();
|
||||
app.handle().listen("dictate:show", move |_event| {
|
||||
show_dictate_window(&handle_for_show);
|
||||
});
|
||||
|
||||
speak_monitor::spawn_speak_monitor(app.handle().clone());
|
||||
}
|
||||
|
||||
// Hide title bar icon on Windows
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
//! Rust-side subscriber for the backend `/events/speak` SSE stream.
|
||||
//!
|
||||
//! Owns the pill-window lifecycle for agent-initiated speech. The dictate
|
||||
//! webview used to do this itself via `EventSource`, but hidden WebKit
|
||||
//! windows on macOS throttle long-lived network connections, so speak events
|
||||
//! never reached the pill. Tauri's event bus, on the other hand, reliably
|
||||
//! delivers events to hidden webviews (the chord path proves it), so we
|
||||
//! subscribe here and fan out via `emit`.
|
||||
//!
|
||||
//! Flow:
|
||||
//! backend speak-start → show dictate window + emit("dictate:speak-start")
|
||||
//! backend speak-end → emit("dictate:speak-end")
|
||||
//! The pill webview handles the rest (audio playback, then emits
|
||||
//! `dictate:hide` back to Rust when the audio element's `ended` fires).
|
||||
//!
|
||||
//! The task reconnects on any error with a 2 s backoff. There's no fancy
|
||||
//! exponential backoff — the backend either dies with the app or comes back
|
||||
//! quickly, and constant 2 s polling is cheap.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use tauri::{AppHandle, Emitter};
|
||||
|
||||
use crate::{ensure_dictate_window, SERVER_PORT};
|
||||
|
||||
pub fn spawn_speak_monitor(app: AppHandle) {
|
||||
tauri::async_runtime::spawn(async move {
|
||||
run(app).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn run(app: AppHandle) {
|
||||
let url = format!("http://127.0.0.1:{}/events/speak", SERVER_PORT);
|
||||
let client = match reqwest::Client::builder().build() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("speak_monitor: failed to build HTTP client: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
if let Err(e) = stream_once(&client, &url, &app).await {
|
||||
eprintln!("speak_monitor: stream err: {e}");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_once(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
app: &AppHandle,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut resp = client
|
||||
.get(url)
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("speak_monitor: backend returned {}", resp.status()).into());
|
||||
}
|
||||
let mut buf = String::new();
|
||||
while let Some(chunk) = resp.chunk().await? {
|
||||
buf.push_str(std::str::from_utf8(&chunk)?);
|
||||
// sse-starlette emits CRLF framing; the spec also permits LF, so
|
||||
// handle either. Drain whichever separator appears first.
|
||||
loop {
|
||||
let crlf = buf.find("\r\n\r\n");
|
||||
let lf = buf.find("\n\n");
|
||||
let (end, sep_len) = match (crlf, lf) {
|
||||
(Some(c), Some(l)) if c <= l => (c, 4),
|
||||
(Some(c), None) => (c, 4),
|
||||
(_, Some(l)) => (l, 2),
|
||||
(None, None) => break,
|
||||
};
|
||||
let frame: String = buf.drain(..end + sep_len).collect();
|
||||
if let Some((event, data)) = parse_frame(&frame) {
|
||||
dispatch(app, &event, &data);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse a single SSE frame into (event_name, data_json).
|
||||
///
|
||||
/// Returns None for comment-only frames (lines starting with `:`) and
|
||||
/// for frames without a recognizable `event:` or `data:` line.
|
||||
fn parse_frame(frame: &str) -> Option<(String, String)> {
|
||||
let mut event: Option<String> = None;
|
||||
let mut data_lines: Vec<&str> = Vec::new();
|
||||
for line in frame.lines() {
|
||||
if line.is_empty() || line.starts_with(':') {
|
||||
continue;
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix("event:") {
|
||||
event = Some(rest.trim().to_string());
|
||||
} else if let Some(rest) = line.strip_prefix("data:") {
|
||||
data_lines.push(rest.trim_start());
|
||||
}
|
||||
}
|
||||
let event = event?;
|
||||
let data = data_lines.join("\n");
|
||||
Some((event, data))
|
||||
}
|
||||
|
||||
fn dispatch(app: &AppHandle, event: &str, data: &str) {
|
||||
match event {
|
||||
"speak-start" => {
|
||||
// Build the pill webview hidden if it doesn't exist yet so its
|
||||
// listeners can register — but don't *show* it here. The pill
|
||||
// surfaces itself from `audio.onplaying` via `dictate:show`, so
|
||||
// users never see the empty-silent generation window.
|
||||
ensure_dictate_window(app);
|
||||
let _ = app.emit("dictate:speak-start", data.to_string());
|
||||
}
|
||||
"speak-end" => {
|
||||
let _ = app.emit("dictate:speak-end", data.to_string());
|
||||
}
|
||||
// `ready` and `ping` are heartbeats; ignore.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user