mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-18 22:30:40 -07:00
feat(mcp): Rust-owned speaking pill with self-contained audio playback
The pill window now surfaces for agent-initiated speech without main-window
involvement. Rust subscribes to /events/speak via a tokio task + reqwest
streaming body (speak_monitor.rs), shows the pill, and forwards events to
the dictate webview over Tauri's event bus. The pill plays audio via a
plain HTMLAudioElement and emits dictate:hide when playback ends. The
pill stays hidden through the ~1 s generation wait and only surfaces when
audio actually starts, with the counter armed at that moment.
Fixes a shared-dict mutation in mcp_server/events.publish() that caused
the second subscriber (Rust speak_monitor) to receive `event: message`
instead of named speak-start/speak-end frames. Also teaches the speak_monitor
parser to handle CRLF framing (sse-starlette default). Main-window
AudioPlayer now skips autoplay for source in {mcp, rest} to avoid
double-play when both windows are alive.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James Pine
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
0cef2c9fe1
commit
6b75e097e1
@@ -1,19 +1,25 @@
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
import { emit, listen, type UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { CapturePill } from '@/components/CapturePill/CapturePill';
|
||||
import { apiClient } from '@/lib/api/client';
|
||||
import type { FocusSnapshot } from '@/lib/api/types';
|
||||
import { useCaptureRecordingSession } from '@/lib/hooks/useCaptureRecordingSession';
|
||||
import { useSpeakEvents } from '@/lib/hooks/useSpeakEvents';
|
||||
|
||||
/**
|
||||
* Floating dictate surface shown in a separate transparent Tauri window.
|
||||
* Mounted when the URL contains ``?view=dictate``. The main window bypasses
|
||||
* this branch and renders the full app shell.
|
||||
*
|
||||
* The pill is driven entirely by the global chord / toggle shortcut — there
|
||||
* is no fallback button here because the window is only visible while a
|
||||
* capture cycle is in flight.
|
||||
* The pill surfaces for two independent cycles:
|
||||
* 1. User dictation — driven by ``dictate:start`` / ``dictate:stop``
|
||||
* from the Rust hotkey monitor.
|
||||
* 2. Agent speech — driven by ``dictate:speak-start`` / ``dictate:speak-end``
|
||||
* from the Rust ``speak_monitor`` (which owns the backend SSE stream).
|
||||
* On speak-start we subscribe to this single generation's status SSE,
|
||||
* then play ``/audio/{id}`` via a plain ``HTMLAudioElement`` when it
|
||||
* lands. When the audio element's ``ended`` fires, we emit
|
||||
* ``dictate:hide`` so Rust tucks the window away.
|
||||
*/
|
||||
export function DictateWindow() {
|
||||
// Force the host document chrome to be transparent so the Tauri window
|
||||
@@ -83,27 +89,163 @@ export function DictateWindow() {
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Subscribe to agent-initiated speak events so the pill surfaces while
|
||||
// voicebox.speak (MCP) or POST /speak is producing audio. We ask Rust
|
||||
// to show the pill window by emitting `dictate:show` — the existing
|
||||
// `dictate:hide` happens on cycle end below.
|
||||
const speaking = useSpeakEvents();
|
||||
const prevSpeakingIdRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
const id = speaking?.generationId ?? null;
|
||||
if (id && id !== prevSpeakingIdRef.current) {
|
||||
emit('dictate:show').catch(() => {});
|
||||
}
|
||||
prevSpeakingIdRef.current = id;
|
||||
}, [speaking?.generationId]);
|
||||
// --- Agent-speak cycle ---------------------------------------------------
|
||||
|
||||
const [speaking, setSpeaking] = useState<{
|
||||
generationId: string;
|
||||
// Null while the backend is still generating audio; set to the
|
||||
// wall-clock timestamp when audio playback actually begins, so the
|
||||
// pill's elapsed counter only ticks while sound is coming out.
|
||||
startedAt: number | null;
|
||||
} | null>(null);
|
||||
const [speakElapsed, setSpeakElapsed] = useState(0);
|
||||
|
||||
// Refs so handlers inside long-lived `listen()` callbacks can read the
|
||||
// latest state without re-subscribing on every render.
|
||||
const speakingRef = useRef<typeof speaking>(null);
|
||||
speakingRef.current = speaking;
|
||||
const statusSourceRef = useRef<EventSource | null>(null);
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
|
||||
const dismissSpeak = (id?: string) => {
|
||||
// Guard against a late dismiss targeting a stale cycle (a new speak
|
||||
// already started by the time audio.ended from the previous one fired).
|
||||
if (id && speakingRef.current && speakingRef.current.generationId !== id) return;
|
||||
statusSourceRef.current?.close();
|
||||
statusSourceRef.current = null;
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
audioRef.current.src = '';
|
||||
audioRef.current = null;
|
||||
}
|
||||
setSpeaking(null);
|
||||
};
|
||||
|
||||
const startSpeakPlayback = (generationId: string) => {
|
||||
const audio = new Audio(apiClient.getAudioUrl(generationId));
|
||||
audio.onended = () => dismissSpeak(generationId);
|
||||
audio.onerror = () => dismissSpeak(generationId);
|
||||
// The pill window stays hidden through the ~1 s generation wait so the
|
||||
// user doesn't see a silent pill. We surface it the moment audio
|
||||
// actually starts playing, and that's also when the elapsed counter
|
||||
// arms.
|
||||
audio.onplaying = () => {
|
||||
emit('dictate:show').catch(() => {});
|
||||
setSpeaking((prev) =>
|
||||
prev && prev.generationId === generationId
|
||||
? { ...prev, startedAt: Date.now() }
|
||||
: prev,
|
||||
);
|
||||
setSpeakElapsed(0);
|
||||
};
|
||||
audioRef.current = audio;
|
||||
audio.play().catch((err) => {
|
||||
console.warn('[dictate] audio.play failed:', err);
|
||||
dismissSpeak(generationId);
|
||||
});
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const unlistens: Promise<UnlistenFn>[] = [];
|
||||
|
||||
// Rust emits the SSE payload as a JSON *string* (not a parsed object);
|
||||
// the payload shape for speak-start is
|
||||
// {generation_id, profile_name, source, client_id}.
|
||||
unlistens.push(
|
||||
listen<string>('dictate:speak-start', (event) => {
|
||||
let parsed: { generation_id?: string } = {};
|
||||
try {
|
||||
parsed = typeof event.payload === 'string' ? JSON.parse(event.payload) : {};
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const id = parsed.generation_id;
|
||||
if (!id) return;
|
||||
|
||||
// Tear down any previous cycle — last speak wins.
|
||||
dismissSpeak();
|
||||
|
||||
setSpeaking({ generationId: id, startedAt: null });
|
||||
setSpeakElapsed(0);
|
||||
|
||||
// Subscribe to this one generation's status. When it completes, the
|
||||
// `/audio/{id}` endpoint will serve the WAV we need to play.
|
||||
const source = new EventSource(apiClient.getGenerationStatusUrl(id));
|
||||
statusSourceRef.current = source;
|
||||
source.onmessage = (msg) => {
|
||||
try {
|
||||
const data = JSON.parse(msg.data) as { status?: string };
|
||||
if (data.status === 'completed') {
|
||||
source.close();
|
||||
if (statusSourceRef.current === source) statusSourceRef.current = null;
|
||||
startSpeakPlayback(id);
|
||||
} else if (data.status === 'failed' || data.status === 'not_found') {
|
||||
source.close();
|
||||
dismissSpeak(id);
|
||||
}
|
||||
} catch {
|
||||
// heartbeats / junk — ignore.
|
||||
}
|
||||
};
|
||||
source.onerror = () => {
|
||||
// Let browser retry a few times; if it gives up, force-dismiss.
|
||||
// We don't close here because EventSource auto-reconnects on
|
||||
// transient drops and we'd like to keep trying.
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// Speak-end from the backend is advisory: the authoritative dismiss is
|
||||
// `audio.ended`. But if generation failed or nothing ever triggered
|
||||
// playback, a short grace window followed by forced dismiss avoids a
|
||||
// stuck-visible pill.
|
||||
unlistens.push(
|
||||
listen<string>('dictate:speak-end', (event) => {
|
||||
let parsed: { generation_id?: string; status?: string } = {};
|
||||
try {
|
||||
parsed = typeof event.payload === 'string' ? JSON.parse(event.payload) : {};
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (parsed.status && parsed.status !== 'completed') {
|
||||
// Failed / cancelled — dismiss immediately.
|
||||
if (parsed.generation_id) dismissSpeak(parsed.generation_id);
|
||||
return;
|
||||
}
|
||||
// Completed: if audio never started (shouldn't happen, but guard),
|
||||
// auto-dismiss after 15 s so the pill never stays forever.
|
||||
const id = parsed.generation_id;
|
||||
window.setTimeout(() => {
|
||||
if (speakingRef.current?.generationId === id && !audioRef.current) {
|
||||
dismissSpeak(id);
|
||||
}
|
||||
}, 15_000);
|
||||
}),
|
||||
);
|
||||
|
||||
return () => {
|
||||
for (const p of unlistens) p.then((fn) => fn()).catch(() => {});
|
||||
dismissSpeak();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Advance the pill's elapsed-time label while audio is playing. Paused
|
||||
// during the pre-playback generation window (startedAt is null) so the
|
||||
// counter stays at 0:00 until sound actually starts.
|
||||
useEffect(() => {
|
||||
if (!speaking?.startedAt) return;
|
||||
const anchor = speaking.startedAt;
|
||||
const iv = window.setInterval(() => {
|
||||
setSpeakElapsed(Date.now() - anchor);
|
||||
}, 250);
|
||||
return () => window.clearInterval(iv);
|
||||
}, [speaking?.generationId, speaking?.startedAt]);
|
||||
|
||||
// --- Effective pill state -----------------------------------------------
|
||||
|
||||
// Compose the effective pill state: speak events override the capture
|
||||
// session when both would render, because agent speech is always
|
||||
// category-mattering (the user can't hear two pills). Elapsed time
|
||||
// restarts for speaking so the timer reflects playback length.
|
||||
const isSpeaking = Boolean(speaking);
|
||||
const effectiveState = isSpeaking ? 'speaking' : session.pillState;
|
||||
const effectiveElapsed = isSpeaking ? speaking!.elapsedMs : session.pillElapsedMs;
|
||||
const effectiveElapsed = isSpeaking ? speakElapsed : session.pillElapsedMs;
|
||||
|
||||
// When the pill cycle ends (no capture AND no speak), tell Rust to tuck
|
||||
// the window away. Rust owns the hide + park-off-screen + click-through
|
||||
|
||||
Reference in New Issue
Block a user