mirror of
https://github.com/jamiepine/voicebox.git
synced 2026-09-26 13:45:16 -07:00
Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
51f49dea19 | ||
|
|
80610d880e | ||
|
|
397051ba44 | ||
|
|
1ba935e83b | ||
|
|
6a6f4643da | ||
|
|
44ef8daba3 | ||
|
|
68ece25a80 | ||
|
|
1db0fdf645 | ||
|
|
2a001fd63f | ||
|
|
e5813304ef | ||
|
|
a5773807a5 | ||
|
|
ed54347e81 | ||
|
|
624f6a2140 | ||
|
|
669f85024f |
@@ -0,0 +1,2 @@
|
||||
package.json text eol=lf
|
||||
scripts/*.sh text eol=lf
|
||||
+10
-3
@@ -20,8 +20,11 @@ COPY package.json bun.lock CHANGELOG.md ./
|
||||
COPY app/ ./app/
|
||||
COPY web/ ./web/
|
||||
|
||||
# Strip workspaces not needed for web build, and fix trailing comma
|
||||
RUN sed -i '/"tauri"/d; /"landing"/d' package.json && \
|
||||
# Normalize line endings first (a Windows CRLF checkout would otherwise
|
||||
# defeat the `-z 's/,\n ]/…/'` match below, since it's LF-anchored), then
|
||||
# strip workspaces not needed for web build, and fix trailing comma
|
||||
RUN sed -i 's/\r$//' package.json && \
|
||||
sed -i '/"tauri"/d; /"landing"/d' package.json && \
|
||||
sed -i -z 's/,\n ]/\n ]/' package.json
|
||||
RUN bun install --no-save
|
||||
# Build frontend (skip tsc — upstream has pre-existing type errors)
|
||||
@@ -100,7 +103,11 @@ EXPOSE 17493
|
||||
HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=60s \
|
||||
CMD curl -f http://localhost:17493/health || exit 1
|
||||
|
||||
# Entrypoint joins GPU groups then drops to the voicebox user
|
||||
# Entrypoint joins GPU groups then drops to the voicebox user.
|
||||
# Normalize CRLF (a Windows checkout otherwise leaves the shebang as
|
||||
# `#!/bin/sh\r`, which Linux can't resolve — reported as a misleading
|
||||
# "no such file or directory" even though the file exists).
|
||||
COPY --chmod=755 scripts/rocm-entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN sed -i 's/\r$//' /usr/local/bin/entrypoint.sh
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "17493"]
|
||||
|
||||
@@ -139,7 +139,7 @@ export function EngineModelSelector({ form, compact, selectedProfile }: EngineMo
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectContent side={compact ? 'top' : undefined}>
|
||||
{availableOptions.map((opt) => (
|
||||
<SelectItem key={opt.value} value={opt.value} className={itemClass}>
|
||||
{opt.label}
|
||||
|
||||
@@ -555,7 +555,7 @@ export function FloatingGenerateBox({
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all w-full">
|
||||
<SelectValue placeholder={t('generation.voiceSelector.placeholder')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectContent side="top">
|
||||
{profiles?.map((profile) => (
|
||||
<SelectItem key={profile.id} value={profile.id} className="text-xs">
|
||||
{profile.name}
|
||||
@@ -582,7 +582,7 @@ export function FloatingGenerateBox({
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectContent side="top">
|
||||
{engineLangs.map((lang) => (
|
||||
<SelectItem key={lang.value} value={lang.value} className="text-xs">
|
||||
{lang.label}
|
||||
@@ -610,7 +610,7 @@ export function FloatingGenerateBox({
|
||||
<SelectTrigger className="h-8 text-xs bg-card border-border rounded-full hover:bg-background/50 transition-all">
|
||||
<SelectValue placeholder={t('generation.effects.none')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectContent side="top">
|
||||
<SelectItem value="none" className="text-xs">
|
||||
{t('generation.effects.none')}
|
||||
</SelectItem>
|
||||
|
||||
@@ -47,12 +47,14 @@ export function useExportGeneration() {
|
||||
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
||||
const blob = await apiClient.exportGeneration(generationId);
|
||||
|
||||
// Create safe filename from text
|
||||
// Create safe filename from text. Append a short id so exports of
|
||||
// similarly-worded generations don't collide on the same filename
|
||||
// (the first 30 chars are frequently identical).
|
||||
const safeText = text
|
||||
.substring(0, 30)
|
||||
.replace(/[^a-z0-9]/gi, '-')
|
||||
.toLowerCase();
|
||||
const filename = `generation-${safeText}.voicebox.zip`;
|
||||
const filename = `generation-${safeText}-${generationId.substring(0, 8)}.voicebox.zip`;
|
||||
|
||||
await platform.filesystem.saveFile(filename, blob, [
|
||||
{
|
||||
@@ -73,12 +75,14 @@ export function useExportGenerationAudio() {
|
||||
mutationFn: async ({ generationId, text }: { generationId: string; text: string }) => {
|
||||
const blob = await apiClient.exportGenerationAudio(generationId);
|
||||
|
||||
// Create safe filename from text
|
||||
// Create safe filename from text. Append a short id so exports of
|
||||
// similarly-worded generations don't collide on the same filename
|
||||
// (the first 30 chars are frequently identical).
|
||||
const safeText = text
|
||||
.substring(0, 30)
|
||||
.replace(/[^a-z0-9]/gi, '-')
|
||||
.toLowerCase();
|
||||
const filename = `${safeText}.wav`;
|
||||
const filename = `${safeText}-${generationId.substring(0, 8)}.wav`;
|
||||
|
||||
await platform.filesystem.saveFile(filename, blob, [
|
||||
{
|
||||
|
||||
+14
-13
@@ -25,27 +25,28 @@ function getDateLocale() {
|
||||
}
|
||||
}
|
||||
|
||||
export function formatDate(date: string | Date): string {
|
||||
let dateObj: Date;
|
||||
if (typeof date === 'string') {
|
||||
const dateStr = date.trim();
|
||||
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
|
||||
dateObj = new Date(`${dateStr}Z`);
|
||||
} else {
|
||||
dateObj = new Date(dateStr);
|
||||
}
|
||||
} else {
|
||||
dateObj = date;
|
||||
// Backend timestamps are naive UTC — append `Z` so JS doesn't parse a
|
||||
// timezone-less date-time string as local time.
|
||||
function parseServerDate(date: string | Date): Date {
|
||||
if (typeof date !== 'string') {
|
||||
return date;
|
||||
}
|
||||
const dateStr = date.trim();
|
||||
if (!dateStr.includes('Z') && !dateStr.match(/[+-]\d{2}:\d{2}$/)) {
|
||||
return new Date(`${dateStr}Z`);
|
||||
}
|
||||
return new Date(dateStr);
|
||||
}
|
||||
|
||||
return formatDistance(dateObj, new Date(), {
|
||||
export function formatDate(date: string | Date): string {
|
||||
return formatDistance(parseServerDate(date), new Date(), {
|
||||
addSuffix: true,
|
||||
locale: getDateLocale(),
|
||||
}).replace(/^about /i, '');
|
||||
}
|
||||
|
||||
export function formatAbsoluteDate(date: string | Date): string {
|
||||
const dateObj = typeof date === 'string' ? new Date(date) : date;
|
||||
const dateObj = parseServerDate(date);
|
||||
return dateObj.toLocaleString(i18n.language, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
|
||||
@@ -56,6 +56,7 @@ class ModelConfig:
|
||||
model_size: str = "default"
|
||||
size_mb: int = 0
|
||||
needs_trim: bool = False
|
||||
retries_runaway: bool = False
|
||||
supports_instruct: bool = False
|
||||
languages: list[str] = field(default_factory=lambda: ["en"])
|
||||
|
||||
@@ -232,6 +233,10 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
||||
repo_1_7b = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
repo_0_6b = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
|
||||
# mlx-audio can continue after an EOS miss with silence followed by
|
||||
# codec noise. Retry only the affected text as smaller chunks.
|
||||
retries_runaway = backend_type == "mlx"
|
||||
|
||||
return [
|
||||
ModelConfig(
|
||||
model_name="qwen-tts-1.7B",
|
||||
@@ -240,6 +245,7 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
||||
hf_repo_id=repo_1_7b,
|
||||
model_size="1.7B",
|
||||
size_mb=3500,
|
||||
retries_runaway=retries_runaway,
|
||||
supports_instruct=False, # Base model drops instruct silently
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
@@ -250,6 +256,7 @@ def _get_qwen_model_configs() -> list[ModelConfig]:
|
||||
hf_repo_id=repo_0_6b,
|
||||
model_size="0.6B",
|
||||
size_mb=1200,
|
||||
retries_runaway=retries_runaway,
|
||||
supports_instruct=False,
|
||||
languages=["zh", "en", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"],
|
||||
),
|
||||
@@ -504,6 +511,14 @@ def engine_needs_trim(engine: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def engine_retries_runaway(engine: str) -> bool:
|
||||
"""Whether unstable output should be retried in smaller chunks."""
|
||||
for cfg in get_tts_model_configs():
|
||||
if cfg.engine == engine:
|
||||
return cfg.retries_runaway
|
||||
return False
|
||||
|
||||
|
||||
def engine_has_model_sizes(engine: str) -> bool:
|
||||
"""Whether this engine supports multiple model sizes (only Qwen currently)."""
|
||||
configs = [c for c in get_tts_model_configs() if c.engine == engine]
|
||||
|
||||
@@ -248,9 +248,13 @@ class HumeTadaBackend:
|
||||
audio = audio.T # (samples, channels) -> (channels, samples)
|
||||
audio = audio.to(device)
|
||||
|
||||
# Encode with forced alignment
|
||||
# Encode with forced alignment.
|
||||
# Must run under inference_mode: encoder params still require
|
||||
# grad by default, and an autograd graph across the DAC/Snake
|
||||
# stack can balloon VRAM far past the model footprint (#890).
|
||||
text_arg = [reference_text] if reference_text else None
|
||||
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
|
||||
with torch.inference_mode():
|
||||
prompt = self.encoder(audio, text=text_arg, sample_rate=sr)
|
||||
|
||||
# Serialize EncoderOutput to a dict of CPU tensors for caching
|
||||
prompt_dict = {}
|
||||
|
||||
@@ -12,7 +12,7 @@ import base64 as b64
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastmcp import FastMCP
|
||||
|
||||
@@ -49,6 +49,7 @@ def register_tools(mcp: FastMCP) -> None:
|
||||
engine: str | None = None,
|
||||
personality: bool | None = None,
|
||||
language: str | None = None,
|
||||
model_size: Literal["1.7B", "0.6B", "1B", "3B"] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Speak ``text`` in a voice profile.
|
||||
|
||||
@@ -61,6 +62,12 @@ def register_tools(mcp: FastMCP) -> None:
|
||||
LLM before TTS. When omitted, the per-client binding's
|
||||
``default_personality`` flag decides; when that is unset, the
|
||||
default is plain TTS.
|
||||
|
||||
``model_size`` selects a model variant for engines that ship more
|
||||
than one — ``qwen`` and ``qwen_custom_voice`` accept "1.7B" (default)
|
||||
or "0.6B"; ``tada`` accepts "1B" or "3B". Other engines ignore it.
|
||||
Omit to use the engine default. Requesting a smaller variant (e.g.
|
||||
"0.6B") is faster and avoids reloading a heavier model between calls.
|
||||
"""
|
||||
from ..database.models import MCPClientBinding
|
||||
|
||||
@@ -99,6 +106,7 @@ def register_tools(mcp: FastMCP) -> None:
|
||||
engine=resolved_engine,
|
||||
language=language,
|
||||
personality=use_persona,
|
||||
model_size=model_size,
|
||||
db=db,
|
||||
)
|
||||
finally:
|
||||
@@ -228,18 +236,23 @@ async def _speak(
|
||||
engine: str | None,
|
||||
language: str | None,
|
||||
personality: bool,
|
||||
model_size: str | None = None,
|
||||
db,
|
||||
) -> dict[str, Any]:
|
||||
"""Delegate to POST /generate — the route handles personality-rewrite
|
||||
internally when ``personality=true`` and the profile has a prompt."""
|
||||
from ..routes.generations import generate_speech
|
||||
|
||||
# model_size=None is intentional: generate_speech normalizes it to the
|
||||
# engine default (see routes/generations.py), so an omitted size behaves
|
||||
# exactly like the REST /generate endpoint with no model_size in the body.
|
||||
req = models.GenerationRequest(
|
||||
profile_id=profile_id,
|
||||
text=text,
|
||||
language=language or "en",
|
||||
engine=engine,
|
||||
personality=personality,
|
||||
model_size=model_size,
|
||||
)
|
||||
generation = await generate_speech(req, db)
|
||||
return _speak_response(generation, profile_name, source="mcp")
|
||||
|
||||
@@ -321,7 +321,13 @@ async def stream_speech(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Generate speech and stream the WAV audio directly without saving to disk."""
|
||||
from ..backends import get_tts_backend_for_engine, ensure_model_cached_or_raise, load_engine_model, engine_needs_trim
|
||||
from ..backends import (
|
||||
engine_needs_trim,
|
||||
engine_retries_runaway,
|
||||
ensure_model_cached_or_raise,
|
||||
get_tts_backend_for_engine,
|
||||
load_engine_model,
|
||||
)
|
||||
|
||||
profile = await profiles.get_profile(data.profile_id, db)
|
||||
if not profile:
|
||||
@@ -347,10 +353,15 @@ async def stream_speech(
|
||||
from ..utils.chunked_tts import generate_chunked
|
||||
|
||||
trim_fn = None
|
||||
runaway_detector = None
|
||||
if engine_needs_trim(engine):
|
||||
from ..utils.audio import trim_tts_output
|
||||
|
||||
trim_fn = trim_tts_output
|
||||
if engine_retries_runaway(engine):
|
||||
from ..utils.audio import has_tts_runaway
|
||||
|
||||
runaway_detector = has_tts_runaway
|
||||
|
||||
audio, sample_rate = await generate_chunked(
|
||||
tts_model,
|
||||
@@ -362,6 +373,7 @@ async def stream_speech(
|
||||
max_chunk_chars=data.max_chunk_chars,
|
||||
crossfade_ms=data.crossfade_ms,
|
||||
trim_fn=trim_fn,
|
||||
runaway_detector=runaway_detector,
|
||||
)
|
||||
|
||||
effects_chain_config = None
|
||||
|
||||
@@ -151,7 +151,9 @@ async def export_generation(
|
||||
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
||||
if not safe_text:
|
||||
safe_text = "generation"
|
||||
filename = f"generation-{safe_text}.voicebox.zip"
|
||||
# Append a short id so exports of similarly-worded generations don't collide
|
||||
# on the same filename (the first 30 chars are frequently identical).
|
||||
filename = f"generation-{safe_text}-{generation_id[:8]}.voicebox.zip"
|
||||
|
||||
return StreamingResponse(
|
||||
io.BytesIO(zip_bytes),
|
||||
@@ -180,7 +182,9 @@ async def export_generation_audio(
|
||||
safe_text = "".join(c for c in generation.text[:30] if c.isalnum() or c in (" ", "-", "_")).strip()
|
||||
if not safe_text:
|
||||
safe_text = "generation"
|
||||
filename = f"{safe_text}.wav"
|
||||
# Append a short id so exports of similarly-worded generations don't collide
|
||||
# on the same filename (the first 30 chars are frequently identical).
|
||||
filename = f"{safe_text}-{generation_id[:8]}.wav"
|
||||
|
||||
return FileResponse(
|
||||
audio_path,
|
||||
|
||||
@@ -232,7 +232,7 @@ async def upload_profile_avatar(
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Upload or update avatar image for a profile."""
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename).suffix) as tmp:
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=Path(file.filename or "").suffix) as tmp:
|
||||
content = await file.read()
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
@@ -35,13 +35,25 @@ async def transcribe_audio(
|
||||
tmp.write(chunk)
|
||||
tmp_path = tmp.name
|
||||
|
||||
stt_path = tmp_path
|
||||
try:
|
||||
from ..utils.audio import load_audio
|
||||
from ..utils.audio import load_audio, save_audio
|
||||
from ..backends import WHISPER_HF_REPOS
|
||||
|
||||
audio, sr = await asyncio.to_thread(load_audio, tmp_path)
|
||||
duration = len(audio) / sr
|
||||
|
||||
# The STT backend (mlx_audio.stt -> miniaudio) only decodes
|
||||
# WAV/FLAC/MP3/Vorbis, so browser recordings uploaded as WebM/Opus
|
||||
# fail with "unsupported file format" (issue: web-mode dictation).
|
||||
# librosa already decoded the file above (it falls back to
|
||||
# audioread/ffmpeg for exotic containers), so re-encode that PCM to a
|
||||
# temp WAV and hand *that* to Whisper. WAV inputs pass through
|
||||
# unchanged.
|
||||
if file_suffix != ".wav":
|
||||
stt_path = f"{tmp_path}.stt.wav"
|
||||
await asyncio.to_thread(save_audio, audio, stt_path, sr)
|
||||
|
||||
whisper_model = transcribe.get_whisper_model()
|
||||
model_size = model if model else whisper_model.model_size
|
||||
|
||||
@@ -76,7 +88,7 @@ async def transcribe_audio(
|
||||
},
|
||||
)
|
||||
|
||||
text = await whisper_model.transcribe(tmp_path, language, model_size)
|
||||
text = await whisper_model.transcribe(stt_path, language, model_size)
|
||||
|
||||
return models.TranscriptionResponse(
|
||||
text=text,
|
||||
@@ -89,3 +101,5 @@ async def transcribe_audio(
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
finally:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
if stt_path != tmp_path:
|
||||
Path(stt_path).unlink(missing_ok=True)
|
||||
|
||||
@@ -48,9 +48,14 @@ async def run_generation(
|
||||
This is the single entry point for all background generation work.
|
||||
It is designed to be enqueued via ``services.task_queue.enqueue_generation``.
|
||||
"""
|
||||
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
|
||||
from ..backends import (
|
||||
engine_needs_trim,
|
||||
engine_retries_runaway,
|
||||
get_tts_backend_for_engine,
|
||||
load_engine_model,
|
||||
)
|
||||
from ..utils.chunked_tts import generate_chunked
|
||||
from ..utils.audio import normalize_audio, save_audio, trim_tts_output
|
||||
from ..utils.audio import has_tts_runaway, normalize_audio, save_audio, trim_tts_output
|
||||
|
||||
task_manager = get_task_manager()
|
||||
bg_db = next(get_db())
|
||||
@@ -72,12 +77,14 @@ async def run_generation(
|
||||
|
||||
await history.update_generation_status(generation_id, "generating", bg_db)
|
||||
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
||||
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
|
||||
|
||||
gen_kwargs: dict = dict(
|
||||
language=language,
|
||||
seed=seed if mode != "regenerate" else None,
|
||||
instruct=instruct,
|
||||
trim_fn=trim_fn,
|
||||
runaway_detector=runaway_detector,
|
||||
)
|
||||
if max_chunk_chars is not None:
|
||||
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
||||
@@ -267,9 +274,14 @@ async def generate_audio_sync(
|
||||
normalize, then encodes in-memory via :func:`tts.audio_to_wav_bytes`
|
||||
(same helper ``/generate/stream`` uses).
|
||||
"""
|
||||
from ..backends import load_engine_model, get_tts_backend_for_engine, engine_needs_trim
|
||||
from ..backends import (
|
||||
engine_needs_trim,
|
||||
engine_retries_runaway,
|
||||
get_tts_backend_for_engine,
|
||||
load_engine_model,
|
||||
)
|
||||
from ..utils.chunked_tts import generate_chunked
|
||||
from ..utils.audio import normalize_audio, trim_tts_output
|
||||
from ..utils.audio import has_tts_runaway, normalize_audio, trim_tts_output
|
||||
from . import tts
|
||||
|
||||
bg_db = next(get_db())
|
||||
@@ -287,12 +299,14 @@ async def generate_audio_sync(
|
||||
bg_db.close()
|
||||
|
||||
trim_fn = trim_tts_output if engine_needs_trim(engine) else None
|
||||
runaway_detector = has_tts_runaway if engine_retries_runaway(engine) else None
|
||||
|
||||
gen_kwargs: dict = dict(
|
||||
language=language,
|
||||
seed=seed,
|
||||
instruct=instruct,
|
||||
trim_fn=trim_fn,
|
||||
runaway_detector=runaway_detector,
|
||||
)
|
||||
if max_chunk_chars is not None:
|
||||
gen_kwargs["max_chunk_chars"] = max_chunk_chars
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Ensure TADA voice-prompt encoding disables autograd (#890)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from backend.backends.hume_backend import HumeTadaBackend
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeEncoderOutput:
|
||||
emb: torch.Tensor
|
||||
|
||||
|
||||
class _GradTrackingEncoder:
|
||||
"""Raises unless called under torch.inference_mode()."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.called_under_inference_mode = False
|
||||
|
||||
def __call__(self, audio, text=None, sample_rate=None):
|
||||
self.called_under_inference_mode = torch.is_inference_mode_enabled()
|
||||
if not self.called_under_inference_mode:
|
||||
raise AssertionError("encoder forward must run under inference_mode")
|
||||
# Touch a requires_grad tensor the way Snake1d alpha would.
|
||||
alpha = torch.nn.Parameter(torch.ones(1, device=audio.device))
|
||||
_ = audio.mean() * alpha
|
||||
return _FakeEncoderOutput(emb=torch.zeros(1, 4, device=audio.device))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_voice_prompt_runs_encoder_under_inference_mode(tmp_path, monkeypatch):
|
||||
wav = tmp_path / "ref.wav"
|
||||
sf.write(str(wav), np.zeros(24000, dtype=np.float32), 24000)
|
||||
|
||||
backend = HumeTadaBackend()
|
||||
backend.model = object() # mark loaded
|
||||
backend.model_size = "1B"
|
||||
backend._device = "cpu"
|
||||
encoder = _GradTrackingEncoder()
|
||||
backend.encoder = encoder
|
||||
|
||||
monkeypatch.setattr(backend, "load_model", AsyncMock(return_value=None))
|
||||
monkeypatch.setattr(
|
||||
"backend.backends.hume_backend.get_cached_voice_prompt",
|
||||
lambda key: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"backend.backends.hume_backend.cache_voice_prompt",
|
||||
lambda key, value: None,
|
||||
)
|
||||
|
||||
prompt, from_cache = await backend.create_voice_prompt(
|
||||
str(wav),
|
||||
reference_text="hello world",
|
||||
use_cache=False,
|
||||
)
|
||||
|
||||
assert from_cache is False
|
||||
assert encoder.called_under_inference_mode is True
|
||||
assert isinstance(prompt["emb"], torch.Tensor)
|
||||
assert prompt["emb"].device.type == "cpu"
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Tests for the voicebox.speak MCP tool's ``model_size`` plumbing (issue #884).
|
||||
|
||||
The MCP speak path used to build its ``GenerationRequest`` without a
|
||||
``model_size``, so every agent-triggered generation silently fell back to the
|
||||
schema default ("1.7B") — there was no way to reach 0.6B (or TADA's 1B/3B)
|
||||
through MCP. These tests pin the fix: ``_speak`` now forwards ``model_size``
|
||||
straight into the request, matching the REST ``/generate`` surface.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
import backend.routes.generations as generations
|
||||
from backend.mcp_server import tools
|
||||
|
||||
|
||||
class _FakeGeneration:
|
||||
"""Minimal stand-in for GenerationResponse consumed by ``_speak_response``."""
|
||||
|
||||
def model_dump(self, mode="json"):
|
||||
return {"id": "gen-test", "status": "generating"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def captured_request(monkeypatch):
|
||||
"""Replace the real (torch-backed) generate_speech with a capturing stub.
|
||||
|
||||
``_speak`` imports ``generate_speech`` lazily from ``routes.generations``,
|
||||
so patching the attribute on that module intercepts the call and lets us
|
||||
inspect the ``GenerationRequest`` it would have run.
|
||||
"""
|
||||
captured = {}
|
||||
|
||||
async def fake_generate_speech(req, db):
|
||||
captured["req"] = req
|
||||
return _FakeGeneration()
|
||||
|
||||
monkeypatch.setattr(generations, "generate_speech", fake_generate_speech)
|
||||
# Isolate the unit from the MCP event bus — _speak_response fires a
|
||||
# speak-start event we don't care about here.
|
||||
monkeypatch.setattr(tools.mcp_events, "publish", lambda *a, **k: None)
|
||||
return captured
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_speak_forwards_explicit_model_size(captured_request):
|
||||
await tools._speak(
|
||||
profile_id="p1",
|
||||
profile_name="Morgan",
|
||||
text="hello",
|
||||
engine="qwen",
|
||||
language="en",
|
||||
personality=False,
|
||||
model_size="0.6B",
|
||||
db=None,
|
||||
)
|
||||
assert captured_request["req"].model_size == "0.6B"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_speak_omitted_model_size_is_none(captured_request):
|
||||
# Omitted → None; generate_speech normalizes None to the engine default,
|
||||
# so this reproduces the pre-fix behaviour for callers that don't ask.
|
||||
await tools._speak(
|
||||
profile_id="p1",
|
||||
profile_name="Morgan",
|
||||
text="hello",
|
||||
engine="qwen",
|
||||
language="en",
|
||||
personality=False,
|
||||
db=None,
|
||||
)
|
||||
assert captured_request["req"].model_size is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_speak_rejects_invalid_model_size(captured_request):
|
||||
# The GenerationRequest schema pattern is the single source of truth for
|
||||
# valid sizes; a bad value is rejected before any generation runs.
|
||||
with pytest.raises(ValidationError):
|
||||
await tools._speak(
|
||||
profile_id="p1",
|
||||
profile_name="Morgan",
|
||||
text="hello",
|
||||
engine="qwen",
|
||||
language="en",
|
||||
personality=False,
|
||||
model_size="9B",
|
||||
db=None,
|
||||
)
|
||||
assert "req" not in captured_request
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Regression coverage for runaway MLX Qwen TTS output."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from backend.backends import engine_needs_trim, engine_retries_runaway
|
||||
from backend.utils.audio import has_tts_runaway
|
||||
from backend.utils.chunked_tts import generate_chunked
|
||||
|
||||
SAMPLE_RATE = 1000
|
||||
|
||||
|
||||
def test_mlx_qwen_enables_runaway_retry_without_aggressive_trim():
|
||||
with patch("backend.backends.get_backend_type", return_value="mlx"):
|
||||
assert engine_needs_trim("qwen") is False
|
||||
assert engine_retries_runaway("qwen") is True
|
||||
|
||||
|
||||
def test_pytorch_qwen_keeps_runaway_retry_disabled():
|
||||
with patch("backend.backends.get_backend_type", return_value="pytorch"):
|
||||
assert engine_needs_trim("qwen") is False
|
||||
assert engine_retries_runaway("qwen") is False
|
||||
|
||||
|
||||
def test_detector_flags_long_internal_silence():
|
||||
speech = np.full(2 * SAMPLE_RATE, 0.2, dtype=np.float32)
|
||||
runaway_gap = np.zeros(2500, dtype=np.float32)
|
||||
hallucinated_noise = np.full(2 * SAMPLE_RATE, 0.8, dtype=np.float32)
|
||||
audio = np.concatenate([speech, runaway_gap, hallucinated_noise])
|
||||
|
||||
assert has_tts_runaway(audio, SAMPLE_RATE) is True
|
||||
|
||||
|
||||
def test_detector_ignores_normal_internal_pause():
|
||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
||||
normal_pause = np.zeros(1200, dtype=np.float32)
|
||||
audio = np.concatenate([speech, normal_pause, speech])
|
||||
|
||||
assert has_tts_runaway(audio, SAMPLE_RATE) is False
|
||||
|
||||
|
||||
def test_trailing_silence_is_not_a_runaway():
|
||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
||||
trailing_silence = np.zeros(2 * SAMPLE_RATE, dtype=np.float32)
|
||||
|
||||
assert (
|
||||
has_tts_runaway(
|
||||
np.concatenate([speech, trailing_silence]),
|
||||
SAMPLE_RATE,
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runaway_chunk_is_retried_as_smaller_chunks():
|
||||
class FakeBackend:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
async def generate(self, text, *_args):
|
||||
self.calls.append(text)
|
||||
if len(text) > 200:
|
||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
||||
silence = np.zeros(2500, dtype=np.float32)
|
||||
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
|
||||
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
|
||||
return np.full(SAMPLE_RATE, 0.2, dtype=np.float32), SAMPLE_RATE
|
||||
|
||||
backend = FakeBackend()
|
||||
text = f"{'A' * 119}. {'B' * 119}."
|
||||
|
||||
audio, sample_rate = await generate_chunked(
|
||||
backend,
|
||||
text,
|
||||
{},
|
||||
max_chunk_chars=800,
|
||||
crossfade_ms=50,
|
||||
runaway_detector=has_tts_runaway,
|
||||
)
|
||||
|
||||
assert sample_rate == SAMPLE_RATE
|
||||
assert backend.calls == [text, f"{'A' * 119}.", f"{'B' * 119}."]
|
||||
assert len(audio) == 1950
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_persistent_runaway_fails_instead_of_returning_corrupt_audio():
|
||||
class AlwaysRunawayBackend:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
async def generate(self, text, *_args):
|
||||
self.calls.append(text)
|
||||
speech = np.full(SAMPLE_RATE, 0.2, dtype=np.float32)
|
||||
silence = np.zeros(2500, dtype=np.float32)
|
||||
noise = np.full(SAMPLE_RATE, 0.8, dtype=np.float32)
|
||||
return np.concatenate([speech, silence, noise]), SAMPLE_RATE
|
||||
|
||||
backend = AlwaysRunawayBackend()
|
||||
text = f"{'A' * 119}. {'B' * 119}."
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="remained unstable after retrying smaller text chunks",
|
||||
):
|
||||
await generate_chunked(
|
||||
backend,
|
||||
text,
|
||||
{},
|
||||
max_chunk_chars=800,
|
||||
runaway_detector=has_tts_runaway,
|
||||
)
|
||||
|
||||
assert [len(call) for call in backend.calls] == [241, 120, 100]
|
||||
@@ -110,6 +110,43 @@ def save_audio(
|
||||
raise OSError(f"Failed to save audio to {path}: {e}") from e
|
||||
|
||||
|
||||
def has_tts_runaway(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = 24000,
|
||||
frame_ms: int = 20,
|
||||
silence_threshold_db: float = -40.0,
|
||||
max_internal_silence_ms: int = 2000,
|
||||
) -> bool:
|
||||
"""Detect speech followed by a long silence and then more output.
|
||||
|
||||
This shape is a reliable signal that a TTS model missed EOS and resumed
|
||||
with hallucinated speech or codec noise. Leading and trailing silence do
|
||||
not count because they are not bounded by non-silent audio.
|
||||
"""
|
||||
frame_len = int(sample_rate * frame_ms / 1000)
|
||||
if frame_len == 0 or len(audio) < frame_len:
|
||||
return False
|
||||
|
||||
n_frames = len(audio) // frame_len
|
||||
threshold_linear = 10 ** (silence_threshold_db / 20)
|
||||
max_silence_frames = int(max_internal_silence_ms / frame_ms)
|
||||
seen_speech = False
|
||||
consecutive_silence = 0
|
||||
|
||||
for i in range(n_frames):
|
||||
frame = audio[i * frame_len : (i + 1) * frame_len]
|
||||
is_speech = np.sqrt(np.mean(frame**2)) >= threshold_linear
|
||||
if is_speech:
|
||||
if seen_speech and consecutive_silence >= max_silence_frames:
|
||||
return True
|
||||
seen_speech = True
|
||||
consecutive_silence = 0
|
||||
elif seen_speech:
|
||||
consecutive_silence += 1
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def trim_tts_output(
|
||||
audio: np.ndarray,
|
||||
sample_rate: int = 24000,
|
||||
|
||||
@@ -20,6 +20,8 @@ logger = logging.getLogger("voicebox.chunked-tts")
|
||||
# Default chunk size in characters. Can be overridden per-request via
|
||||
# the ``max_chunk_chars`` field on GenerationRequest.
|
||||
DEFAULT_MAX_CHUNK_CHARS = 800
|
||||
MAX_RUNAWAY_RETRIES = 2
|
||||
MIN_RUNAWAY_RETRY_CHARS = 100
|
||||
|
||||
# Common abbreviations that should NOT be treated as sentence endings.
|
||||
# Lowercase for case-insensitive matching.
|
||||
@@ -211,6 +213,7 @@ async def generate_chunked(
|
||||
max_chunk_chars: int = DEFAULT_MAX_CHUNK_CHARS,
|
||||
crossfade_ms: int = 50,
|
||||
trim_fn=None,
|
||||
runaway_detector=None,
|
||||
) -> Tuple[np.ndarray, int]:
|
||||
"""Generate audio with automatic chunking for long text.
|
||||
|
||||
@@ -239,25 +242,75 @@ async def generate_chunked(
|
||||
Optional ``(audio, sample_rate) -> audio`` post-processing
|
||||
function applied to each chunk before concatenation (e.g.
|
||||
``trim_tts_output`` for Chatterbox engines).
|
||||
runaway_detector : callable | None
|
||||
Optional ``(audio, sample_rate) -> bool`` detector. When it flags
|
||||
unstable output, the affected text is split in half and retried.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(audio, sample_rate) : Tuple[np.ndarray, int]
|
||||
"""
|
||||
async def generate_one(
|
||||
chunk_text: str,
|
||||
chunk_seed: int | None,
|
||||
retry_depth: int = 0,
|
||||
) -> tuple[np.ndarray, int]:
|
||||
chunk_audio, chunk_sr = await backend.generate(
|
||||
chunk_text,
|
||||
voice_prompt,
|
||||
language,
|
||||
chunk_seed,
|
||||
instruct,
|
||||
)
|
||||
|
||||
if runaway_detector is not None and runaway_detector(chunk_audio, chunk_sr):
|
||||
if retry_depth >= MAX_RUNAWAY_RETRIES or len(chunk_text) <= MIN_RUNAWAY_RETRY_CHARS:
|
||||
raise RuntimeError(
|
||||
"TTS output remained unstable after retrying smaller text chunks"
|
||||
)
|
||||
|
||||
retry_max_chars = max(MIN_RUNAWAY_RETRY_CHARS, len(chunk_text) // 2)
|
||||
retry_chunks = split_text_into_chunks(chunk_text, retry_max_chars)
|
||||
if len(retry_chunks) <= 1:
|
||||
raise RuntimeError("Unable to split unstable TTS output for retry")
|
||||
|
||||
logger.warning(
|
||||
"Detected unstable TTS output for %d chars; retrying as %d smaller chunks",
|
||||
len(chunk_text),
|
||||
len(retry_chunks),
|
||||
)
|
||||
retry_audio: list[np.ndarray] = []
|
||||
for i, retry_text in enumerate(retry_chunks):
|
||||
retry_seed = (
|
||||
chunk_seed + ((retry_depth + 1) * 1000) + i
|
||||
if chunk_seed is not None
|
||||
else None
|
||||
)
|
||||
audio, sample_rate = await generate_one(
|
||||
retry_text,
|
||||
retry_seed,
|
||||
retry_depth + 1,
|
||||
)
|
||||
retry_audio.append(np.asarray(audio, dtype=np.float32))
|
||||
|
||||
return (
|
||||
concatenate_audio_chunks(
|
||||
retry_audio,
|
||||
sample_rate,
|
||||
crossfade_ms=crossfade_ms,
|
||||
),
|
||||
sample_rate,
|
||||
)
|
||||
|
||||
if trim_fn is not None:
|
||||
chunk_audio = trim_fn(chunk_audio, chunk_sr)
|
||||
return np.asarray(chunk_audio, dtype=np.float32), chunk_sr
|
||||
|
||||
chunks = split_text_into_chunks(text, max_chunk_chars)
|
||||
|
||||
if len(chunks) <= 1:
|
||||
# Short text — single-shot fast path
|
||||
audio, sample_rate = await backend.generate(
|
||||
text,
|
||||
voice_prompt,
|
||||
language,
|
||||
seed,
|
||||
instruct,
|
||||
)
|
||||
if trim_fn is not None:
|
||||
audio = trim_fn(audio, sample_rate)
|
||||
return audio, sample_rate
|
||||
return await generate_one(text, seed)
|
||||
|
||||
# Long text — chunked generation
|
||||
logger.info(
|
||||
@@ -281,17 +334,12 @@ async def generate_chunked(
|
||||
# always produces the same output.
|
||||
chunk_seed = (seed + i) if seed is not None else None
|
||||
|
||||
chunk_audio, chunk_sr = await backend.generate(
|
||||
chunk_audio, chunk_sr = await generate_one(
|
||||
chunk_text,
|
||||
voice_prompt,
|
||||
language,
|
||||
chunk_seed,
|
||||
instruct,
|
||||
)
|
||||
if trim_fn is not None:
|
||||
chunk_audio = trim_fn(chunk_audio, chunk_sr)
|
||||
|
||||
audio_chunks.append(np.asarray(chunk_audio, dtype=np.float32))
|
||||
audio_chunks.append(chunk_audio)
|
||||
if sample_rate is None:
|
||||
sample_rate = chunk_sr
|
||||
|
||||
|
||||
@@ -34,3 +34,15 @@ services:
|
||||
|
||||
# Tune the ROCm memory allocator
|
||||
- PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,max_split_size_mb:512
|
||||
|
||||
# Redirect MIOpen kernel cache to a writable, persistent directory.
|
||||
# Without this, MIOpen may fail to write its cache and throw
|
||||
# miopenStatusUnknownError on fresh containers.
|
||||
- MIOPEN_USER_DB_PATH=/app/data/cache/miopen_db
|
||||
- MIOPEN_CUSTOM_CACHE_DIR=/app/data/cache/miopen_cache
|
||||
|
||||
# Use fast heuristics for kernel selection instead of exhaustive
|
||||
# benchmarking. On RDNA4, exhaustive mode tries kernels that fail to
|
||||
# allocate workspace memory (ptr: 0 size: 0), causing system stuttering
|
||||
# on every generation even when the cache is present.
|
||||
- MIOPEN_FIND_MODE=FAST
|
||||
|
||||
@@ -49,6 +49,7 @@ class ModelConfig:
|
||||
model_size: str = "default"
|
||||
size_mb: int = 0
|
||||
needs_trim: bool = False
|
||||
retries_runaway: bool = False
|
||||
supports_instruct: bool = False
|
||||
languages: list[str] = field(default_factory=lambda: ["en"])
|
||||
```
|
||||
@@ -59,6 +60,7 @@ Registry helpers in `backends/__init__.py` replace what used to be per-engine `i
|
||||
- `get_tts_model_configs()` — only TTS variants
|
||||
- `get_model_config(model_name)` — lookup by name
|
||||
- `engine_needs_trim(engine)` — whether output should run through `trim_tts_output()`
|
||||
- `engine_retries_runaway(engine)` — whether unstable output should be retried as smaller chunks
|
||||
- `load_engine_model(engine, model_size)` — downloads + loads, handles engines with multiple sizes
|
||||
- `get_tts_backend_for_engine(engine)` — thread-safe backend factory with double-checked locking
|
||||
|
||||
@@ -152,7 +154,7 @@ The request path from frontend to audio file:
|
||||
|
||||
6. **Inference** — the engine's `generate()` returns `(audio_array, sample_rate)`.
|
||||
|
||||
7. **Post-process** — if `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
|
||||
7. **Validate and post-process** — engines with `retries_runaway=True` retry unstable output as smaller chunks. If `engine_needs_trim(engine)` is True, `trim_tts_output()` strips trailing silence. Effects chains (if any) are applied per generation version, not the clean version.
|
||||
|
||||
8. **Persist** — audio is written to the generations directory, a row is inserted into the `generations` table, and the response includes the generation metadata.
|
||||
|
||||
|
||||
@@ -14,12 +14,12 @@ Make sure you have [installed Voicebox](/overview/installation) and launched the
|
||||
Voice profiles are the foundation of Voicebox. Each profile contains voice samples that the AI uses to clone the voice.
|
||||
|
||||
<Steps>
|
||||
<Step title="Navigate to Profiles">
|
||||
Click the **Profiles** tab in the sidebar
|
||||
<Step title="Navigate to Voices">
|
||||
Click the **Voices** tab in the sidebar
|
||||
</Step>
|
||||
|
||||
<Step title="Create New Profile">
|
||||
Click the **+ New Profile** button
|
||||
<Step title="Create New Voice">
|
||||
Click the **+ New Voice** button
|
||||
|
||||
Fill in the details:
|
||||
- **Name:** A descriptive name (e.g., "John Smith")
|
||||
|
||||
@@ -66,13 +66,46 @@ fn find_monitor_source_via_pactl() -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Select the capture device: prefer an exact match against the monitor
|
||||
/// source name reported by `pactl`, then fall back to any device whose name
|
||||
/// contains "monitor", then the host's default input device.
|
||||
fn select_capture_device(host: &cpal::Host, monitor_source: Option<&str>) -> Option<cpal::Device> {
|
||||
let devices: Vec<cpal::Device> = host.input_devices().ok()?.collect();
|
||||
|
||||
if let Some(target) = monitor_source {
|
||||
if let Some(pos) = devices
|
||||
.iter()
|
||||
.position(|d| d.name().map(|n| n == target).unwrap_or(false))
|
||||
{
|
||||
eprintln!(
|
||||
"Linux audio capture: Using pactl monitor device: {}",
|
||||
target
|
||||
);
|
||||
return devices.into_iter().nth(pos);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(pos) = devices.iter().position(|d| {
|
||||
d.name()
|
||||
.map(|n| n.to_lowercase().contains("monitor"))
|
||||
.unwrap_or(false)
|
||||
}) {
|
||||
let name = devices[pos].name().unwrap_or_default();
|
||||
eprintln!("Linux audio capture: Found monitor device by name: {}", name);
|
||||
return devices.into_iter().nth(pos);
|
||||
}
|
||||
|
||||
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
||||
host.default_input_device()
|
||||
}
|
||||
|
||||
/// Start capturing system audio on Linux using PulseAudio monitor sources.
|
||||
///
|
||||
/// On modern Linux with PulseAudio or PipeWire, we first try to detect the
|
||||
/// monitor source via `pactl` and set the `PULSE_SOURCE` environment variable.
|
||||
/// This tells PulseAudio's ALSA plugin to use the monitor as the default input
|
||||
/// source for this process. If `pactl` is unavailable, we fall back to searching
|
||||
/// cpal device names for "monitor".
|
||||
/// monitor source via `pactl`, then select the matching cpal input device by
|
||||
/// name. This avoids mutating the process environment (`PULSE_SOURCE`), which
|
||||
/// is not thread-safe and would affect every thread in the process. If `pactl`
|
||||
/// is unavailable, we fall back to searching cpal device names for "monitor".
|
||||
pub async fn start_capture(
|
||||
state: &AudioCaptureState,
|
||||
max_duration_secs: u32,
|
||||
@@ -101,73 +134,16 @@ pub async fn start_capture(
|
||||
|
||||
// Spawn capture on a dedicated thread
|
||||
thread::spawn(move || {
|
||||
// Try to set PULSE_SOURCE to a monitor before initializing cpal.
|
||||
// This tells PulseAudio/PipeWire's ALSA plugin to use the monitor
|
||||
// as the default input source for this process.
|
||||
let monitor_source = find_monitor_source_via_pactl();
|
||||
if let Some(ref source_name) = monitor_source {
|
||||
eprintln!(
|
||||
"Linux audio capture: Setting PULSE_SOURCE={}",
|
||||
source_name
|
||||
);
|
||||
std::env::set_var("PULSE_SOURCE", source_name);
|
||||
}
|
||||
|
||||
let host = cpal::default_host();
|
||||
let monitor_source = find_monitor_source_via_pactl();
|
||||
|
||||
// Select the capture device.
|
||||
// If PULSE_SOURCE was set, the default input device IS the monitor.
|
||||
// Otherwise, fall back to searching device names for "monitor".
|
||||
let device = if monitor_source.is_some() {
|
||||
// PULSE_SOURCE was set — default input IS the monitor now
|
||||
match host.default_input_device() {
|
||||
Some(d) => {
|
||||
let name = d.name().unwrap_or_default();
|
||||
eprintln!(
|
||||
"Linux audio capture: Using PULSE_SOURCE monitor device: {}",
|
||||
name
|
||||
);
|
||||
d
|
||||
}
|
||||
None => {
|
||||
let error_msg = "No audio input device available".to_string();
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// pactl not available — try to find monitor by name (original approach)
|
||||
let mut monitor_device = None;
|
||||
if let Ok(devices) = host.input_devices() {
|
||||
for d in devices {
|
||||
if let Ok(name) = d.name() {
|
||||
let name_lower = name.to_lowercase();
|
||||
if name_lower.contains("monitor") {
|
||||
eprintln!(
|
||||
"Linux audio capture: Found monitor device by name: {}",
|
||||
name
|
||||
);
|
||||
monitor_device = Some(d);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
match monitor_device {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
eprintln!("Linux audio capture: No monitor device found, falling back to default input");
|
||||
match host.default_input_device() {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
let error_msg = "No audio input device available".to_string();
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
let device = match select_capture_device(&host, monitor_source.as_deref()) {
|
||||
Some(d) => d,
|
||||
None => {
|
||||
let error_msg = "No audio input device available".to_string();
|
||||
eprintln!("{}", error_msg);
|
||||
*error_arc.lock().unwrap() = Some(error_msg);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ pub fn key_from_str(name: &str) -> Option<Key> {
|
||||
"ShiftLeft" => Key::ShiftLeft,
|
||||
"ShiftRight" => Key::ShiftRight,
|
||||
"CapsLock" => Key::CapsLock,
|
||||
"Function" => Key::Function,
|
||||
|
||||
// Whitespace / navigation
|
||||
"Space" => Key::Space,
|
||||
|
||||
@@ -4,10 +4,14 @@
|
||||
//! pipeline so the focused app performs its native paste action against
|
||||
//! whatever the clipboard module has just staged.
|
||||
//!
|
||||
//! - **macOS** — Cmd down, V down with Cmd flag, V up with Cmd flag, Cmd
|
||||
//! up via `CGEventPost` at `kCGHIDEventTap`. Accessibility permission is
|
||||
//! load-bearing: without it the system swallows the events silently, so
|
||||
//! callers must gate on [`crate::accessibility::is_trusted`].
|
||||
//! - **macOS** — Cmd down with Cmd flag, V down with Cmd flag, V up with
|
||||
//! Cmd flag, Cmd up via `CGEventPost` at `kCGHIDEventTap`. The Cmd-down
|
||||
//! event carries the Command flag so its `flagsChanged` representation
|
||||
//! matches hardware — Electron/Chromium tracks modifier state from that
|
||||
//! flag and drops the paste otherwise (see the note on the event table).
|
||||
//! Accessibility permission is load-bearing: without it the system
|
||||
//! swallows the events silently, so callers must gate on
|
||||
//! [`crate::accessibility::is_trusted`].
|
||||
//! - **Windows** — Ctrl down, V down, V up, Ctrl up via `SendInput`. No
|
||||
//! permission gate, but UAC/UIPI blocks delivery into elevated target
|
||||
//! windows when we run non-elevated — nothing we can do short of also
|
||||
@@ -101,7 +105,18 @@ pub fn send_paste() -> Result<(), String> {
|
||||
let _source_guard = scopeguard::guard(source, |s| CFRelease(s as *const c_void));
|
||||
|
||||
let events = [
|
||||
(KEYCODE_LEFT_CMD, true, 0),
|
||||
// The Cmd-down event must carry the Command flag itself. On real
|
||||
// hardware the Cmd keyDown is a flagsChanged event whose flags
|
||||
// already include Command; Chromium/Electron builds its tracked
|
||||
// modifier state from that flag. Posting Cmd-down with flags = 0
|
||||
// leaves that tracker showing "Command up", so the following V —
|
||||
// even though its own flags carry Command — matches neither the
|
||||
// Cmd+V accelerator (tracker says no modifier) nor plain-text
|
||||
// insertion (event flags say Command), and Electron drops it
|
||||
// silently. AppKit reads the V event's own flags and pastes
|
||||
// regardless, which is why native apps worked but Electron
|
||||
// targets (Slack, VS Code) silently no-op'd.
|
||||
(KEYCODE_LEFT_CMD, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(v_keycode, true, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(v_keycode, false, K_CG_EVENT_FLAG_MASK_COMMAND),
|
||||
(KEYCODE_LEFT_CMD, false, 0),
|
||||
|
||||
Reference in New Issue
Block a user