From 37b7110e526aa3663959c18ad58a618b789389c4 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Sat, 8 Aug 2026 15:40:36 -0700 Subject: [PATCH] test: Playwright E2E with real backend + fake TTS engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e2e/ drives the web build (vite preview) against a per-worker uvicorn: each Playwright worker boots its own backend on its own port with a temp cwd, so SQLite and audio files are fully isolated and parallel- safe. The page fixture seeds the persisted voicebox-server store with the worker's URL before any script runs. VOICEBOX_FAKE_TTS=1 short-circuits get_tts_backend_for_engine to a backend that synthesizes a sine tone sized to the text — the real task queue, SSE progress, history rows, and audio serving all run, only inference is fake. backend/requirements-ci.txt is the slim dependency set validated to boot the app on a CPU-only runner. Specs: startup, settings layout, seeded profile in /voices, and generate-to-completed-audio through the full pipeline. --- .gitignore | Bin 948 -> 1056 bytes backend/backends/__init__.py | 8 +++ backend/backends/fake_backend.py | 92 ++++++++++++++++++++++++++++ backend/requirements-ci.txt | 25 ++++++++ bun.lock | 3 + e2e/fixtures.ts | 99 +++++++++++++++++++++++++++++++ e2e/helpers/api.ts | 58 ++++++++++++++++++ e2e/playwright.config.ts | 36 +++++++++++ e2e/specs/generate.spec.ts | 31 ++++++++++ e2e/specs/startup.spec.ts | 16 +++++ e2e/specs/voices.spec.ts | 10 ++++ package.json | 1 + 12 files changed, 379 insertions(+) create mode 100644 backend/backends/fake_backend.py create mode 100644 backend/requirements-ci.txt create mode 100644 e2e/fixtures.ts create mode 100644 e2e/helpers/api.ts create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/specs/generate.spec.ts create mode 100644 e2e/specs/startup.spec.ts create mode 100644 e2e/specs/voices.spec.ts diff --git a/.gitignore b/.gitignore index 853c5060975fbeadd1fbd442d6dfb9c9a648813d..3b6c46e14469f4ba06d37ef0b75f28f869c84ab4 100644 GIT binary patch delta 116 zcmW;C!41MN3`XGt2au}@=r-jV F=?BcEDrf)z delta 7 OcmZ3$v4wrZ7G?kovI5Hh diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py index abaf70c1..c7870268 100644 --- a/backend/backends/__init__.py +++ b/backend/backends/__init__.py @@ -12,6 +12,7 @@ and a model config registry that eliminates per-engine dispatch maps. # HF_HUB_OFFLINE=1 and on network failures. from ..utils import hf_offline_patch # noqa: F401 +import os import threading from dataclasses import dataclass, field from typing import Protocol, Optional, Tuple, List @@ -678,6 +679,13 @@ def get_tts_backend_for_engine(engine: str) -> TTSBackend: """ global _tts_backends + # Test mode: every engine resolves to the fake backend so the full + # generation pipeline runs without model weights (see fake_backend.py). + if os.environ.get("VOICEBOX_FAKE_TTS") == "1": + from .fake_backend import get_fake_backend + + return get_fake_backend() + # Fast path: check without lock if engine in _tts_backends: return _tts_backends[engine] diff --git a/backend/backends/fake_backend.py b/backend/backends/fake_backend.py new file mode 100644 index 00000000..cb6b3b2c --- /dev/null +++ b/backend/backends/fake_backend.py @@ -0,0 +1,92 @@ +"""Fake TTS backend for UI and E2E testing. + +Activated by ``VOICEBOX_FAKE_TTS=1``. Every engine resolves to this backend, +which synthesizes a quiet sine tone sized to the input text — so the full +generation pipeline (task queue, SSE progress, database rows, audio serving) +runs exactly as in production, minus model weights and GPU time. +""" + +import asyncio +import logging +from typing import ClassVar, Optional + +import numpy as np + +logger = logging.getLogger(__name__) + +SAMPLE_RATE = 24_000 +SECONDS_PER_CHAR = 0.02 +MIN_DURATION_S = 0.25 +TONE_HZ = 440.0 +AMPLITUDE = 0.1 + + +class FakeTTSBackend: + """Implements the TTSBackend protocol without any model.""" + + MODEL_CONFIGS: ClassVar[list] = [] + + def __init__(self) -> None: + self._loaded = False + + async def load_model(self, model_size: str = "default") -> None: + if self._loaded: + return + # Brief pause so the UI's loading_model state is observable. + await asyncio.sleep(0.1) + self._loaded = True + logger.info("Fake TTS backend loaded (VOICEBOX_FAKE_TTS)") + + async def load_model_async(self, model_size: str = "default") -> None: + # Qwen engines are loaded through this variant (see load_engine_model). + await self.load_model(model_size) + + async def create_voice_prompt( + self, + audio_path: str, + reference_text: str, + use_cache: bool = True, + ) -> tuple[dict, bool]: + return ({"fake": True, "audio_path": audio_path, "reference_text": reference_text}, False) + + async def combine_voice_prompts( + self, + audio_paths: list[str], + reference_texts: list[str], + ) -> tuple[np.ndarray, str]: + combined_text = " ".join(reference_texts) + return np.zeros(SAMPLE_RATE, dtype=np.float32), combined_text + + async def generate( + self, + text: str, + voice_prompt: dict, + language: str = "en", + seed: Optional[int] = None, + instruct: Optional[str] = None, + ) -> tuple[np.ndarray, int]: + duration_s = max(MIN_DURATION_S, len(text) * SECONDS_PER_CHAR) + # Yield once so cancellation has a window, mirroring real inference. + await asyncio.sleep(0.05) + t = np.linspace(0.0, duration_s, int(SAMPLE_RATE * duration_s), endpoint=False) + audio = (AMPLITUDE * np.sin(2.0 * np.pi * TONE_HZ * t)).astype(np.float32) + return audio, SAMPLE_RATE + + def unload_model(self) -> None: + self._loaded = False + + def is_loaded(self) -> bool: + return self._loaded + + def _get_model_path(self, model_size: str) -> str: + return "fake" + + +_fake_backend: Optional[FakeTTSBackend] = None + + +def get_fake_backend() -> FakeTTSBackend: + global _fake_backend + if _fake_backend is None: + _fake_backend = FakeTTSBackend() + return _fake_backend diff --git a/backend/requirements-ci.txt b/backend/requirements-ci.txt new file mode 100644 index 00000000..8e328d34 --- /dev/null +++ b/backend/requirements-ci.txt @@ -0,0 +1,25 @@ +# Minimal dependency set to boot the backend on a CPU-only CI runner. +# No TTS/STT model libraries — inference is covered by the fake TTS +# backend (VOICEBOX_FAKE_TTS=1). Install CPU torch first on Linux: +# pip install torch --index-url https://download.pytorch.org/whl/cpu +# then: pip install -r backend/requirements-ci.txt + +fastapi>=0.109.0 +uvicorn[standard]>=0.27.0 +pydantic>=2.5.0 +sqlalchemy>=2.0.0 +alembic>=1.13.0 +torch>=2.2.0 +huggingface_hub>=0.20.0 +numpy +soundfile +python-multipart +sse-starlette +psutil +requests +httpx +fastmcp +librosa +pillow +pydub +pedalboard diff --git a/bun.lock b/bun.lock index 76e4b201..c72f6b8e 100644 --- a/bun.lock +++ b/bun.lock @@ -10,6 +10,7 @@ }, "devDependencies": { "@biomejs/biome": "2.3.12", + "@playwright/test": "^1.62.1", "@types/node": "^20.0.0", "@vitejs/plugin-react": "^6.0.5", "@vitest/browser": "^4.1.10", @@ -304,6 +305,8 @@ "@oxc-project/types": ["@oxc-project/types@0.143.0", "", {}, "sha512-u6JZdLBTLotrNC9Vd6vPssINdzcCzleKAH6EJKImQb7GtYvX5keN2dxkoK44stCc4tffE6QQRtZTXVSzsLUlWA=="], + "@playwright/test": ["@playwright/test@1.62.1", "", { "dependencies": { "playwright": "1.62.1" }, "bin": { "playwright": "cli.js" } }, "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ=="], + "@polka/url": ["@polka/url@1.0.0-next.29", "", {}, "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww=="], "@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts new file mode 100644 index 00000000..2210a79e --- /dev/null +++ b/e2e/fixtures.ts @@ -0,0 +1,99 @@ +import { type ChildProcess, spawn } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { test as base } from '@playwright/test'; + +const REPO_ROOT = path.resolve(__dirname, '..'); +const BASE_PORT = 18100; + +export interface BackendFixture { + url: string; + dataDir: string; +} + +async function waitForHealth(url: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const res = await fetch(`${url}/health`); + if (res.ok) return; + } catch { + // not up yet + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + throw new Error(`backend at ${url} did not become healthy within ${timeoutMs}ms`); +} + +export const test = base.extend, { backend: BackendFixture }>({ + backend: [ + async ({}, use, workerInfo) => { + const port = BASE_PORT + workerInfo.workerIndex; + const url = `http://127.0.0.1:${port}`; + // uvicorn resolves the data dir from cwd, so a temp cwd isolates + // each worker's SQLite and audio files completely. + const dataDir = mkdtempSync(path.join(tmpdir(), `voicebox-e2e-${workerInfo.workerIndex}-`)); + const python = process.env.VOICEBOX_PYTHON ?? path.join(REPO_ROOT, '.venv-ci/bin/python'); + + const proc: ChildProcess = spawn( + python, + ['-m', 'uvicorn', 'backend.main:app', '--port', String(port), '--log-level', 'warning'], + { + cwd: dataDir, + env: { + ...process.env, + PYTHONPATH: REPO_ROOT, + VOICEBOX_FAKE_TTS: '1', + VOICEBOX_CORS_ORIGINS: + 'http://localhost:4173,http://127.0.0.1:4173,http://localhost:5173', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + const logs: Buffer[] = []; + proc.stdout?.on('data', (chunk) => logs.push(chunk)); + proc.stderr?.on('data', (chunk) => logs.push(chunk)); + + try { + await waitForHealth(url); + } catch (error) { + proc.kill('SIGKILL'); + throw new Error(`${(error as Error).message}\nbackend log:\n${Buffer.concat(logs)}`); + } + + await use({ url, dataDir }); + + proc.kill('SIGTERM'); + await new Promise((resolve) => { + proc.once('exit', resolve); + setTimeout(resolve, 5000); + }); + rmSync(dataDir, { recursive: true, force: true }); + }, + { scope: 'worker' }, + ], + + // Point the app's persisted server store at this worker's backend before + // any page script runs. + page: async ({ page, backend }, use) => { + await page.addInitScript((serverUrl) => { + window.localStorage.setItem( + 'voicebox-server', + JSON.stringify({ + state: { + serverUrl, + isConnected: false, + mode: 'local', + keepServerRunningOnClose: false, + customModelsDir: null, + }, + version: 0, + }), + ); + }, backend.url); + await use(page); + }, +}); + +export { expect } from '@playwright/test'; diff --git a/e2e/helpers/api.ts b/e2e/helpers/api.ts new file mode 100644 index 00000000..a3e3d26d --- /dev/null +++ b/e2e/helpers/api.ts @@ -0,0 +1,58 @@ +/** Direct REST seeding against the per-worker backend — faster and less + * brittle than driving every prerequisite through the UI. */ + +export interface SeededProfile { + id: string; + name: string; +} + +/** A 3-second 220 Hz sine WAV (backend requires samples >= 2s). */ +export function buildSampleWav(): Blob { + const sampleRate = 24_000; + const seconds = 3; + const samples = sampleRate * seconds; + const dataSize = samples * 2; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + const writeString = (offset: number, s: string) => { + for (let i = 0; i < s.length; i++) view.setUint8(offset + i, s.charCodeAt(i)); + }; + writeString(0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeString(36, 'data'); + view.setUint32(40, dataSize, true); + for (let i = 0; i < samples; i++) { + view.setInt16(44 + i * 2, Math.round(0.1 * 32767 * Math.sin((2 * Math.PI * 220 * i) / sampleRate)), true); + } + return new Blob([buffer], { type: 'audio/wav' }); +} + +export async function seedProfile(backendUrl: string, name: string): Promise { + const createRes = await fetch(`${backendUrl}/profiles`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name, language: 'en' }), + }); + if (!createRes.ok) throw new Error(`create profile failed: ${createRes.status}`); + const profile = (await createRes.json()) as { id: string }; + + const form = new FormData(); + form.append('file', buildSampleWav(), 'sample.wav'); + form.append('reference_text', 'hello world this is a reference sample'); + const sampleRes = await fetch(`${backendUrl}/profiles/${profile.id}/samples`, { + method: 'POST', + body: form, + }); + if (!sampleRes.ok) throw new Error(`add sample failed: ${sampleRes.status} ${await sampleRes.text()}`); + + return { id: profile.id, name }; +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 00000000..737f0a90 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,36 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * E2E suite driving the web build (same app as Tauri, browser platform) + * against a real CPU backend with the fake TTS engine. Each worker gets + * its own uvicorn on its own port with its own data dir — see fixtures.ts. + * + * PW_DEV=1 targets `bun run dev:web` (port 5173) instead of the preview + * build for faster local iteration. + */ +const DEV = !!process.env.PW_DEV; +const PORT = DEV ? 5173 : 4173; + +export default defineConfig({ + testDir: './specs', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [['html', { open: 'never' }], ['github']] : 'list', + use: { + baseURL: `http://localhost:${PORT}`, + trace: 'on-first-retry', + video: 'retain-on-failure', + screenshot: 'only-on-failure', + }, + projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + webServer: { + command: DEV + ? 'bun run dev:web' + : 'bun run build:web && cd web && bunx vite preview --port 4173 --strictPort', + cwd: '..', + port: PORT, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, +}); diff --git a/e2e/specs/generate.spec.ts b/e2e/specs/generate.spec.ts new file mode 100644 index 00000000..e59dbd97 --- /dev/null +++ b/e2e/specs/generate.spec.ts @@ -0,0 +1,31 @@ +import { expect, test } from '../fixtures'; +import { seedProfile } from '../helpers/api'; + +test('generate speech end to end through the fake TTS pipeline', async ({ page, backend }) => { + const profile = await seedProfile(backend.url, 'Narrator'); + + await page.goto('/'); + + // Select the seeded voice, type into the generate box, and submit. + await page.getByText(profile.name).first().click(); + const input = page.getByRole('textbox').first(); + await input.click(); + await page.keyboard.type('The quick brown fox jumps over the lazy dog.'); + await page.getByRole('button', { name: 'Generate speech' }).click(); + + // The row lands in history and completes via the real queue + SSE. + await expect(page.getByText('The quick brown fox', { exact: false }).first()).toBeVisible({ + timeout: 15_000, + }); + + await expect + .poll( + async () => { + const res = await fetch(`${backend.url}/history?limit=10`); + const body = (await res.json()) as { items: { status: string }[] }; + return body.items[0]?.status; + }, + { timeout: 20_000 }, + ) + .toBe('completed'); +}); diff --git a/e2e/specs/startup.spec.ts b/e2e/specs/startup.spec.ts new file mode 100644 index 00000000..fd5c9722 --- /dev/null +++ b/e2e/specs/startup.spec.ts @@ -0,0 +1,16 @@ +import { expect, test } from '../fixtures'; + +test('app boots against the backend and shows the main editor', async ({ page }) => { + await page.goto('/'); + + await expect(page.getByRole('heading', { name: 'Voicebox' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Create Voice' }).first()).toBeVisible(); +}); + +test('settings layout renders its pages', async ({ page }) => { + await page.goto('/settings/about'); + + await expect(page.getByText('General', { exact: true })).toBeVisible(); + await expect(page.getByText('Changelog', { exact: true })).toBeVisible(); + await expect(page.getByText('About', { exact: true })).toBeVisible(); +}); diff --git a/e2e/specs/voices.spec.ts b/e2e/specs/voices.spec.ts new file mode 100644 index 00000000..92e7cf7c --- /dev/null +++ b/e2e/specs/voices.spec.ts @@ -0,0 +1,10 @@ +import { expect, test } from '../fixtures'; +import { seedProfile } from '../helpers/api'; + +test('a seeded voice profile appears in the voices tab', async ({ page, backend }) => { + const profile = await seedProfile(backend.url, 'Marcus Aurelius'); + + await page.goto('/voices'); + + await expect(page.getByText(profile.name).first()).toBeVisible(); +}); diff --git a/package.json b/package.json index 6ec9e052..5e1120b1 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ }, "devDependencies": { "@biomejs/biome": "2.3.12", + "@playwright/test": "^1.62.1", "@types/node": "^20.0.0", "@vitejs/plugin-react": "^6.0.5", "@vitest/browser": "^4.1.10",