fix: make transcript refinement language-aware

backend/tests/evaluate_multilingual_refinement.py

@@ -0,0 +1,197 @@
+"""Real-model evaluation for language-aware transcript refinement.
+
+This is deliberately an executable evaluation harness rather than a pytest test:
+Qwen output is non-deterministic and failures need human inspection.
+
+Usage:
+    python backend/tests/evaluate_multilingual_refinement.py
+    python backend/tests/evaluate_multilingual_refinement.py --model 0.6B --quick
+    python backend/tests/evaluate_multilingual_refinement.py --json results.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import re
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT))
+
+from backend.backends.qwen_llm_backend import MLXQwenLLMBackend  # noqa: E402
+from backend.services import refinement  # noqa: E402
+
+
+@dataclass(frozen=True)
+class EvalCase:
+    language: str
+    category: str
+    raw: str
+    must_contain: tuple[str, ...] = ()
+    must_not_contain: tuple[str, ...] = ()
+    question: bool = False
+
+
+CASES: tuple[EvalCase, ...] = (
+    EvalCase("en", "question", "uh what time is the deployment in Tokyo on Friday", ("Tokyo", "Friday"), question=True),
+    EvalCase("en", "self-correction", "remind me at seven no actually six pm to call mom", ("six",), ("seven",)),
+    EvalCase(
+        "en", "code-switch", "open package dot json then run the tests on GitHub", ("package.json", "tests", "GitHub")
+    ),
+    EvalCase(
+        "es", "question", "eh a qué hora es el despliegue en Tokio el viernes", ("Tokio", "viernes"), question=True
+    ),
+    EvalCase(
+        "es", "self-correction", "recuérdame a las siete no en realidad a las seis llamar a mamá", ("seis",), ("siete",)
+    ),
+    EvalCase(
+        "es", "code-switch", "abre package dot json y ejecuta los tests en GitHub", ("package.json", "tests", "GitHub")
+    ),
+    EvalCase(
+        "fr", "question", "euh à quelle heure est le déploiement à Tokyo vendredi", ("Tokyo", "vendredi"), question=True
+    ),
+    EvalCase(
+        "fr",
+        "self-correction",
+        "rappelle-moi à sept heures non en fait à six heures d'appeler maman",
+        ("six",),
+        ("sept",),
+    ),
+    EvalCase(
+        "fr",
+        "code-switch",
+        "ouvre package dot json puis lance les tests sur GitHub",
+        ("package.json", "tests", "GitHub"),
+    ),
+    EvalCase("de", "question", "äh wann ist das Deployment in Tokio am Freitag", ("Tokio", "Freitag"), question=True),
+    EvalCase(
+        "de",
+        "self-correction",
+        "erinnere mich um sieben nein eigentlich um sechs Mama anzurufen",
+        ("sechs",),
+        ("sieben",),
+    ),
+    EvalCase(
+        "de",
+        "code-switch",
+        "öffne package dot json und führe die tests auf GitHub aus",
+        ("package.json", "tests", "GitHub"),
+    ),
+    EvalCase(
+        "ja",
+        "question",
+        "えっと金曜日の東京でのdeploymentは何時ですか",
+        ("東京", "金曜日", "deployment"),
+        question=True,
+    ),
+    EvalCase("ja", "self-correction", "母に電話するのを7時いや6時にリマインドして", ("6時",), ("7時",)),
+    EvalCase(
+        "ja", "code-switch", "package dot jsonを開いてGitHubでtestsを実行して", ("package.json", "GitHub", "tests")
+    ),
+    EvalCase("zh", "question", "嗯周五在东京的deployment是几点", ("周五", "东京", "deployment"), question=True),
+    EvalCase("zh", "self-correction", "提醒我七点不对六点给妈妈打电话", ("六点",), ("七点",)),
+    EvalCase("zh", "code-switch", "打开package dot json然后在GitHub运行tests", ("package.json", "GitHub", "tests")),
+    EvalCase(
+        "hi", "question", "उम शुक्रवार को टोक्यो में deployment कितने बजे है", ("शुक्रवार", "टोक्यो", "deployment"), question=True
+    ),
+    EvalCase("hi", "self-correction", "मुझे सात बजे नहीं असल में छह बजे माँ को फ़ोन करने की याद दिलाना", ("छह",), ("सात",)),
+    EvalCase("hi", "code-switch", "package dot json खोलो और GitHub पर tests चलाओ", ("package.json", "GitHub", "tests")),
+)
+
+SCRIPT_PATTERNS = {
+    "ja": re.compile(r"[\u3040-\u30ff\u4e00-\u9fff]"),
+    "zh": re.compile(r"[\u4e00-\u9fff]"),
+    "hi": re.compile(r"[\u0900-\u097f]"),
+}
+
+
+@dataclass
+class EvalResult:
+    model: str
+    language: str
+    category: str
+    raw: str
+    output: str
+    passed: bool
+    failures: list[str]
+
+
+def score(case: EvalCase, output: str, model: str) -> EvalResult:
+    folded = output.casefold()
+    failures = [f"missing {token!r}" for token in case.must_contain if token.casefold() not in folded]
+    failures.extend(
+        f"retained retracted token {token!r}" for token in case.must_not_contain if token.casefold() in folded
+    )
+    japanese_question = case.language == "ja" and output.rstrip().endswith("か。")
+    if case.question and not japanese_question and not output.rstrip().endswith(("?", "？")):
+        failures.append("question did not remain a question")
+    script = SCRIPT_PATTERNS.get(case.language)
+    if script is not None and script.search(output) is None:
+        failures.append("source script was not preserved")
+    if not output.strip():
+        failures.append("empty output")
+    return EvalResult(
+        model=model,
+        language=case.language,
+        category=case.category,
+        raw=case.raw,
+        output=output,
+        passed=not failures,
+        failures=failures,
+    )
+
+
+async def run(models: list[str], quick: bool, category: str | None) -> list[EvalResult]:
+    backend = MLXQwenLLMBackend(models[0])
+    original_getter = refinement.llm_service.get_llm_model
+    refinement.llm_service.get_llm_model = lambda: backend
+    cases = [
+        case
+        for case in CASES
+        if (not quick or case.category == "code-switch") and (category is None or case.category == category)
+    ]
+    results: list[EvalResult] = []
+    try:
+        for model in models:
+            for case in cases:
+                output, _ = await refinement.refine_transcript(
+                    case.raw,
+                    refinement.RefinementFlags(),
+                    model_size=model,
+                    language=case.language,
+                )
+                result = score(case, output, model)
+                results.append(result)
+                mark = "PASS" if result.passed else "FAIL"
+                print(f"[{mark}] {model:4} {case.language}/{case.category}: {output}")
+                for failure in result.failures:
+                    print(f"       - {failure}")
+    finally:
+        refinement.llm_service.get_llm_model = original_getter
+        backend.unload_model()
+    return results
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", action="append", choices=("0.6B", "4B"))
+    parser.add_argument("--quick", action="store_true", help="Run code-switch cases only")
+    parser.add_argument("--category", choices=("question", "self-correction", "code-switch"))
+    parser.add_argument("--json", type=Path)
+    args = parser.parse_args()
+    models = args.model or ["0.6B", "4B"]
+    results = asyncio.run(run(models, args.quick, args.category))
+    if args.json:
+        args.json.parent.mkdir(parents=True, exist_ok=True)
+        args.json.write_text(json.dumps([asdict(result) for result in results], ensure_ascii=False, indent=2) + "\n")
+    failures = sum(not result.passed for result in results)
+    print(f"\n{len(results) - failures}/{len(results)} checks passed")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/tests/test_capture_language_pipeline.py

@@ -0,0 +1,117 @@
+from io import BytesIO
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi import UploadFile
+
+from backend.backends import TranscriptionResult
+from backend.mcp_server import tools
+from backend.routes import transcription as transcription_route
+from backend.services import captures, transcribe
+from backend.services.refinement import RefinementFlags
+from backend.utils import audio as audio_utils
+
+
+@pytest.mark.asyncio
+async def test_retranscribe_persists_auto_detected_language(monkeypatch, tmp_path):
+    audio_path = tmp_path / "capture.wav"
+    audio_path.write_bytes(b"audio")
+    row = SimpleNamespace(
+        id="capture-1",
+        audio_path="captures/capture.wav",
+        transcript_raw="old",
+        transcript_refined="old refined",
+        stt_model="base",
+        language=None,
+        llm_model="0.6B",
+        refinement_flags="{}",
+    )
+    db = MagicMock()
+    db.query.return_value.filter.return_value.first.return_value = row
+    whisper = SimpleNamespace(
+        model_size="turbo",
+        transcribe_with_metadata=AsyncMock(return_value=TranscriptionResult(text="bonjour le monde", language="fr")),
+    )
+    monkeypatch.setattr(captures.config, "resolve_storage_path", lambda _path: audio_path)
+    monkeypatch.setattr(captures, "get_whisper_model", lambda: whisper)
+    monkeypatch.setattr(captures, "_to_response", lambda value: value)
+
+    result = await captures.retranscribe_capture(
+        capture_id="capture-1",
+        stt_model=None,
+        language=None,
+        db=db,
+    )
+
+    assert result.transcript_raw == "bonjour le monde"
+    assert result.language == "fr"
+    assert result.transcript_refined is None
+
+
+@pytest.mark.asyncio
+async def test_mcp_transcribe_returns_detected_language(monkeypatch, tmp_path):
+    audio_path = tmp_path / "sample.wav"
+    audio_path.write_bytes(b"audio")
+    whisper = SimpleNamespace(
+        model_size="turbo",
+        is_loaded=lambda: True,
+        transcribe_with_metadata=AsyncMock(return_value=TranscriptionResult(text="hola mundo", language="es")),
+    )
+    monkeypatch.setattr(transcribe, "get_whisper_model", lambda: whisper)
+    monkeypatch.setattr(audio_utils, "load_audio", lambda _path: ([0.0] * 16000, 16000))
+
+    result = await tools._transcribe_file(audio_path, language=" ES ", model=None)
+
+    assert result["text"] == "hola mundo"
+    assert result["language"] == "es"
+    assert whisper.transcribe_with_metadata.await_args.args[1] == "es"
+
+
+@pytest.mark.asyncio
+async def test_http_transcribe_returns_detected_language(monkeypatch):
+    whisper = SimpleNamespace(
+        model_size="turbo",
+        is_loaded=lambda: True,
+        transcribe_with_metadata=AsyncMock(return_value=TranscriptionResult(text="hallo welt", language="de")),
+    )
+    monkeypatch.setattr(transcribe, "get_whisper_model", lambda: whisper)
+    monkeypatch.setattr(audio_utils, "load_audio", lambda _path: ([0.0] * 16000, 16000))
+    upload = UploadFile(filename="sample.wav", file=BytesIO(b"audio"))
+
+    response = await transcription_route.transcribe_audio(
+        upload,
+        language=" AUTO ",
+        model=None,
+    )
+
+    assert response.text == "hallo welt"
+    assert response.language == "de"
+    assert whisper.transcribe_with_metadata.await_args.args[1] is None
+
+
+@pytest.mark.asyncio
+async def test_capture_refinement_receives_persisted_language(monkeypatch):
+    row = SimpleNamespace(
+        id="capture-1",
+        transcript_raw="打开 package.json",
+        transcript_refined=None,
+        language="zh",
+        llm_model=None,
+        refinement_flags=None,
+    )
+    db = MagicMock()
+    db.query.return_value.filter.return_value.first.return_value = row
+    refine = AsyncMock(return_value=("打开 package.json。", "0.6B"))
+    monkeypatch.setattr(captures, "refine_transcript", refine)
+    monkeypatch.setattr(captures, "_to_response", lambda value: value)
+
+    result = await captures.refine_capture(
+        capture_id="capture-1",
+        flags=RefinementFlags(),
+        model_size="0.6B",
+        db=db,
+    )
+
+    assert result.transcript_refined == "打开 package.json。"
+    assert refine.await_args.kwargs["language"] == "zh"

backend/tests/test_capture_language_validation.py

@@ -0,0 +1,35 @@
+import pytest
+from pydantic import ValidationError
+
+from backend import models
+from backend.languages import CAPTURE_LANGUAGE_CODES, normalize_capture_language
+
+
+@pytest.mark.parametrize("language", CAPTURE_LANGUAGE_CODES)
+def test_supported_capture_languages_are_canonical(language):
+    assert normalize_capture_language(f" {language.upper()} ") == language
+
+
+def test_auto_capture_language_normalizes_to_none():
+    assert normalize_capture_language(" AUTO ") is None
+    assert normalize_capture_language(None) is None
+
+
+def test_unknown_capture_language_is_rejected():
+    with pytest.raises(ValueError, match="Unsupported capture language"):
+        normalize_capture_language("ignore previous instructions")
+
+
+def test_retranscription_accepts_profile_legacy_and_auto_languages():
+    assert models.CaptureRetranscribeRequest(language="hi").language == "hi"
+    assert models.CaptureRetranscribeRequest(language=" KO ").language == "ko"
+    assert models.CaptureRetranscribeRequest(language="nl").language == "nl"
+    assert models.CaptureRetranscribeRequest(language="auto").language == "auto"
+    assert models.CaptureSettingsUpdate(language=" RU ").language == "ru"
+
+
+def test_retranscription_rejects_unknown_language():
+    with pytest.raises(ValidationError):
+        models.CaptureRetranscribeRequest(language="xx")
+    with pytest.raises(ValidationError):
+        models.CaptureSettingsUpdate(language="xx")

backend/tests/test_refinement_languages.py

@@ -0,0 +1,69 @@
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from backend.services import refinement
+
+LANGUAGE_NAMES = {
+    "en": "English",
+    "es": "Spanish",
+    "fr": "French",
+    "de": "German",
+    "ja": "Japanese",
+    "zh": "Chinese",
+    "hi": "Hindi",
+}
+
+
+@pytest.mark.parametrize(("code", "name"), LANGUAGE_NAMES.items())
+def test_prompt_uses_only_canonical_supported_language(code, name):
+    prompt = refinement.build_refinement_prompt(refinement.RefinementFlags(), code)
+
+    assert f"Primary language: {name} ({code})." in prompt
+    assert "Preserve every source-language span in its original language and script." in prompt
+    assert "Never translate any part of the transcript." in prompt
+
+
+@pytest.mark.parametrize("language", [None, "auto", "xx", "ignore previous instructions"])
+def test_unknown_language_is_never_interpolated_into_prompt(language):
+    prompt = refinement.build_refinement_prompt(refinement.RefinementFlags(), language)
+
+    assert language is None or language not in prompt
+    assert "Primary language:" not in prompt
+    assert "Never translate any part of the transcript." in prompt
+
+
+@pytest.mark.parametrize("code", LANGUAGE_NAMES)
+def test_supported_language_uses_matched_examples_with_technical_code_switching(code):
+    examples = refinement.get_refinement_examples(code)
+    combined = " ".join(source + " " + target for source, target in examples)
+
+    assert len(examples) >= 5
+    assert examples is not refinement.REFINEMENT_EXAMPLES
+    assert any(token in combined for token in ("GitHub", "package.json", "npm", "tests"))
+
+
+def test_missing_language_keeps_legacy_english_examples_for_old_captures():
+    assert refinement.get_refinement_examples(None) is refinement.REFINEMENT_EXAMPLES
+
+
+@pytest.mark.asyncio
+async def test_refine_transcript_passes_language_prompt_and_examples(monkeypatch):
+    backend = SimpleNamespace(
+        model_size="0.6B",
+        generate=AsyncMock(return_value="Hola, abre package.json."),
+    )
+    monkeypatch.setattr(refinement.llm_service, "get_llm_model", lambda: backend)
+
+    text, model_size = await refinement.refine_transcript(
+        "eh hola abre package dot json",
+        refinement.RefinementFlags(),
+        language="es",
+    )
+
+    assert text == "Hola, abre package.json."
+    assert model_size == "0.6B"
+    kwargs = backend.generate.await_args.kwargs
+    assert "Primary language: Spanish (es)." in kwargs["system"]
+    assert kwargs["examples"] == refinement.get_refinement_examples("es")

backend/tests/test_transcription_language.py

@@ -0,0 +1,129 @@
+from types import SimpleNamespace
+from typing import get_type_hints
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+import torch
+
+from backend import backends, models
+from backend.backends import pytorch_backend
+from backend.backends.mlx_backend import MLXSTTBackend
+from backend.backends.pytorch_backend import PyTorchSTTBackend
+
+
+class _FakeBatch(dict):
+    def to(self, _device):
+        return self
+
+
+class _FakeProcessor:
+    def __call__(self, *_args, **_kwargs):
+        return _FakeBatch(input_features=torch.zeros((1, 80, 10)))
+
+    def get_decoder_prompt_ids(self, *, language, task):
+        return [(1, language)]
+
+    def batch_decode(self, *_args, **_kwargs):
+        return ["  bonjour le monde  "]
+
+
+def test_transcription_result_contract_exists():
+    assert hasattr(backends, "TranscriptionResult")
+    assert get_type_hints(backends.STTBackend.transcribe)["return"] is str
+    assert get_type_hints(backends.STTBackend.transcribe_with_metadata)["return"] is backends.TranscriptionResult
+
+
+@pytest.mark.asyncio
+async def test_metadata_adapter_preserves_legacy_text_only_backends():
+    class LegacyBackend:
+        async def transcribe(self, audio_path, language=None, model_size=None):
+            assert audio_path == "sample.wav"
+            assert model_size == "small"
+            return "  hola mundo  "
+
+    result = await backends.transcribe_with_metadata(LegacyBackend(), "sample.wav", language="es", model_size="small")
+
+    assert result == backends.TranscriptionResult(text="hola mundo", language="es")
+
+
+def test_transcription_response_exposes_detected_language():
+    response = models.TranscriptionResponse(
+        text="bonjour",
+        duration=1.0,
+        language="fr",
+    )
+
+    assert response.language == "fr"
+
+
+def test_pytorch_whisper_language_token_maps_to_code():
+    generation_config = SimpleNamespace(
+        lang_to_id={"<|en|>": 100, "<|zh|>": 200},
+    )
+
+    assert pytorch_backend.whisper_language_code_from_token_id(generation_config, 200) == "zh"
+
+
+@pytest.mark.asyncio
+async def test_pytorch_transcribe_returns_auto_detected_language(monkeypatch):
+    processor = _FakeProcessor()
+    detect_language = MagicMock(return_value=torch.tensor([200]))
+    generate = MagicMock(return_value=torch.tensor([[1, 2, 3]]))
+    model = SimpleNamespace(
+        generation_config=SimpleNamespace(lang_to_id={"<|en|>": 100, "<|fr|>": 200}),
+        detect_language=detect_language,
+        generate=generate,
+    )
+    backend = object.__new__(PyTorchSTTBackend)
+    backend.model = model
+    backend.processor = processor
+    backend.model_size = "base"
+    backend.device = "cpu"
+    backend.load_model_async = AsyncMock()
+    monkeypatch.setattr(pytorch_backend, "load_audio", lambda *_args, **_kwargs: ([0.0], 16000))
+
+    result = await backend.transcribe_with_metadata("sample.wav")
+
+    assert result == backends.TranscriptionResult(text="bonjour le monde", language="fr")
+    assert "forced_decoder_ids" not in generate.call_args.kwargs
+    assert await backend.transcribe("sample.wav") == "bonjour le monde"
+
+
+@pytest.mark.asyncio
+async def test_pytorch_transcribe_forces_only_explicit_language(monkeypatch):
+    processor = _FakeProcessor()
+    detect_language = MagicMock()
+    generate = MagicMock(return_value=torch.tensor([[1, 2, 3]]))
+    backend = object.__new__(PyTorchSTTBackend)
+    backend.model = SimpleNamespace(
+        generation_config=SimpleNamespace(lang_to_id={"<|en|>": 100}),
+        detect_language=detect_language,
+        generate=generate,
+    )
+    backend.processor = processor
+    backend.model_size = "base"
+    backend.device = "cpu"
+    backend.load_model_async = AsyncMock()
+    monkeypatch.setattr(
+        pytorch_backend, "load_audio", lambda *_args, **_kwargs: ([0.0], 16000)
+    )
+
+    result = await backend.transcribe_with_metadata("sample.wav", language="en")
+
+    assert result.language == "en"
+    detect_language.assert_not_called()
+    assert generate.call_args.kwargs["forced_decoder_ids"] == [(1, "en")]
+
+
+@pytest.mark.asyncio
+async def test_mlx_transcribe_returns_detected_language():
+    backend = MLXSTTBackend()
+    backend.model = SimpleNamespace(
+        generate=lambda *_args, **_kwargs: SimpleNamespace(text="  你好世界  ", language="zh")
+    )
+    backend.load_model_async = AsyncMock()
+
+    result = await backend.transcribe_with_metadata("sample.wav")
+
+    assert result == backends.TranscriptionResult(text="你好世界", language="zh")
+    assert await backend.transcribe("sample.wav") == "你好世界"