fix(refinement): character-level loop collapse + pytest coverage

The word-level pass catches single-word Whisper loops ("URL URL URL…")
but misses two common hallucination patterns the PR had to claim as
"edge cases":

1. Multi-word English loops — "thanks for watching thanks for watching…"
   × 6 sails through because no two consecutive tokens are identical
   after text.split().
2. CJK loops — "謝謝觀看" × 7 sails through because text.split() returns
   a single unsplit token for the whole loop (no whitespace between
   characters).

Add a character-level second pass: a non-greedy regex finds any 2–60
char substring that repeats min_run+ times immediately after itself and
strips the run. The 2-char floor keeps emphasised single-letter runs
("wooooooow") intact. The 60-char ceiling covers every observed
Whisper tail hallucination ("Please like and subscribe to my
channel.", "Subtitles by the Amara.org community") while staying short
enough that coincidental long-phrase repetition in legitimate speech
doesn't hit the threshold. Whitespace normalisation only runs when the
pass actually stripped something, so untouched transcripts keep their
original spacing.

New test_refinement_collapse.py gives the pre-processor its first
deterministic unit-test coverage: 17 tests pinning the word-level
legacy behaviour plus the new multi-word English / CJK / Japanese /
emphasis-preservation cases.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>

backend/services/refinement.py

@@ -14,14 +14,24 @@ from dataclasses import dataclass
 from . import llm as llm_service
 
 
-# A run of identical tokens this long gets collapsed before the LLM sees
-# the transcript. Whisper occasionally loops a single word hundreds of
-# times when audio trails off (the "URL URL URL…" tail); smaller refine
-# models truncate legitimate output to "make room" for the loop, and
-# bigger ones echo the run verbatim because "never omit ideas" overrides
-# the no-garbage heuristic. Stripping deterministically sidesteps both.
+# A run that repeats this many times gets collapsed before the LLM sees
+# the transcript. Whisper occasionally loops content hundreds of times
+# when audio trails off — "URL URL URL…" (single word), "thanks for
+# watching thanks for watching…" (multi-word phrase), or
+# "谢谢观看谢谢观看…" (CJK with no spaces). Smaller refine models truncate
+# legitimate output to "make room" for the loop, and bigger ones echo
+# the run verbatim because "never omit ideas" overrides the no-garbage
+# heuristic. Stripping deterministically sidesteps both.
 _REPETITION_RUN_THRESHOLD = 6
 
+# Upper bound on the length of a repeating unit that the character-level
+# pass will detect. Covers every Whisper hallucination phrase we've
+# observed ("Please like and subscribe to my channel." ≈ 41 chars,
+# "Subtitles by the Amara.org community" ≈ 36 chars) while being short
+# enough that coincidental long-phrase repetition stays below the
+# threshold in legitimate speech.
+_MAX_REPETITION_UNIT_CHARS = 60
+
 
 def _token_key(word: str) -> str:
     """Normalize a token for repetition comparison — strip surrounding
@@ -31,10 +41,29 @@ def _token_key(word: str) -> str:
 
 
 def collapse_repetitive_artifacts(text: str, min_run: int = _REPETITION_RUN_THRESHOLD) -> str:
-    """Strip STT-artifact runs: any token repeated ``min_run``+ times in
-    a row is treated as a Whisper hallucination and dropped entirely.
-    Legitimate rhetorical repetition ("no, no, no, no, no") doesn't hit
-    the threshold, and anything shorter passes through unchanged."""
+    """Strip STT-artifact loops. Two passes handle the full space:
+
+    1. Word-level: any token repeated ``min_run``+ times consecutively
+       (with surrounding punctuation stripped for comparison). Catches
+       single-word loops like "URL URL URL…" and normalizes punctuated
+       variants like "URL, URL, URL, URL, URL, URL".
+    2. Character-level: any substring 2–60 chars long that repeats
+       ``min_run``+ times immediately after itself. Catches multi-word
+       English loops ("thanks for watching" × 6) that the word-level
+       pass misses (no consecutive identical tokens) and CJK loops
+       ("谢谢观看" × 6) where ``text.split()`` yields a single unsplit
+       token.
+
+    Both passes preserve rhetorical repetition: "no, no, no, no, no"
+    (5 repeats) and "yeah yeah yeah" (3 repeats) stay in the transcript
+    because they don't cross the threshold.
+    """
+    collapsed = _collapse_word_runs(text, min_run)
+    collapsed = _collapse_character_runs(collapsed, min_run)
+    return collapsed
+
+
+def _collapse_word_runs(text: str, min_run: int) -> str:
     words = text.split()
     if len(words) < min_run:
         return text
@@ -63,6 +92,25 @@ def collapse_repetitive_artifacts(text: str, min_run: int = _REPETITION_RUN_THRE
     return " ".join(out)
 
 
+def _collapse_character_runs(text: str, min_run: int) -> str:
+    # Non-greedy unit so the shortest repeating substring wins. Lower
+    # bound of 2 chars avoids stripping emphasized single-letter runs
+    # ("wooooooow", "hmmmmm") that aren't hallucinations. re.DOTALL so a
+    # newline inside a looped unit (rare) doesn't break the match.
+    pattern = re.compile(
+        r"(.{2," + str(_MAX_REPETITION_UNIT_CHARS) + r"}?)\1{" + str(min_run - 1) + r",}",
+        flags=re.DOTALL,
+    )
+    result = pattern.sub("", text)
+    if result == text:
+        return text
+    # Stripping a run leaves double whitespace where the loop used to
+    # bridge surrounding context; normalize so the LLM prompt stays
+    # clean. Only runs when we actually modified the text so transcripts
+    # that didn't hit any loop keep their original whitespace.
+    return re.sub(r"\s+", " ", result).strip()
+
+
 @dataclass
 class RefinementFlags:
     """Which refinement behaviours to apply."""

backend/tests/test_refinement_collapse.py

@@ -0,0 +1,145 @@
+"""Unit tests for ``collapse_repetitive_artifacts``.
+
+The eval harness (``test_refinement_samples.py``) is interactive and
+LLM-dependent; these are the fast, deterministic tests for the
+deterministic pre-processor that runs before the LLM ever sees a
+transcript. They pin the behaviour for both the single-word loops the
+original algorithm handled and the multi-word / CJK / emoji loops the
+character-level pass added.
+"""
+
+from backend.services.refinement import collapse_repetitive_artifacts
+
+
+# ── single-word loops (word-level pass) ─────────────────────────────────
+
+
+def test_single_word_loop_stripped():
+    raw = "Hello " + ("URL " * 8).strip() + " goodbye"
+    assert collapse_repetitive_artifacts(raw) == "Hello goodbye"
+
+
+def test_single_word_loop_with_punctuation_normalized():
+    # URL, URL, URL, URL, URL, URL. — six repeats if you normalize
+    # trailing punctuation; word-level pass strips them all.
+    raw = "Hello URL, URL, URL, URL, URL, URL. goodbye"
+    assert collapse_repetitive_artifacts(raw) == "Hello goodbye"
+
+
+def test_single_word_loop_case_insensitive():
+    raw = "hi " + " ".join(["Url", "URL", "url", "Url", "URL", "url"]) + " bye"
+    assert collapse_repetitive_artifacts(raw) == "hi bye"
+
+
+def test_short_single_word_run_preserved():
+    # Five repeats — below threshold.
+    raw = "no no no no no"
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+def test_rhetorical_repetition_preserved():
+    raw = "I said no, no, no, no, no and she left"
+    # Five repeats of "no" — below threshold.
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+# ── multi-word loops (character-level pass) ─────────────────────────────
+
+
+def test_multi_word_english_loop_stripped():
+    # Classic Whisper tail hallucination. Word-level pass sees no
+    # consecutive identical tokens, so it's the character-level pass's
+    # job to catch this.
+    loop = "thanks for watching " * 6
+    raw = f"Okay so the meeting is at three. {loop}"
+    result = collapse_repetitive_artifacts(raw)
+    assert "thanks for watching" not in result
+    assert "Okay so the meeting is at three" in result
+
+
+def test_three_word_loop_stripped():
+    loop = "please like and " * 7
+    raw = f"The point is clear. {loop}right"
+    result = collapse_repetitive_artifacts(raw)
+    assert "please like and" not in result
+    assert "The point is clear" in result
+
+
+def test_long_phrase_loop_within_60_char_cap():
+    unit = "Please like and subscribe to my channel. "  # 41 chars, within cap
+    raw = "End of video. " + unit * 6
+    result = collapse_repetitive_artifacts(raw)
+    assert unit.strip() not in result
+    assert "End of video" in result
+
+
+def test_multi_word_short_run_preserved():
+    # Five repeats of a multi-word unit — below threshold.
+    raw = "thanks for watching thanks for watching thanks for watching thanks for watching thanks for watching"
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+# ── CJK loops (character-level pass, no whitespace) ──────────────────────
+
+
+def test_cjk_loop_stripped():
+    # Common Chinese Whisper hallucination: "thanks for watching".
+    # text.split() yields one token for the whole loop; only the
+    # character-level pass can catch this.
+    prefix = "會議在三點開始"
+    loop = "謝謝觀看" * 7
+    raw = prefix + loop
+    result = collapse_repetitive_artifacts(raw)
+    assert "謝謝觀看" not in result
+    assert prefix in result
+
+
+def test_japanese_loop_stripped():
+    # Same pattern, kana/kanji mix. "ご視聴ありがとうございました" is a
+    # frequent Japanese Whisper tail hallucination.
+    loop = "ご視聴ありがとうございました" * 6
+    raw = f"明日の会議は午後三時です。{loop}"
+    result = collapse_repetitive_artifacts(raw)
+    assert "ご視聴ありがとうございました" not in result
+    assert "明日の会議は午後三時です" in result
+
+
+def test_cjk_short_run_preserved():
+    # Five repeats — below threshold, stays in.
+    raw = "好好好好好"
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+# ── whitespace / edge cases ──────────────────────────────────────────────
+
+
+def test_empty_string_passes_through():
+    assert collapse_repetitive_artifacts("") == ""
+
+
+def test_below_word_threshold_passes_through_unmodified():
+    raw = "just three words"
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+def test_emphasis_vowel_run_preserved():
+    # "wooooooow" is 1 char (plus 8 o's). Character-level min unit is 2,
+    # so "oo…" doesn't get stripped and this legitimate emphasis stays.
+    raw = "that's wooooooow amazing"
+    assert collapse_repetitive_artifacts(raw) == raw
+
+
+def test_custom_threshold_honored():
+    # With min_run=3, even short rhetorical repetition should now strip.
+    raw = "ha ha ha ha context"
+    result = collapse_repetitive_artifacts(raw, min_run=3)
+    assert "ha ha" not in result
+    assert "context" in result
+
+
+def test_leading_and_trailing_whitespace_stripped_after_collapse():
+    # When character pass fires, the normalised result is stripped so
+    # downstream prompts don't carry edge whitespace from the removal.
+    loop = "loop-phrase " * 7
+    raw = loop
+    assert collapse_repetitive_artifacts(raw) == ""