diff --git a/backend/services/refinement.py b/backend/services/refinement.py index aa8ef037..b23d2192 100644 --- a/backend/services/refinement.py +++ b/backend/services/refinement.py @@ -14,14 +14,24 @@ from dataclasses import dataclass from . import llm as llm_service -# A run of identical tokens this long gets collapsed before the LLM sees -# the transcript. Whisper occasionally loops a single word hundreds of -# times when audio trails off (the "URL URL URL…" tail); smaller refine -# models truncate legitimate output to "make room" for the loop, and -# bigger ones echo the run verbatim because "never omit ideas" overrides -# the no-garbage heuristic. Stripping deterministically sidesteps both. +# A run that repeats this many times gets collapsed before the LLM sees +# the transcript. Whisper occasionally loops content hundreds of times +# when audio trails off — "URL URL URL…" (single word), "thanks for +# watching thanks for watching…" (multi-word phrase), or +# "谢谢观看谢谢观看…" (CJK with no spaces). Smaller refine models truncate +# legitimate output to "make room" for the loop, and bigger ones echo +# the run verbatim because "never omit ideas" overrides the no-garbage +# heuristic. Stripping deterministically sidesteps both. _REPETITION_RUN_THRESHOLD = 6 +# Upper bound on the length of a repeating unit that the character-level +# pass will detect. Covers every Whisper hallucination phrase we've +# observed ("Please like and subscribe to my channel." ≈ 41 chars, +# "Subtitles by the Amara.org community" ≈ 36 chars) while being short +# enough that coincidental long-phrase repetition stays below the +# threshold in legitimate speech. +_MAX_REPETITION_UNIT_CHARS = 60 + def _token_key(word: str) -> str: """Normalize a token for repetition comparison — strip surrounding @@ -31,10 +41,29 @@ def _token_key(word: str) -> str: def collapse_repetitive_artifacts(text: str, min_run: int = _REPETITION_RUN_THRESHOLD) -> str: - """Strip STT-artifact runs: any token repeated ``min_run``+ times in - a row is treated as a Whisper hallucination and dropped entirely. - Legitimate rhetorical repetition ("no, no, no, no, no") doesn't hit - the threshold, and anything shorter passes through unchanged.""" + """Strip STT-artifact loops. Two passes handle the full space: + + 1. Word-level: any token repeated ``min_run``+ times consecutively + (with surrounding punctuation stripped for comparison). Catches + single-word loops like "URL URL URL…" and normalizes punctuated + variants like "URL, URL, URL, URL, URL, URL". + 2. Character-level: any substring 2–60 chars long that repeats + ``min_run``+ times immediately after itself. Catches multi-word + English loops ("thanks for watching" × 6) that the word-level + pass misses (no consecutive identical tokens) and CJK loops + ("谢谢观看" × 6) where ``text.split()`` yields a single unsplit + token. + + Both passes preserve rhetorical repetition: "no, no, no, no, no" + (5 repeats) and "yeah yeah yeah" (3 repeats) stay in the transcript + because they don't cross the threshold. + """ + collapsed = _collapse_word_runs(text, min_run) + collapsed = _collapse_character_runs(collapsed, min_run) + return collapsed + + +def _collapse_word_runs(text: str, min_run: int) -> str: words = text.split() if len(words) < min_run: return text @@ -63,6 +92,25 @@ def collapse_repetitive_artifacts(text: str, min_run: int = _REPETITION_RUN_THRE return " ".join(out) +def _collapse_character_runs(text: str, min_run: int) -> str: + # Non-greedy unit so the shortest repeating substring wins. Lower + # bound of 2 chars avoids stripping emphasized single-letter runs + # ("wooooooow", "hmmmmm") that aren't hallucinations. re.DOTALL so a + # newline inside a looped unit (rare) doesn't break the match. + pattern = re.compile( + r"(.{2," + str(_MAX_REPETITION_UNIT_CHARS) + r"}?)\1{" + str(min_run - 1) + r",}", + flags=re.DOTALL, + ) + result = pattern.sub("", text) + if result == text: + return text + # Stripping a run leaves double whitespace where the loop used to + # bridge surrounding context; normalize so the LLM prompt stays + # clean. Only runs when we actually modified the text so transcripts + # that didn't hit any loop keep their original whitespace. + return re.sub(r"\s+", " ", result).strip() + + @dataclass class RefinementFlags: """Which refinement behaviours to apply.""" diff --git a/backend/tests/test_refinement_collapse.py b/backend/tests/test_refinement_collapse.py new file mode 100644 index 00000000..c74ca622 --- /dev/null +++ b/backend/tests/test_refinement_collapse.py @@ -0,0 +1,145 @@ +"""Unit tests for ``collapse_repetitive_artifacts``. + +The eval harness (``test_refinement_samples.py``) is interactive and +LLM-dependent; these are the fast, deterministic tests for the +deterministic pre-processor that runs before the LLM ever sees a +transcript. They pin the behaviour for both the single-word loops the +original algorithm handled and the multi-word / CJK / emoji loops the +character-level pass added. +""" + +from backend.services.refinement import collapse_repetitive_artifacts + + +# ── single-word loops (word-level pass) ───────────────────────────────── + + +def test_single_word_loop_stripped(): + raw = "Hello " + ("URL " * 8).strip() + " goodbye" + assert collapse_repetitive_artifacts(raw) == "Hello goodbye" + + +def test_single_word_loop_with_punctuation_normalized(): + # URL, URL, URL, URL, URL, URL. — six repeats if you normalize + # trailing punctuation; word-level pass strips them all. + raw = "Hello URL, URL, URL, URL, URL, URL. goodbye" + assert collapse_repetitive_artifacts(raw) == "Hello goodbye" + + +def test_single_word_loop_case_insensitive(): + raw = "hi " + " ".join(["Url", "URL", "url", "Url", "URL", "url"]) + " bye" + assert collapse_repetitive_artifacts(raw) == "hi bye" + + +def test_short_single_word_run_preserved(): + # Five repeats — below threshold. + raw = "no no no no no" + assert collapse_repetitive_artifacts(raw) == raw + + +def test_rhetorical_repetition_preserved(): + raw = "I said no, no, no, no, no and she left" + # Five repeats of "no" — below threshold. + assert collapse_repetitive_artifacts(raw) == raw + + +# ── multi-word loops (character-level pass) ───────────────────────────── + + +def test_multi_word_english_loop_stripped(): + # Classic Whisper tail hallucination. Word-level pass sees no + # consecutive identical tokens, so it's the character-level pass's + # job to catch this. + loop = "thanks for watching " * 6 + raw = f"Okay so the meeting is at three. {loop}" + result = collapse_repetitive_artifacts(raw) + assert "thanks for watching" not in result + assert "Okay so the meeting is at three" in result + + +def test_three_word_loop_stripped(): + loop = "please like and " * 7 + raw = f"The point is clear. {loop}right" + result = collapse_repetitive_artifacts(raw) + assert "please like and" not in result + assert "The point is clear" in result + + +def test_long_phrase_loop_within_60_char_cap(): + unit = "Please like and subscribe to my channel. " # 41 chars, within cap + raw = "End of video. " + unit * 6 + result = collapse_repetitive_artifacts(raw) + assert unit.strip() not in result + assert "End of video" in result + + +def test_multi_word_short_run_preserved(): + # Five repeats of a multi-word unit — below threshold. + raw = "thanks for watching thanks for watching thanks for watching thanks for watching thanks for watching" + assert collapse_repetitive_artifacts(raw) == raw + + +# ── CJK loops (character-level pass, no whitespace) ────────────────────── + + +def test_cjk_loop_stripped(): + # Common Chinese Whisper hallucination: "thanks for watching". + # text.split() yields one token for the whole loop; only the + # character-level pass can catch this. + prefix = "會議在三點開始" + loop = "謝謝觀看" * 7 + raw = prefix + loop + result = collapse_repetitive_artifacts(raw) + assert "謝謝觀看" not in result + assert prefix in result + + +def test_japanese_loop_stripped(): + # Same pattern, kana/kanji mix. "ご視聴ありがとうございました" is a + # frequent Japanese Whisper tail hallucination. + loop = "ご視聴ありがとうございました" * 6 + raw = f"明日の会議は午後三時です。{loop}" + result = collapse_repetitive_artifacts(raw) + assert "ご視聴ありがとうございました" not in result + assert "明日の会議は午後三時です" in result + + +def test_cjk_short_run_preserved(): + # Five repeats — below threshold, stays in. + raw = "好好好好好" + assert collapse_repetitive_artifacts(raw) == raw + + +# ── whitespace / edge cases ────────────────────────────────────────────── + + +def test_empty_string_passes_through(): + assert collapse_repetitive_artifacts("") == "" + + +def test_below_word_threshold_passes_through_unmodified(): + raw = "just three words" + assert collapse_repetitive_artifacts(raw) == raw + + +def test_emphasis_vowel_run_preserved(): + # "wooooooow" is 1 char (plus 8 o's). Character-level min unit is 2, + # so "oo…" doesn't get stripped and this legitimate emphasis stays. + raw = "that's wooooooow amazing" + assert collapse_repetitive_artifacts(raw) == raw + + +def test_custom_threshold_honored(): + # With min_run=3, even short rhetorical repetition should now strip. + raw = "ha ha ha ha context" + result = collapse_repetitive_artifacts(raw, min_run=3) + assert "ha ha" not in result + assert "context" in result + + +def test_leading_and_trailing_whitespace_stripped_after_collapse(): + # When character pass fires, the normalised result is stripped so + # downstream prompts don't carry edge whitespace from the removal. + loop = "loop-phrase " * 7 + raw = loop + assert collapse_repetitive_artifacts(raw) == ""