"""normalize_title: de-shout / de-clutter video titles without mangling non-English text.""" from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeout import pytest from app.titles import normalize_title def test_empty_and_whitespace_pass_through_unchanged(): assert normalize_title(None) is None assert normalize_title("") == "" assert normalize_title(" ") == " " def test_plain_title_is_left_alone(): assert normalize_title("How I built a boat") == "How I built a boat" def test_shouting_title_is_de_capsed_and_first_letter_kept_upper(): # >=2 all-caps runs and >=50% caps → treated as shouting: title-case the long words. out = normalize_title("THIS IS A HUGE ANNOUNCEMENT") assert out is not None assert out == out[0].upper() + out[1:] # first alpha char stays capitalized assert "HUGE" not in out and "ANNOUNCEMENT" not in out def test_trailing_hashtags_are_stripped(): assert normalize_title("Great cooking video #food #yummy") == "Great cooking video" def test_numeric_episode_marker_is_not_treated_as_a_hashtag(): # A bare "#3" is an episode marker, not an SEO hashtag (no letters) → it must survive. assert normalize_title("My show #3") == "My show #3" def test_midstring_hashtags_dont_hang_and_are_kept_redos_regression(): # Facebook prepends an engagement prefix and appends a page name, leaving a long hashtag cluster # in the MIDDLE. That is not a trailing SEO cluster, so the tags stay — and, critically, this # exact shape used to trigger catastrophic backtracking (ReDoS) in `_TRAILING_HASHTAGS` that # pinned a download-worker thread at 100% CPU forever inside the yt-dlp progress hook. raw = ( "398K views · 6.1K reactions | The money should follow 🙄 #reel #reels #meme #memes " "#fun #funny #work #worklife #office #viral #trending #fyp #foryou #foryoupage | Memebonanza4U" ) # Run in a worker thread with a generous ceiling: the fixed regex returns in microseconds, so # only a genuine catastrophic-backtracking regression fails to complete — a hang is caught # deterministically rather than by asserting on wall-clock slack in the happy path. ex = ThreadPoolExecutor(max_workers=1) fut = ex.submit(normalize_title, raw) try: out = fut.result(timeout=5) or "" except FuturesTimeout: pytest.fail( "normalize_title did not return within 5s — catastrophic backtracking regressed" ) finally: ex.shutdown(wait=False) # never join a (hypothetically) wedged thread assert "#foryoupage" in out and "#reel" in out # mid-string hashtags preserved (not trailing) assert "views" not in out # the engagement-count prefix was still stripped def test_emoji_and_symbols_are_dropped(): assert "🔥" not in (normalize_title("Fire recipe 🔥🔥") or "") def test_repeated_punctuation_is_collapsed(): assert normalize_title("Wait what!!!!") == "Wait what!" def test_accented_non_english_letters_survive(): # Siftlode is trilingual — Hungarian/German letters must not be ASCII-folded away. for title in ["Gulyásleves főzés", "Über die Straße"]: out = normalize_title(title) or "" for ch in "áöüőűé": if ch in title: assert ch in out