"""normalize_title: de-shout / de-clutter video titles without mangling non-English text.""" from app.titles import normalize_title def test_empty_and_whitespace_pass_through_unchanged(): assert normalize_title(None) is None assert normalize_title("") == "" assert normalize_title(" ") == " " def test_plain_title_is_left_alone(): assert normalize_title("How I built a boat") == "How I built a boat" def test_shouting_title_is_de_capsed_and_first_letter_kept_upper(): # >=2 all-caps runs and >=50% caps → treated as shouting: title-case the long words. out = normalize_title("THIS IS A HUGE ANNOUNCEMENT") assert out is not None assert out == out[0].upper() + out[1:] # first alpha char stays capitalized assert "HUGE" not in out and "ANNOUNCEMENT" not in out def test_trailing_hashtags_are_stripped(): assert normalize_title("Great cooking video #food #yummy") == "Great cooking video" def test_numeric_episode_marker_is_not_treated_as_a_hashtag(): # A bare "#3" is an episode marker, not an SEO hashtag (no letters) → it must survive. assert normalize_title("My show #3") == "My show #3" def test_midstring_hashtags_are_fast_and_kept_redos_regression(): # Facebook prepends an engagement prefix and appends a page name, leaving a long hashtag cluster # in the MIDDLE. That is not a trailing SEO cluster, so the tags stay — and, critically, this # exact shape used to trigger catastrophic backtracking (ReDoS) in `_TRAILING_HASHTAGS` that # pinned a download-worker thread at 100% CPU forever inside the yt-dlp progress hook. import time raw = ( "398K views · 6.1K reactions | The money should follow 🙄 #reel #reels #meme #memes " "#fun #funny #work #worklife #office #viral #trending #fyp #foryou #foryoupage | Memebonanza4U" ) t = time.perf_counter() out = normalize_title(raw) or "" assert time.perf_counter() - t < 1.0 # linear now; the old regex ran for minutes / never returned assert "#foryoupage" in out and "#reel" in out # mid-string hashtags preserved (not trailing) assert "views" not in out # the engagement-count prefix was still stripped def test_emoji_and_symbols_are_dropped(): assert "🔥" not in (normalize_title("Fire recipe 🔥🔥") or "") def test_repeated_punctuation_is_collapsed(): assert normalize_title("Wait what!!!!") == "Wait what!" def test_accented_non_english_letters_survive(): # Siftlode is trilingual — Hungarian/German letters must not be ASCII-folded away. for title in ["Gulyásleves főzés", "Über die Straße"]: out = normalize_title(title) or "" for ch in "áöüőűé": if ch in title: assert ch in out