diff --git a/backend/app/utils.py b/backend/app/utils.py index 93793d7c..0260b591 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -39,11 +39,14 @@ def to_tsquery_str(q: str) -> str | None: def normalize_description(text: str | None) -> str | None: - """Strip whitespace-only lines from a channel/video description before storing. YouTube - descriptions pad contact/link blocks with runs of blank lines; keeping them wastes storage, + """Strip whitespace-only lines from a CHANNEL description before storing. YouTube channel + descriptions pad their contact/link blocks with runs of blank lines; keeping them wastes storage, bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and - right-strips each kept line. None stays None; all-blank collapses to "". Applied at the - sync/store site so the DB (and the derived search_vector) holds the clean text.""" + right-strips each kept line; `str.splitlines()` also folds CR / CRLF and Unicode line separators + to "\\n" (a benign line-ending normalization). None stays None; all-blank collapses to "". + + Deliberately CHANNEL-only — video descriptions are left with their original formatting: they are + user-facing prose where paragraph breaks carry meaning, unlike a channel's metadata block.""" if text is None: return None kept = [line.rstrip() for line in text.splitlines() if line.strip()] diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 2ddb2883..c826f8ef 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -50,3 +50,8 @@ def test_normalize_description_all_blank_collapses_to_empty(): def test_normalize_description_single_line_unchanged(): assert normalize_description("just one line") == "just one line" + + +def test_normalize_description_folds_crlf_line_endings(): + # splitlines() folds CR/CRLF to \n — a documented, benign line-ending normalization. + assert normalize_description("a\r\nb\r\n\r\nc") == "a\nb\nc"