fix(sync): strip blank lines from channel descriptions before storing

YouTube channel descriptions pad their contact/link blocks with runs of blank
lines, which the About popover renders verbatim (whitespace-pre-wrap) — wasted
space and storage, and noise in the search_vector. Add normalize_description()
and apply it at both channel store sites (full details + initial import) so the
DB holds the clean text. New channels are clean immediately; existing rows heal
on their next metadata re-sync. Unit-tested in test_utils.
This commit is contained in:
2026-08-19 01:13:56 +02:00
parent 48787af11c
commit ddb7818d38
3 changed files with 38 additions and 3 deletions
+12
View File
@@ -36,3 +36,15 @@ def to_tsquery_str(q: str) -> str | None:
return None
tokens[-1] = tokens[-1] + ":*"
return " & ".join(tokens)
def normalize_description(text: str | None) -> str | None:
"""Strip whitespace-only lines from a channel/video description before storing. YouTube
descriptions pad contact/link blocks with runs of blank lines; keeping them wastes storage,
bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and
right-strips each kept line. None stays None; all-blank collapses to "". Applied at the
sync/store site so the DB (and the derived search_vector) holds the clean text."""
if text is None:
return None
kept = [line.rstrip() for line in text.splitlines() if line.strip()]
return "\n".join(kept)