diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index 019445af..3969bed2 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -6,6 +6,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session from app.models import Channel, Subscription, User +from app.utils import normalize_description from app.youtube.client import YouTubeClient, best_thumbnail log = logging.getLogger("siftlode.sync") @@ -55,7 +56,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None: channel.title = snippet.get("title") or channel.title channel.handle = snippet.get("customUrl") - channel.description = snippet.get("description") + channel.description = normalize_description(snippet.get("description")) channel.thumbnail_url = ( best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url ) @@ -104,7 +105,7 @@ def import_subscriptions(db: Session, user: User) -> dict: channel = Channel( id=cid, title=sub.get("title"), - description=sub.get("description"), + description=normalize_description(sub.get("description")), thumbnail_url=sub.get("thumbnail_url"), ) db.add(channel) diff --git a/backend/app/utils.py b/backend/app/utils.py index c6cfb179..93793d7c 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -36,3 +36,15 @@ def to_tsquery_str(q: str) -> str | None: return None tokens[-1] = tokens[-1] + ":*" return " & ".join(tokens) + + +def normalize_description(text: str | None) -> str | None: + """Strip whitespace-only lines from a channel/video description before storing. YouTube + descriptions pad contact/link blocks with runs of blank lines; keeping them wastes storage, + bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and + right-strips each kept line. None stays None; all-blank collapses to "". Applied at the + sync/store site so the DB (and the derived search_vector) holds the clean text.""" + if text is None: + return None + kept = [line.rstrip() for line in text.splitlines() if line.strip()] + return "\n".join(kept) diff --git a/backend/tests/test_utils.py b/backend/tests/test_utils.py index 3e93b7d7..2ddb2883 100644 --- a/backend/tests/test_utils.py +++ b/backend/tests/test_utils.py @@ -1,7 +1,7 @@ """iso(): the single date/datetime→JSON serializer the routes share (R6 S1a).""" from datetime import date, datetime, timezone -from app.utils import iso +from app.utils import iso, normalize_description def test_none_maps_to_none(): @@ -28,3 +28,25 @@ def test_matches_datetime_isoformat_contract(): # hand-roll `x.isoformat() if x else None`. dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc) assert iso(dt) == dt.isoformat() + + +def test_normalize_description_none_stays_none(): + assert normalize_description(None) is None + + +def test_normalize_description_drops_blank_lines(): + # A YouTube contact block padded with runs of blank lines — the waste we strip before storing. + raw = "Contact us\n\n\nEmail: a@b.com\n \nInstagram: @x\n\n\n" + assert normalize_description(raw) == "Contact us\nEmail: a@b.com\nInstagram: @x" + + +def test_normalize_description_right_strips_and_trims(): + assert normalize_description(" hi there \n\n bye ") == " hi there\n bye" + + +def test_normalize_description_all_blank_collapses_to_empty(): + assert normalize_description("\n \n\t\n") == "" + + +def test_normalize_description_single_line_unchanged(): + assert normalize_description("just one line") == "just one line"