fix(sync): strip blank lines from channel descriptions before storing
YouTube channel descriptions pad their contact/link blocks with runs of blank lines, which the About popover renders verbatim (whitespace-pre-wrap) — wasted space and storage, and noise in the search_vector. Add normalize_description() and apply it at both channel store sites (full details + initial import) so the DB holds the clean text. New channels are clean immediately; existing rows heal on their next metadata re-sync. Unit-tested in test_utils.
This commit is contained in:
@@ -6,6 +6,7 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models import Channel, Subscription, User
|
from app.models import Channel, Subscription, User
|
||||||
|
from app.utils import normalize_description
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.sync")
|
log = logging.getLogger("siftlode.sync")
|
||||||
@@ -55,7 +56,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
|
|||||||
|
|
||||||
channel.title = snippet.get("title") or channel.title
|
channel.title = snippet.get("title") or channel.title
|
||||||
channel.handle = snippet.get("customUrl")
|
channel.handle = snippet.get("customUrl")
|
||||||
channel.description = snippet.get("description")
|
channel.description = normalize_description(snippet.get("description"))
|
||||||
channel.thumbnail_url = (
|
channel.thumbnail_url = (
|
||||||
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
||||||
)
|
)
|
||||||
@@ -104,7 +105,7 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
|||||||
channel = Channel(
|
channel = Channel(
|
||||||
id=cid,
|
id=cid,
|
||||||
title=sub.get("title"),
|
title=sub.get("title"),
|
||||||
description=sub.get("description"),
|
description=normalize_description(sub.get("description")),
|
||||||
thumbnail_url=sub.get("thumbnail_url"),
|
thumbnail_url=sub.get("thumbnail_url"),
|
||||||
)
|
)
|
||||||
db.add(channel)
|
db.add(channel)
|
||||||
|
|||||||
@@ -36,3 +36,15 @@ def to_tsquery_str(q: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
tokens[-1] = tokens[-1] + ":*"
|
tokens[-1] = tokens[-1] + ":*"
|
||||||
return " & ".join(tokens)
|
return " & ".join(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_description(text: str | None) -> str | None:
|
||||||
|
"""Strip whitespace-only lines from a channel/video description before storing. YouTube
|
||||||
|
descriptions pad contact/link blocks with runs of blank lines; keeping them wastes storage,
|
||||||
|
bloats the About popover, and adds nothing to the search document. Drops purely-empty lines and
|
||||||
|
right-strips each kept line. None stays None; all-blank collapses to "". Applied at the
|
||||||
|
sync/store site so the DB (and the derived search_vector) holds the clean text."""
|
||||||
|
if text is None:
|
||||||
|
return None
|
||||||
|
kept = [line.rstrip() for line in text.splitlines() if line.strip()]
|
||||||
|
return "\n".join(kept)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
"""iso(): the single date/datetime→JSON serializer the routes share (R6 S1a)."""
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from app.utils import iso
|
from app.utils import iso, normalize_description
|
||||||
|
|
||||||
|
|
||||||
def test_none_maps_to_none():
|
def test_none_maps_to_none():
|
||||||
@@ -28,3 +28,25 @@ def test_matches_datetime_isoformat_contract():
|
|||||||
# hand-roll `x.isoformat() if x else None`.
|
# hand-roll `x.isoformat() if x else None`.
|
||||||
dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc)
|
dt = datetime(2020, 1, 2, 3, 4, 5, 678901, tzinfo=timezone.utc)
|
||||||
assert iso(dt) == dt.isoformat()
|
assert iso(dt) == dt.isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_none_stays_none():
|
||||||
|
assert normalize_description(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_drops_blank_lines():
|
||||||
|
# A YouTube contact block padded with runs of blank lines — the waste we strip before storing.
|
||||||
|
raw = "Contact us\n\n\nEmail: a@b.com\n \nInstagram: @x\n\n\n"
|
||||||
|
assert normalize_description(raw) == "Contact us\nEmail: a@b.com\nInstagram: @x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_right_strips_and_trims():
|
||||||
|
assert normalize_description(" hi there \n\n bye ") == " hi there\n bye"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_all_blank_collapses_to_empty():
|
||||||
|
assert normalize_description("\n \n\t\n") == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_description_single_line_unchanged():
|
||||||
|
assert normalize_description("just one line") == "just one line"
|
||||||
|
|||||||
Reference in New Issue
Block a user