From c9e95ecea678146f4fd8b5d528636bdace7baf92 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sun, 27 Sep 2026 13:58:23 +0200 Subject: [PATCH] fix(channels): a channel YouTube no longer returns is not asked for on every view channels.list simply omits a terminated channel, so nothing was stamped and the channel page spent a quota unit on it every time. The miss is stamped like a fetch; the published_at sentinel that kept it stale is gone. --- backend/app/routes/channels.py | 24 +++++------ backend/app/routes/search.py | 2 +- backend/app/sync/subscriptions.py | 16 ++++++-- backend/tests/test_channel_details_refresh.py | 40 ++++++++++++++++++- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 0f72f8c2..f28a9d20 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -190,7 +190,7 @@ def discover_channels( quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt, ): - apply_channel_details(db, yt.get_channels(need)) + apply_channel_details(db, yt.get_channels(need), need) db.commit() # Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST, # as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up). @@ -296,20 +296,20 @@ def channel_detail( """One channel's "About" detail for the channel page header/About tab, plus this user's relationship to it (subscribed / explored) and how many of its videos are in the catalog. Lazily fills the About metadata (view count, join date, banner, links) the first time it's - needed — a stub, or a channel last enriched before those fields existed (published_at is a - reliable sentinel: every real channel has a creation date) — and refreshes it once it is stale - (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the + needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit, cached after), so it's skipped for the quota-less demo account.""" channel = get_or_404(db, Channel, channel_id, what="channel") - stale = channel.published_at is None or details_stale(channel, datetime.now(timezone.utc)) + # No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on + # every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway. + stale = details_stale(channel, datetime.now(timezone.utc)) if stale and not user.is_demo: try: with ( quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), YouTubeClient(db, user) as yt, ): - apply_channel_details(db, yt.get_channels([channel_id])) + apply_channel_details(db, yt.get_channels([channel_id]), [channel_id]) db.commit() channel = db.get(Channel, channel_id) except YouTubeError as exc: @@ -415,12 +415,10 @@ def explore_channel( YouTubeClient(db, user) as yt, ): # Enrich for the About panel + the uploads playlist id (needed to ingest videos). - if ( - channel.details_synced_at is None - or channel.uploads_playlist_id is None - or channel.published_at is None - ): - apply_channel_details(db, yt.get_channels([channel_id])) + # Stale, not "a field is missing": a terminated channel never gets an uploads playlist + # or a creation date, and asking again on every explore would spend quota on a miss. + if details_stale(channel, datetime.now(timezone.utc)): + apply_channel_details(db, yt.get_channels([channel_id]), [channel_id]) db.commit() channel = db.get(Channel, channel_id) result = explore_ingest_page(db, yt, channel, page_token) @@ -593,7 +591,7 @@ def subscribe( # must not block recording the subscription the user asked for. if channel.details_synced_at is None: try: - apply_channel_details(db, yt.get_channels([channel_id])) + apply_channel_details(db, yt.get_channels([channel_id]), [channel_id]) except YouTubeError as exc: log.warning( "subscribe: stub enrich failed: %s", diff --git a/backend/app/routes/search.py b/backend/app/routes/search.py index 7fa9af3b..31bc2fce 100644 --- a/backend/app/routes/search.py +++ b/backend/app/routes/search.py @@ -130,7 +130,7 @@ def _ingest_candidates( try: with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER): details = yt.get_channels(new_channel_ids) - apply_channel_details(db, details) + apply_channel_details(db, details, new_channel_ids) db.commit() except YouTubeError: # The stubs remain and the details can be backfilled later — but until they are, those diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index 6bd948e0..ae3cbe92 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -1,5 +1,6 @@ """Import the authenticated user's YouTube subscriptions and channel metadata.""" +from collections.abc import Iterable from datetime import datetime, timedelta, timezone from sqlalchemy import or_, select @@ -68,8 +69,16 @@ def details_stale_clause(now: datetime): ) -def apply_channel_details(db: Session, items: list[dict]) -> None: +def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str]) -> None: + """Write `channels.list`'s answer onto the rows. `asked` is every id the call asked for: one + YouTube did not return (terminated, deleted) is stamped as synced too — otherwise it stays + stale forever and every view / import asks for it again, spending quota on a known miss.""" now = datetime.now(timezone.utc) + returned = {item["id"] for item in items} + for cid in set(asked) - returned: + missing = db.get(Channel, cid) + if missing is not None: + missing.details_synced_at = now for item in items: channel = db.get(Channel, item["id"]) if channel is None: @@ -179,8 +188,9 @@ def import_subscriptions(db: Session, user: User) -> dict: ) detailed = 0 if need_details: - items = yt.get_channels([c.id for c in need_details]) - apply_channel_details(db, items) + ids = [c.id for c in need_details] + items = yt.get_channels(ids) + apply_channel_details(db, items, ids) detailed = len(items) db.commit() diff --git a/backend/tests/test_channel_details_refresh.py b/backend/tests/test_channel_details_refresh.py index 0c06f220..f4906cf1 100644 --- a/backend/tests/test_channel_details_refresh.py +++ b/backend/tests/test_channel_details_refresh.py @@ -28,7 +28,7 @@ def _item(cid: str) -> dict: } -def _youtube(asked: list[list[str]], subscribed: list[str]): +def _youtube(asked: list[list[str]], subscribed: list[str], gone: tuple[str, ...] = ()): class _Client: def __init__(self, db, user): pass @@ -45,7 +45,8 @@ def _youtube(asked: list[list[str]], subscribed: list[str]): def get_channels(self, ids): asked.append(sorted(ids)) - return [_item(cid) for cid in ids] + # A terminated / deleted channel is simply absent from `channels.list`'s answer. + return [_item(cid) for cid in ids if cid not in gone] return _Client @@ -105,3 +106,38 @@ def test_the_channel_page_refreshes_a_stale_channel_only( assert asked == ([["UCpage"]] if refreshed else []) suffix = "new" if refreshed else "old" assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}" + + +def test_a_channel_youtube_no_longer_returns_is_not_asked_for_on_every_view( + db, make_user, monkeypatch +): + """A terminated channel is missing from `channels.list`'s answer, so nothing was stamped and the + channel page asked for it again (1 quota unit) on every single view. The miss is stamped like a + fetch, so the next ask waits for `DETAILS_MAX_AGE` like any other channel.""" + user = make_user() + db.add(Channel(id="UCgone", title="UCgone")) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [], gone=("UCgone",))) + + channels.channel_detail("UCgone", user=user, db=db) + channels.channel_detail("UCgone", user=user, db=db) + + assert asked == [["UCgone"]] + + +def test_the_import_stamps_a_channel_youtube_no_longer_returns(db, make_user, monkeypatch): + user = make_user() + _channel(db, "UCgone", STALE) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UCgone")) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr( + subscriptions, "YouTubeClient", _youtube(asked, ["UCgone"], gone=("UCgone",)) + ) + + import_subscriptions(db, user) + import_subscriptions(db, user) + + assert asked == [["UCgone"]]