From 663f30feeb64b034e158e597e7983813958c25ec Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 26 Sep 2026 21:45:46 +0200 Subject: [PATCH] fix(sync): refresh a channel's details once they are a week old Channel metadata (banner, avatar, description, counts) was fetched only the first time a channel was seen: the subscription import asked only for channels with no details, and the channel page only for ones missing a join date. 394 of 397 subscribed channels had July metadata. Both now also refresh details older than DETAILS_MAX_AGE (7 days), through one shared staleness rule; channels.list costs 1 unit per 50 channels. --- backend/app/routes/channels.py | 11 +- backend/app/sync/subscriptions.py | 31 ++++- backend/tests/test_channel_details_refresh.py | 107 ++++++++++++++++++ 3 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_channel_details_refresh.py diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 5fdd68e8..0f72f8c2 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -28,7 +28,7 @@ from app.models import ( from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal from app.sync.explore import explore_ingest_page from app.sync.runner import backfill_channel_now, run_recent_backfill -from app.sync.subscriptions import apply_channel_details +from app.sync.subscriptions import apply_channel_details, details_stale from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused @@ -297,10 +297,13 @@ def channel_detail( relationship to it (subscribed / explored) and how many of its videos are in the catalog. Lazily fills the About metadata (view count, join date, banner, links) the first time it's needed — a stub, or a channel last enriched before those fields existed (published_at is a - reliable sentinel: every real channel has a creation date). The enrich uses the API key - (1 unit, cached after), so it's skipped for the quota-less demo account.""" + reliable sentinel: every real channel has a creation date) — and refreshes it once it is stale + (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the + subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit, + cached after), so it's skipped for the quota-less demo account.""" channel = get_or_404(db, Channel, channel_id, what="channel") - if channel.published_at is None and not user.is_demo: + stale = channel.published_at is None or details_stale(channel, datetime.now(timezone.utc)) + if stale and not user.is_demo: try: with ( quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index bf964093..6bd948e0 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -1,8 +1,8 @@ """Import the authenticated user's YouTube subscriptions and channel metadata.""" -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone -from sqlalchemy import select +from sqlalchemy import or_, select from sqlalchemy.orm import Session from app import logs @@ -47,6 +47,27 @@ def _external_links(branding: dict) -> list | None: return out or None +# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and +# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel +# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no +# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for. +DETAILS_MAX_AGE = timedelta(days=7) + + +def details_stale(channel: Channel, now: datetime) -> bool: + """Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`).""" + synced = channel.details_synced_at + return synced is None or synced < now - DETAILS_MAX_AGE + + +def details_stale_clause(now: datetime): + """`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale".""" + return or_( + Channel.details_synced_at.is_(None), + Channel.details_synced_at < now - DETAILS_MAX_AGE, + ) + + def apply_channel_details(db: Session, items: list[dict]) -> None: now = datetime.now(timezone.utc) for item in items: @@ -92,7 +113,7 @@ def should_prune(fetched_count: int, existing_count: int) -> bool: def import_subscriptions(db: Session, user: User) -> dict: """Pull all subscriptions for `user`, upsert channels + subscription links, prune channels the user has unsubscribed from on YouTube, and fetch channel details for - any channel we don't have details for yet.""" + any channel whose details are missing or stale (`DETAILS_MAX_AGE`).""" fetched: dict[str, dict] = {} new_channels = 0 @@ -145,12 +166,12 @@ def import_subscriptions(db: Session, user: User) -> dict: removed += 1 db.commit() - # Fetch details for channels that don't have them yet. + # Fetch details for channels that don't have them yet, or had them too long ago. need_details = ( db.execute( select(Channel).where( Channel.id.in_(fetched.keys()), - Channel.details_synced_at.is_(None), + details_stale_clause(datetime.now(timezone.utc)), ) ) .scalars() diff --git a/backend/tests/test_channel_details_refresh.py b/backend/tests/test_channel_details_refresh.py new file mode 100644 index 00000000..0c06f220 --- /dev/null +++ b/backend/tests/test_channel_details_refresh.py @@ -0,0 +1,107 @@ +"""A channel's metadata is fetched again once it is stale, not only the first time it is seen. + +Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving: +a channel that kept the metadata of the day it was first imported showed a dead banner forever. The +subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and +leave fresh metadata alone (each refresh spends quota). DB lane. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from app.models import Channel, Subscription +from app.routes import channels +from app.sync import subscriptions +from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions + +NOW = datetime.now(timezone.utc) +FRESH = NOW - timedelta(days=1) +STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1) + + +def _item(cid: str) -> dict: + return { + "id": cid, + "snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"}, + "brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}}, + } + + +def _youtube(asked: list[list[str]], subscribed: list[str]): + class _Client: + def __init__(self, db, user): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def iter_subscriptions(self): + for cid in subscribed: + yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"} + + def get_channels(self, ids): + asked.append(sorted(ids)) + return [_item(cid) for cid in ids] + + return _Client + + +def _channel(db, cid: str, synced: datetime | None) -> None: + db.add( + Channel( + id=cid, + title=cid, + banner_url=f"https://yt3.example/{cid}-old", + published_at=datetime(2010, 1, 1, tzinfo=timezone.utc), + details_synced_at=synced, + ) + ) + + +def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones( + db, make_user, monkeypatch +): + user = make_user() + _channel(db, "UCfresh", FRESH) + _channel(db, "UCstale", STALE) + _channel(db, "UCnever", None) + db.flush() + for cid in ("UCfresh", "UCstale", "UCnever"): + db.add(Subscription(user_id=user.id, channel_id=cid)) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr( + subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"]) + ) + + result = import_subscriptions(db, user) + + assert asked == [["UCnever", "UCstale"]] + assert result["channels_detailed"] == 2 + stale = db.get(Channel, "UCstale") + assert stale is not None + assert stale.banner_url == "https://yt3.example/UCstale-new" + assert stale.details_synced_at is not None and stale.details_synced_at > STALE + fresh = db.get(Channel, "UCfresh") + assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old" + + +@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)]) +def test_the_channel_page_refreshes_a_stale_channel_only( + db, make_user, monkeypatch, synced, refreshed +): + user = make_user() + _channel(db, "UCpage", synced) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [])) + + body = channels.channel_detail("UCpage", user=user, db=db) + + assert asked == ([["UCpage"]] if refreshed else []) + suffix = "new" if refreshed else "old" + assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}"