peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(sync): refresh a channel's details once they are a week old

Browse Source 
Channel metadata (banner, avatar, description, counts) was fetched only the first time a
channel was seen: the subscription import asked only for channels with no details, and the
channel page only for ones missing a join date. 394 of 397 subscribed channels had July
metadata. Both now also refresh details older than DETAILS_MAX_AGE (7 days), through one
shared staleness rule; channels.list costs 1 unit per 50 channels.
This commit is contained in:
peter
2026-09-26 21:45:46 +02:00
parent 58878958d8
commit 663f30feeb
3 changed files with 140 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routes/channels.py
+7 -4
View File
@@ -28,7 +28,7 @@ from app.models import (
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
from app.sync.explore import explore_ingest_page
from app.sync.runner import backfill_channel_now, run_recent_backfill
from app.sync.subscriptions import apply_channel_details
from app.sync.subscriptions import apply_channel_details, details_stale
from app.utils import iso
from app.youtube.client import YouTubeClient, YouTubeError
from app.refusal import refused
@@ -297,10 +297,13 @@ def channel_detail(
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
needed — a stub, or a channel last enriched before those fields existed (published_at is a
reliable sentinel: every real channel has a creation date). The enrich uses the API key
(1 unit, cached after), so it's skipped for the quota-less demo account."""
reliable sentinel: every real channel has a creation date) — and refreshes it once it is stale
(`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
cached after), so it's skipped for the quota-less demo account."""
channel = get_or_404(db, Channel, channel_id, what="channel")
if channel.published_at is None and not user.is_demo:
stale = channel.published_at is None or details_stale(channel, datetime.now(timezone.utc))
if stale and not user.is_demo:
try:
with (
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
backend/app/sync/subscriptions.py
+26 -5
View File
@@ -1,8 +1,8 @@
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from app import logs
@@ -47,6 +47,27 @@ def _external_links(branding: dict) -> list | None:
return out or None
# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and
# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel
# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no
# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for.
DETAILS_MAX_AGE = timedelta(days=7)
def details_stale(channel: Channel, now: datetime) -> bool:
"""Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`)."""
synced = channel.details_synced_at
return synced is None or synced < now - DETAILS_MAX_AGE
def details_stale_clause(now: datetime):
"""`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale"."""
return or_(
Channel.details_synced_at.is_(None),
Channel.details_synced_at < now - DETAILS_MAX_AGE,
)
def apply_channel_details(db: Session, items: list[dict]) -> None:
now = datetime.now(timezone.utc)
for item in items:
@@ -92,7 +113,7 @@ def should_prune(fetched_count: int, existing_count: int) -> bool:
def import_subscriptions(db: Session, user: User) -> dict:
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
channels the user has unsubscribed from on YouTube, and fetch channel details for
any channel we don't have details for yet."""
any channel whose details are missing or stale (`DETAILS_MAX_AGE`)."""
fetched: dict[str, dict] = {}
new_channels = 0
@@ -145,12 +166,12 @@ def import_subscriptions(db: Session, user: User) -> dict:
removed += 1
db.commit()
# Fetch details for channels that don't have them yet.
# Fetch details for channels that don't have them yet, or had them too long ago.
need_details = (
db.execute(
select(Channel).where(
Channel.id.in_(fetched.keys()),
Channel.details_synced_at.is_(None),
details_stale_clause(datetime.now(timezone.utc)),
)
)
.scalars()
backend/tests/test_channel_details_refresh.py
+107
View File
@@ -0,0 +1,107 @@
"""A channel's metadata is fetched again once it is stale, not only the first time it is seen.
Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving:
a channel that kept the metadata of the day it was first imported showed a dead banner forever. The
subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and
leave fresh metadata alone (each refresh spends quota). DB lane.
"""
from datetime import datetime, timedelta, timezone
import pytest
from app.models import Channel, Subscription
from app.routes import channels
from app.sync import subscriptions
from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions
NOW = datetime.now(timezone.utc)
FRESH = NOW - timedelta(days=1)
STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1)
def _item(cid: str) -> dict:
return {
"id": cid,
"snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"},
"brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}},
}
def _youtube(asked: list[list[str]], subscribed: list[str]):
class _Client:
def __init__(self, db, user):
pass
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def iter_subscriptions(self):
for cid in subscribed:
yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"}
def get_channels(self, ids):
asked.append(sorted(ids))
return [_item(cid) for cid in ids]
return _Client
def _channel(db, cid: str, synced: datetime | None) -> None:
db.add(
Channel(
id=cid,
title=cid,
banner_url=f"https://yt3.example/{cid}-old",
published_at=datetime(2010, 1, 1, tzinfo=timezone.utc),
details_synced_at=synced,
)
)
def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones(
db, make_user, monkeypatch
):
user = make_user()
_channel(db, "UCfresh", FRESH)
_channel(db, "UCstale", STALE)
_channel(db, "UCnever", None)
db.flush()
for cid in ("UCfresh", "UCstale", "UCnever"):
db.add(Subscription(user_id=user.id, channel_id=cid))
db.commit()
asked: list[list[str]] = []
monkeypatch.setattr(
subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"])
)
result = import_subscriptions(db, user)
assert asked == [["UCnever", "UCstale"]]
assert result["channels_detailed"] == 2
stale = db.get(Channel, "UCstale")
assert stale is not None
assert stale.banner_url == "https://yt3.example/UCstale-new"
assert stale.details_synced_at is not None and stale.details_synced_at > STALE
fresh = db.get(Channel, "UCfresh")
assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old"
@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)])
def test_the_channel_page_refreshes_a_stale_channel_only(
db, make_user, monkeypatch, synced, refreshed
):
user = make_user()
_channel(db, "UCpage", synced)
db.commit()
asked: list[list[str]] = []
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, []))
body = channels.channel_detail("UCpage", user=user, db=db)
assert asked == ([["UCpage"]] if refreshed else [])
suffix = "new" if refreshed else "old"
assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}"