From 663f30feeb64b034e158e597e7983813958c25ec Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 26 Sep 2026 21:45:46 +0200 Subject: [PATCH 1/2] fix(sync): refresh a channel's details once they are a week old Channel metadata (banner, avatar, description, counts) was fetched only the first time a channel was seen: the subscription import asked only for channels with no details, and the channel page only for ones missing a join date. 394 of 397 subscribed channels had July metadata. Both now also refresh details older than DETAILS_MAX_AGE (7 days), through one shared staleness rule; channels.list costs 1 unit per 50 channels. --- backend/app/routes/channels.py | 11 +- backend/app/sync/subscriptions.py | 31 ++++- backend/tests/test_channel_details_refresh.py | 107 ++++++++++++++++++ 3 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 backend/tests/test_channel_details_refresh.py diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index 5fdd68e8..0f72f8c2 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -28,7 +28,7 @@ from app.models import ( from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal from app.sync.explore import explore_ingest_page from app.sync.runner import backfill_channel_now, run_recent_backfill -from app.sync.subscriptions import apply_channel_details +from app.sync.subscriptions import apply_channel_details, details_stale from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused @@ -297,10 +297,13 @@ def channel_detail( relationship to it (subscribed / explored) and how many of its videos are in the catalog. Lazily fills the About metadata (view count, join date, banner, links) the first time it's needed — a stub, or a channel last enriched before those fields existed (published_at is a - reliable sentinel: every real channel has a creation date). The enrich uses the API key - (1 unit, cached after), so it's skipped for the quota-less demo account.""" + reliable sentinel: every real channel has a creation date) — and refreshes it once it is stale + (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the + subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit, + cached after), so it's skipped for the quota-less demo account.""" channel = get_or_404(db, Channel, channel_id, what="channel") - if channel.published_at is None and not user.is_demo: + stale = channel.published_at is None or details_stale(channel, datetime.now(timezone.utc)) + if stale and not user.is_demo: try: with ( quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER), diff --git a/backend/app/sync/subscriptions.py b/backend/app/sync/subscriptions.py index bf964093..6bd948e0 100644 --- a/backend/app/sync/subscriptions.py +++ b/backend/app/sync/subscriptions.py @@ -1,8 +1,8 @@ """Import the authenticated user's YouTube subscriptions and channel metadata.""" -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone -from sqlalchemy import select +from sqlalchemy import or_, select from sqlalchemy.orm import Session from app import logs @@ -47,6 +47,27 @@ def _external_links(branding: dict) -> list | None: return out or None +# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and +# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel +# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no +# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for. +DETAILS_MAX_AGE = timedelta(days=7) + + +def details_stale(channel: Channel, now: datetime) -> bool: + """Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`).""" + synced = channel.details_synced_at + return synced is None or synced < now - DETAILS_MAX_AGE + + +def details_stale_clause(now: datetime): + """`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale".""" + return or_( + Channel.details_synced_at.is_(None), + Channel.details_synced_at < now - DETAILS_MAX_AGE, + ) + + def apply_channel_details(db: Session, items: list[dict]) -> None: now = datetime.now(timezone.utc) for item in items: @@ -92,7 +113,7 @@ def should_prune(fetched_count: int, existing_count: int) -> bool: def import_subscriptions(db: Session, user: User) -> dict: """Pull all subscriptions for `user`, upsert channels + subscription links, prune channels the user has unsubscribed from on YouTube, and fetch channel details for - any channel we don't have details for yet.""" + any channel whose details are missing or stale (`DETAILS_MAX_AGE`).""" fetched: dict[str, dict] = {} new_channels = 0 @@ -145,12 +166,12 @@ def import_subscriptions(db: Session, user: User) -> dict: removed += 1 db.commit() - # Fetch details for channels that don't have them yet. + # Fetch details for channels that don't have them yet, or had them too long ago. need_details = ( db.execute( select(Channel).where( Channel.id.in_(fetched.keys()), - Channel.details_synced_at.is_(None), + details_stale_clause(datetime.now(timezone.utc)), ) ) .scalars() diff --git a/backend/tests/test_channel_details_refresh.py b/backend/tests/test_channel_details_refresh.py new file mode 100644 index 00000000..0c06f220 --- /dev/null +++ b/backend/tests/test_channel_details_refresh.py @@ -0,0 +1,107 @@ +"""A channel's metadata is fetched again once it is stale, not only the first time it is seen. + +Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving: +a channel that kept the metadata of the day it was first imported showed a dead banner forever. The +subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and +leave fresh metadata alone (each refresh spends quota). DB lane. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from app.models import Channel, Subscription +from app.routes import channels +from app.sync import subscriptions +from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions + +NOW = datetime.now(timezone.utc) +FRESH = NOW - timedelta(days=1) +STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1) + + +def _item(cid: str) -> dict: + return { + "id": cid, + "snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"}, + "brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}}, + } + + +def _youtube(asked: list[list[str]], subscribed: list[str]): + class _Client: + def __init__(self, db, user): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def iter_subscriptions(self): + for cid in subscribed: + yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"} + + def get_channels(self, ids): + asked.append(sorted(ids)) + return [_item(cid) for cid in ids] + + return _Client + + +def _channel(db, cid: str, synced: datetime | None) -> None: + db.add( + Channel( + id=cid, + title=cid, + banner_url=f"https://yt3.example/{cid}-old", + published_at=datetime(2010, 1, 1, tzinfo=timezone.utc), + details_synced_at=synced, + ) + ) + + +def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones( + db, make_user, monkeypatch +): + user = make_user() + _channel(db, "UCfresh", FRESH) + _channel(db, "UCstale", STALE) + _channel(db, "UCnever", None) + db.flush() + for cid in ("UCfresh", "UCstale", "UCnever"): + db.add(Subscription(user_id=user.id, channel_id=cid)) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr( + subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"]) + ) + + result = import_subscriptions(db, user) + + assert asked == [["UCnever", "UCstale"]] + assert result["channels_detailed"] == 2 + stale = db.get(Channel, "UCstale") + assert stale is not None + assert stale.banner_url == "https://yt3.example/UCstale-new" + assert stale.details_synced_at is not None and stale.details_synced_at > STALE + fresh = db.get(Channel, "UCfresh") + assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old" + + +@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)]) +def test_the_channel_page_refreshes_a_stale_channel_only( + db, make_user, monkeypatch, synced, refreshed +): + user = make_user() + _channel(db, "UCpage", synced) + db.commit() + asked: list[list[str]] = [] + monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [])) + + body = channels.channel_detail("UCpage", user=user, db=db) + + assert asked == ([["UCpage"]] if refreshed else []) + suffix = "new" if refreshed else "old" + assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}" From 8e1004fe6fbf49cae05eec6689eca2db9cbbea98 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 26 Sep 2026 21:45:46 +0200 Subject: [PATCH 2/2] fix(channel): load the banner without a referrer Google's image CDN refuses a banner request that carries a Referer from our origin - every time, not only under load (measured: 6/6 refused with it, 8/8 served without). The banner now sends none, as ui/Avatar does for the same CDN; the comments that blamed a transient burst throttle are corrected. --- .../ChannelPage.banner.dom.test.tsx | 89 +++++++++++++++++++ frontend/src/components/ChannelPage.tsx | 11 +-- frontend/src/components/ui/Avatar.tsx | 9 +- 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 frontend/src/components/ChannelPage.banner.dom.test.tsx diff --git a/frontend/src/components/ChannelPage.banner.dom.test.tsx b/frontend/src/components/ChannelPage.banner.dom.test.tsx new file mode 100644 index 00000000..5d886379 --- /dev/null +++ b/frontend/src/components/ChannelPage.banner.dom.test.tsx @@ -0,0 +1,89 @@ +// @vitest-environment jsdom +// +// The channel banner is served from Google's image CDN (yt3.googleusercontent.com), which refuses a +// request that carries a Referer from our origin — every time, not only under load (measured +// 2026-09-26: 6/6 refused with the page's referrer or its bare origin, 8/8 served without one). So the +// banner must send none, as ui/Avatar does for the same CDN; with a referrer the banner never +// showed and the page fell back to the blank bar. +import type { ReactNode } from "react"; +import { describe, it, expect, vi, beforeAll, afterEach } from "vitest"; +import { render, cleanup, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ t: (key: string) => key, i18n: { language: "en" } }), +})); +vi.mock("./navigationContext", () => ({ + useNavigationActions: () => ({ closeChannel: vi.fn(), openChannel: vi.fn() }), +})); +vi.mock("./confirmContext", () => ({ useConfirm: () => async () => true })); +vi.mock("./feedFiltersContext", () => ({ + useFeedFilters: () => ({ includeNormal: true, includeShorts: false, includeLive: false }), +})); +vi.mock("./Feed", () => ({ default: () => null })); +vi.mock("./ChannelAbout", () => ({ default: () => null })); +// The page's chrome portals into the shell's fixed toolbar band, which this test has no shell for. +vi.mock("./ui/PageShell", async (importOriginal) => ({ + ...(await importOriginal()), + PageToolbar: ({ children }: { children?: ReactNode }) => <>{children}, +})); + +import ChannelPage from "./ChannelPage"; +import { api, type ChannelDetail, type Me } from "../lib/api"; + +beforeAll(() => { + class Stub { + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } + globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver; + globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver; +}); +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +const CH = "UCqoAEDirJPjEUFcF2FklnBA"; + +describe("ChannelPage — the banner", () => { + it("is requested without a referrer, which Google's image CDN refuses", async () => { + vi.spyOn(api, "channelDetail").mockResolvedValue({ + id: CH, + title: "StarTalk", + banner_url: "https://yt3.googleusercontent.com/banner", + subscribed: true, + blocked: false, + hidden: false, + deep_requested: false, + backfill_done: true, + stored_videos: 10, + } as unknown as ChannelDetail); + vi.spyOn(api, "tags").mockResolvedValue([] as never); + const { container } = render( + + {}} + onShowInFeed={() => {}} + /> + , + ); + const banner = await waitFor(() => { + const img = container.querySelector('img[src^="https://yt3."]'); + expect(img).not.toBeNull(); + return img!; + }); + expect(banner.getAttribute("referrerpolicy")).toBe("no-referrer"); + }); +}); diff --git a/frontend/src/components/ChannelPage.tsx b/frontend/src/components/ChannelPage.tsx index 0075a98a..8fc69a34 100644 --- a/frontend/src/components/ChannelPage.tsx +++ b/frontend/src/components/ChannelPage.tsx @@ -79,11 +79,11 @@ export default function ChannelPage({ // where the channelScoped Feed portals ITS toolbar (via the overridden PageToolbarSlot below), so // the toolbar lands under the tabs inside the fixed band rather than in the scroller. const [chromeSlot, setChromeSlot] = useState(null); - // The banner URL is valid, but googleusercontent intermittently throttles it when the channel - // page fires the banner + avatar + ~16 thumbnails in one burst — the request is dropped and the - // browser then NEGATIVE-CACHES the miss, so the banner stays broken even though a retry would - // succeed. So on error we retry a few times with a cache-buster (forcing a fresh request), and - // only fall back to the blank bar if it keeps failing (a genuinely dead URL — rare). + // The banner comes from Google's image CDN, which refuses any request carrying a Referer from our + // origin — every time, not only in a burst (measured 2026-09-26) — so the sends none + // (`referrerPolicy`, as ui/Avatar). A load can still drop, and the browser NEGATIVE-CACHES the miss, + // so on error we retry a few times with a cache-buster (forcing a fresh request), and only fall + // back to the blank bar if it keeps failing (a genuinely dead URL — rare). const MAX_BANNER_RETRIES = 3; const [bannerAttempt, setBannerAttempt] = useState(0); @@ -335,6 +335,7 @@ export default function ChannelPage({ // browser's negative cache (googleusercontent accepts an extra query param). key={bannerAttempt} src={`${ch.banner_url}=w1707${bannerAttempt ? `?r=${bannerAttempt}` : ""}`} + referrerPolicy="no-referrer" alt="" className="w-full h-full object-cover object-center" onError={() => { diff --git a/frontend/src/components/ui/Avatar.tsx b/frontend/src/components/ui/Avatar.tsx index 7172df31..d81f7621 100644 --- a/frontend/src/components/ui/Avatar.tsx +++ b/frontend/src/components/ui/Avatar.tsx @@ -3,10 +3,11 @@ import { useState } from "react"; // Avatar image with a graceful fallback. // // Uses referrerPolicy="no-referrer" because Google's avatar CDNs -// (yt3.ggpht.com / lh3.googleusercontent.com) rate-limit bursts of -// referrer-bearing requests — when a feed page fires dozens of avatar loads at -// once, a random subset gets throttled (HTTP 429) and the browser shows its -// broken-image icon. Dropping the referrer makes those CDNs serve the images. +// (yt3.ggpht.com / lh3.googleusercontent.com) refuse referrer-bearing requests +// from our origin — yt3 refuses even a single load, not only a burst (measured +// 2026-09-26) — and the browser shows its broken-image icon. Dropping the +// referrer makes those CDNs serve the images. The channel banner, from the same +// CDN, does the same (ChannelPage). // If a load still fails we render a neutral placeholder (optionally an initial) // instead of the broken-image icon. export default function Avatar({