Files
siftlode/frontend/src/components/ChannelPage.tsx
T
peter e3ee1baa94 fix(backfill): finish large channels in one immediate full-history run
The user-triggered "get full history" (backfill_channel_now) capped at
max_pages=60 = 3000 videos. A channel just over that (BBC Earth: 3403
uploads = 69 pages) stopped one page short, so its tail dribbled in over
the scheduler's paced 10-minute ticks (250 videos/tick) and the button
spinner looked stuck for ~18 minutes with no visible progress.

- Raise the immediate cap to max_pages=400 (20 000 videos) so any
  realistic channel completes in the single background run.
- Make a high cap safe: backfill_channel_deep now re-checks the shared
  quota before each page and yields at the backfill reserve (previously
  quota was checked once by the caller before the whole run), and commits
  the advanced cursor after every page so a restart resumes instead of
  re-fetching — and re-charging quota for — pages already stored.
- Channel page: while loading, the button shows the live stored/total
  count and a tooltip explaining the tail finishes in the background, so a
  very large channel's paced remainder reads as progress, not a hang.

Adds deep-backfill paging tests (per-page cursor commit + resume, and the
per-page quota yield).
2026-08-15 17:13:52 +02:00

562 lines
26 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useCallback, useEffect, useRef, useState } from "react";
import { useTranslation } from "react-i18next";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { useLiveQuery } from "../lib/useLiveQuery";
import { channelDetailPollMs } from "../lib/channelDetailPoll";
import { qk } from "../lib/queryKeys";
import { useNavigationActions } from "./NavigationProvider";
import { PriorityStepper } from "./PriorityStepper";
import { useChannelPriority } from "../lib/useChannelPriority";
import {
ArrowLeft,
Ban,
Check,
ExternalLink,
History,
Loader2,
Plus,
RefreshCw,
SlidersHorizontal,
} from "lucide-react";
import Avatar from "./Avatar";
import Tabs from "./Tabs";
import Feed from "./Feed";
import ChannelAboutContent from "./ChannelAbout";
import { PageToolbar, PageToolbarSlot } from "./PageShell";
import { useFeedFilters } from "./FeedFiltersProvider";
import { useConfirm } from "./ConfirmProvider";
import { notify } from "../lib/notifications";
import { notifyYouTubeActionError } from "../lib/youtubeErrors";
import { api, type FeedFilters, type Me, type ChannelDetail } from "../lib/api";
import type { FeedView } from "../lib/feedView";
import { channelYouTubeUrl, formatViews } from "../lib/format";
import { ChannelTagsControl } from "./ChannelTags";
import { useChannelTagActions } from "../lib/useChannelTagActions";
// A dedicated channel page: an "About"-style header (banner, avatar, stats, subscribe) over the
// channel's videos (the catalog filtered to this channel). For an un-subscribed channel it
// auto-ingests recent uploads in the background ("explore") so they're browsable immediately,
// and offers "load more" to page deeper — without permanently polluting everyone's library
// (the backend keeps explored videos private to this user until they subscribe).
export default function ChannelPage({
channelId,
initialName,
me,
view,
setView,
onShowInFeed,
}: {
channelId: string;
initialName?: string;
me: Me;
// The feed's view pref, passed straight through to the Feed this page renders — so its
// toolbar carries the same switcher and a change here is the same account-wide choice.
view: FeedView;
setView: (v: FeedView) => void;
// Narrow the real feed to this channel and go there. Offered only for a channel you're
// subscribed to: the feed defaults to scope "my", so filtering it to a channel you don't
// follow would just land you on an empty page.
onShowInFeed: () => void;
}) {
const { t, i18n } = useTranslation();
const { closeChannel } = useNavigationActions();
const qc = useQueryClient();
const confirm = useConfirm();
// Seed the channel view's content-type prefs from the global feed (read once at mount). The
// channel page is its own filter scope, but WHICH content types the user wants to see (normal /
// shorts / live-upcoming) is a genuine preference, not part of "this channel's catalog" — so we
// inherit it instead of forcing Live/Upcoming on. The channel toolbar can still toggle these.
const globalFilters = useFeedFilters();
const [tab, setTab] = useState<"videos" | "about">("videos");
// The whole channel chrome (banner, identity, tabs, toolbar) is fixed: it portals into the shell's
// fixed band so only the video grid scrolls. This node sits at the bottom of that chrome and is
// where the channelScoped Feed portals ITS toolbar (via the overridden PageToolbarSlot below), so
// the toolbar lands under the tabs inside the fixed band rather than in the scroller.
const [chromeSlot, setChromeSlot] = useState<HTMLDivElement | null>(null);
// The banner URL is valid, but googleusercontent intermittently throttles it when the channel
// page fires the banner + avatar + ~16 thumbnails in one burst — the request is dropped and the
// browser then NEGATIVE-CACHES the miss, so the banner stays broken even though a retry would
// succeed. So on error we retry a few times with a cache-buster (forcing a fresh request), and
// only fall back to the blank bar if it keeps failing (a genuinely dead URL — rare).
const MAX_BANNER_RETRIES = 3;
const [bannerAttempt, setBannerAttempt] = useState(0);
// A LIVE query: while THIS user's full-history backfill is running, poll so the stored count (and
// the N/S/L breakdown) climb on their own — the immediate backfill fills it in seconds, and the
// scheduler tops up any tail on its ~10-minute ticks — then stop polling once the back-catalogue is
// in. Same queryKey as before, so mutation invalidations still hit.
const detail = useLiveQuery(qk.channel(channelId), () => api.channelDetail(channelId), {
intervalMs: channelDetailPollMs,
});
const ch = detail.data;
useEffect(() => setBannerAttempt(0), [ch?.banner_url]); // fresh channel/banner → try again
// Background auto-ingest ("explore") of an un-subscribed channel's uploads.
const [nextToken, setNextToken] = useState<string | null | undefined>(undefined);
const [exploring, setExploring] = useState(false);
const autoTried = useRef<string | null>(null);
const runExplore = useCallback(
(token?: string | null) => {
setExploring(true);
return api
.exploreChannel(channelId, token ?? undefined)
.then((r) => {
setNextToken(r.next_page_token);
qc.invalidateQueries({ queryKey: qk.feed() });
// Refetch the detail so `explored` flips true → the "Exploring" badge shows on the
// first visit (the GET that drove this render predated the explore that just ran).
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
})
.finally(() => setExploring(false));
},
[channelId, qc],
);
// First visit of an un-subscribed channel → ingest its recent uploads. Skipped for the demo
// account (explore spends shared quota) and when videos are already present (a revisit, or a
// subscribed channel whose uploads the scheduler backfills).
useEffect(() => {
if (!ch || me.is_demo) return;
if (autoTried.current === channelId) return;
if (ch.subscribed || ch.blocked) return;
if (ch.explored && ch.stored_videos > 0) return;
autoTried.current = channelId;
runExplore().catch(() => {});
}, [ch, channelId, me.is_demo, runExplore]);
const block = useMutation({
mutationFn: () => (ch?.blocked ? api.unblockChannel(channelId) : api.blockChannel(channelId)),
onSuccess: () => {
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
qc.invalidateQueries({ queryKey: qk.feed() });
qc.invalidateQueries({ queryKey: qk.blockedChannels() });
},
// No onError: block/unblock is a local DB op (not a YouTube write, so no 403 "connect" case),
// and its 400/409/422/500 are already surfaced by the global error modal (see api.ts req) —
// a caller toast here would just double it.
});
const subscribe = useMutation({
mutationFn: () => api.subscribeChannel(channelId),
onSuccess: () => {
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
qc.invalidateQueries({ queryKey: qk.feed() });
qc.invalidateQueries({ queryKey: qk.channels() });
// Match ChannelDiscovery: the channel leaves discovery and the manager's stats move.
qc.invalidateQueries({ queryKey: qk.myStatus() });
qc.invalidateQueries({ queryKey: qk.discoveredChannels() });
notify({ level: "info", message: t("channel.subscribed", { name: ch?.title ?? "" }) });
},
// A 403 (missing write scope) otherwise fails silently on the channel page (no wizard handle
// here, so no Connect button — but the user at least sees why it didn't work).
onError: (err) => notifyYouTubeActionError(err, t("channels.discovery.subscribeFailed")),
});
const unsubscribe = useMutation({
mutationFn: () => api.unsubscribeChannel(channelId),
onSuccess: () => {
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
qc.invalidateQueries({ queryKey: qk.feed() });
qc.invalidateQueries({ queryKey: qk.channels() });
},
onError: (err) => notifyYouTubeActionError(err, t("channels.notify.unsubscribeFailed")),
});
// "Get full history": the catalog only holds the recent-backfill/RSS subset of a channel's
// uploads (video_count is YouTube's raw total), so flipping deep_requested queues the full
// back-catalog pager. Same trigger the channel manager uses.
const deepBackfill = useMutation({
mutationFn: () => api.updateChannel(channelId, { deep_requested: true }),
onSuccess: () => {
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
qc.invalidateQueries({ queryKey: qk.feed() });
qc.invalidateQueries({ queryKey: qk.channels() });
qc.invalidateQueries({ queryKey: qk.myStatus() });
notify({ level: "info", message: t("channel.fullHistoryQueued") });
},
onError: (err) => notifyYouTubeActionError(err, t("channel.fullHistoryFailed")),
});
// Feed priority, editable here with the same stepper + debounced-flush the manager uses (shared
// hook so the two can't drift). Optimistic on this channel's detail cache; a success re-sorts the
// manager list + feed, a failure reverts by refetching the detail.
const { setPriority, bumpPriority } = useChannelPriority({
read: (id) => qc.getQueryData<ChannelDetail>(qk.channel(id))?.priority,
optimistic: (id, next) =>
qc.setQueryData<ChannelDetail>(qk.channel(id), (old) =>
old ? { ...old, priority: next } : old,
),
onSuccess: () => {
qc.invalidateQueries({ queryKey: qk.channels() });
qc.invalidateQueries({ queryKey: qk.feed() });
},
onError: (id) => qc.invalidateQueries({ queryKey: qk.channel(id) }),
});
// Personal tags, editable right here (the same control + flow the channel manager uses). Optimistic
// on this channel's detail cache; a failed attach/detach reverts by refetching, and a success
// invalidates the manager list so a tag added here shows there too.
// Only fetch the tag catalog when the editor will actually render (subscribed, non-demo) — an
// unsubscribed/demo channel page never shows the control, so the request would be thrown away.
const tagsQuery = useQuery({
queryKey: qk.tags(),
queryFn: api.tags,
enabled: !!ch?.subscribed && !me.is_demo,
});
const userTags = (tagsQuery.data ?? []).filter((tg) => !tg.system);
const { toggleTag, createAndAttachTag } = useChannelTagActions({
optimisticToggle: (id, tagId, attach) =>
qc.setQueryData<ChannelDetail>(qk.channel(id), (old) =>
old
? {
...old,
tag_ids: attach ? [...old.tag_ids, tagId] : old.tag_ids.filter((x) => x !== tagId),
}
: old,
),
onSuccess: () => qc.invalidateQueries({ queryKey: qk.channels() }),
onError: (id) => qc.invalidateQueries({ queryKey: qk.channel(id) }),
});
const onUnsubscribe = async () => {
const ok = await confirm({
title: t("channel.unsubTitle"),
message: t("channel.unsubBody", { name: ch?.title ?? channelId }),
confirmLabel: t("channel.unsubscribe"),
danger: true,
});
if (ok) unsubscribe.mutate();
};
// Channel-scoped feed: the whole catalog filtered to this channel (scope=all + source=all so
// the explored, per-user-private videos show — the backend gates them to this explorer).
const [filters, setFilters] = useState<FeedFilters>(() => ({
tags: [],
tagMode: "or",
q: "",
sort: "newest",
scope: "all",
librarySource: "all",
// Content-type prefs inherited from the global feed (seeded once) — no longer forced.
includeNormal: globalFilters.includeNormal,
includeShorts: globalFilters.includeShorts,
includeLive: globalFilters.includeLive,
show: "all",
channelIds: [channelId],
}));
const name = ch?.title ?? initialName ?? channelId;
const ytUrl = channelYouTubeUrl(channelId, ch?.handle);
const joined = ch?.published_at
? new Date(ch.published_at).toLocaleDateString(i18n.language, {
year: "numeric",
month: "short",
})
: null;
// Handle + stats on one compact meta line under the name (saves the separate stats row).
const metaParts = [
ch?.handle ? `@${ch.handle.replace(/^@/, "")}` : null,
ch?.subscriber_count != null
? t("channel.subscribers", { formatted: formatViews(ch.subscriber_count) })
: null,
// Show "stored / total" when we only hold a subset of a subscribed channel's uploads, so the
// gap between the browsable videos and YouTube's raw count is honest rather than looking like
// missing data (the "get full history" button pulls the rest).
ch?.video_count != null
? ch.subscribed && !ch.backfill_done && ch.stored_videos < ch.video_count
? t("channel.videoCountStored", { stored: ch.stored_videos, total: ch.video_count })
: t("channel.videoCount", { count: ch.video_count })
: null,
ch?.total_view_count != null
? t("channel.totalViews", { formatted: formatViews(ch.total_view_count) })
: null,
// Per-type breakdown (R14 moved this off the manager table's N/S/L column). Only meaningful
// once we actually hold some of the channel's videos.
ch && ch.stored_videos > 0
? t("channel.videoTypes", {
normal: ch.count_normal,
short: ch.count_short,
live: ch.count_live,
})
: null,
joined ? t("channel.joined", { date: joined }) : null,
].filter(Boolean) as string[];
// The channel page owns no scroller of its own (PageShell provides the one app scroller). Its
// whole chrome is fixed — wrapped in <PageToolbar> it portals into the shell's fixed band, so the
// banner/identity/tabs/toolbar stay put and only the video grid scrolls.
return (
<>
<PageToolbar>
{/* Banner + back — OPTION C: inset, rounded card (not full-bleed) */}
<div className="relative px-4 sm:px-6 pt-3">
{ch?.banner_url && bannerAttempt <= MAX_BANNER_RETRIES ? (
// Match YouTube's banner: the stored bannerExternalUrl is the full 16:9 template at a
// low default size, so request a crisp wide version (=w1707) and object-cover the
// desktop "safe area" — the centre 2560×423 (~6:1) band.
<div
className="w-full overflow-hidden rounded-2xl bg-surface"
style={{ aspectRatio: "2560 / 423", maxHeight: "150px" }}
>
<img
// key forces a fresh <img> per attempt; the ?r= cache-buster on a retry bypasses the
// browser's negative cache (googleusercontent accepts an extra query param).
key={bannerAttempt}
src={`${ch.banner_url}=w1707${bannerAttempt ? `?r=${bannerAttempt}` : ""}`}
alt=""
className="w-full h-full object-cover object-center"
onError={() => {
if (bannerAttempt <= MAX_BANNER_RETRIES) {
window.setTimeout(
() => setBannerAttempt((a) => a + 1),
500 * (bannerAttempt + 1),
);
}
}}
/>
</div>
) : (
// No banner (or a dead one): a neutral bar the SAME height as a real banner, so the
// overlapping avatar + the Back button have the same room and don't collide (a short bar
// let the -mt-8 avatar ride up into the Back button).
<div
className="w-full rounded-2xl bg-surface"
style={{ aspectRatio: "2560 / 423", maxHeight: "150px" }}
/>
)}
<button
data-testid="channel-back"
onClick={closeChannel}
className="absolute top-5 left-7 sm:left-9 z-base inline-flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm bg-black/45 text-white hover:bg-black/65 backdrop-blur-xs"
>
<ArrowLeft className="w-4 h-4" />
{t("channel.back")}
</button>
</div>
<div className="px-4 sm:px-6">
{/* Identity + actions. relative+z-base so the avatar that overlaps up into the banner
paints ABOVE it (the banner's positioned container would otherwise cover it). */}
<div className="relative z-base flex items-end gap-4 -mt-8">
<Avatar
src={ch?.thumbnail_url ?? null}
fallback={name}
className="w-20 h-20 rounded-full border-4 border-bg shrink-0 bg-bg"
/>
<div className="flex-1 min-w-0 pb-1">
<div className="flex items-center gap-2 flex-wrap">
<h1 data-testid="channel-title" className="text-xl font-semibold truncate">
{name}
</h1>
{ch?.blocked ? (
<span className="text-xs px-2 py-0.5 rounded-full bg-danger/15 text-danger">
{t("channel.blockedBadge")}
</span>
) : (
ch?.explored &&
!ch?.subscribed && (
<span className="text-xs px-2 py-0.5 rounded-full bg-warn/15 text-warn">
{t("channel.exploringBadge")}
</span>
)
)}
</div>
<div className="text-sm text-muted mt-0.5 flex flex-wrap gap-x-1.5 gap-y-0.5">
{metaParts.map((part, i) => (
<span key={i}>
{i > 0 && <span className="mr-1.5">·</span>}
{part}
</span>
))}
</div>
</div>
</div>
{exploring && (
<div className="flex items-center gap-2 text-xs text-muted mt-2">
<Loader2 className="w-3.5 h-3.5 animate-spin" />
{t("channel.loadingVideos")}
</div>
)}
{/* Tabs — the channel actions ride along here rather than off in the header's top-right
corner: there are only ever two tabs, so there's room, and it keeps the things you click
within a few pixels of each other instead of across the page. */}
<div className="flex items-center gap-2 mt-3 border-b border-border pb-2">
<Tabs
className="flex gap-1"
active={tab}
onChange={(id) => setTab(id as "videos" | "about")}
tabs={[
{ id: "videos", label: t("channel.tabVideos") },
{ id: "about", label: t("channel.tabAbout") },
]}
/>
<div className="ml-2 flex items-center gap-2">
<a
href={ytUrl}
target="_blank"
rel="noreferrer"
title={t("channels.row.openOnYouTube")}
aria-label={t("channels.row.openOnYouTube")}
className="inline-flex items-center justify-center w-9 h-9 rounded-lg text-muted hover:text-fg border border-border"
>
<ExternalLink className="w-4 h-4" />
</a>
{ch?.subscribed && (
<button
onClick={onShowInFeed}
title={t("channel.showInFeed")}
aria-label={t("channel.showInFeed")}
className="inline-flex items-center justify-center w-9 h-9 rounded-lg text-muted hover:text-accent border border-border hover:border-accent transition"
>
<SlidersHorizontal className="w-4 h-4" />
</button>
)}
{ch?.subscribed && !me.is_demo && (
<div className="inline-flex items-center gap-1.5 h-9 px-2.5 rounded-lg border border-border">
<span className="text-xs text-muted">{t("channels.cols.priority")}</span>
<PriorityStepper
priority={ch.priority}
onPriority={(d) => bumpPriority(channelId, d)}
onSet={(v) => setPriority(channelId, v)}
/>
</div>
)}
{/* Only the recent subset is stored → offer to pull the channel's full back-catalogue.
Once queued (deep_requested) it shows as loading; hidden once the backfill is done.
While loading, the label carries the live stored/total count (the detail poll refreshes
it) so a large channel's paced tail reads as visible progress rather than a stuck
spinner. */}
{ch?.subscribed && !ch.backfill_done && !me.is_demo && (
<button
onClick={() => !ch.deep_requested && deepBackfill.mutate()}
disabled={ch.deep_requested || deepBackfill.isPending}
title={
ch.deep_requested
? t("channel.fullHistoryLoadingHint")
: t("channel.fullHistory")
}
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-accent hover:border-accent disabled:opacity-60 disabled:hover:text-muted disabled:hover:border-border transition"
>
{ch.deep_requested ? (
<Loader2 className="w-4 h-4 animate-spin" />
) : (
<History className="w-4 h-4" />
)}
<span className="hidden sm:inline whitespace-nowrap">
{!ch.deep_requested
? t("channel.fullHistory")
: ch.video_count != null && ch.stored_videos < ch.video_count
? t("channel.fullHistoryProgress", {
stored: ch.stored_videos,
total: ch.video_count,
})
: t("channel.fullHistoryLoading")}
</span>
</button>
)}
{!me.is_demo && (
<button
onClick={() => block.mutate()}
disabled={block.isPending}
title={ch?.blocked ? t("channel.unblock") : t("channel.block")}
aria-label={ch?.blocked ? t("channel.unblock") : t("channel.block")}
className={`inline-flex items-center justify-center w-9 h-9 rounded-lg border disabled:opacity-50 transition ${
ch?.blocked
? "border-danger text-danger"
: "border-border text-muted hover:text-danger hover:border-danger"
}`}
>
<Ban className="w-4 h-4" />
</button>
)}
{!me.is_demo &&
!ch?.blocked &&
(ch?.subscribed ? (
<button
onClick={onUnsubscribe}
disabled={unsubscribe.isPending}
title={t("channel.unsubscribe")}
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-fg disabled:opacity-50"
>
<Check className="w-4 h-4" />
{t("channel.subscribedState")}
</button>
) : (
<button
onClick={() => subscribe.mutate()}
disabled={subscribe.isPending}
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm btn-primary disabled:opacity-50"
>
<Plus className="w-4 h-4" />
{t("channel.subscribe")}
</button>
))}
{/* Tagging is a subscription-scoped action (attach_tag → _user_subscription 404s
otherwise), and personal tags are for your subscriptions — so only offer it on a
channel you actually follow. */}
{ch?.subscribed && !me.is_demo && (
<ChannelTagsControl
userTags={userTags}
attachedIds={ch.tag_ids}
onToggle={(tagId) => toggleTag(channelId, tagId, ch.tag_ids.includes(tagId))}
onCreate={(name) => createAndAttachTag(channelId, name)}
// The header row has free width — show tags inline instead of collapsing to "+N".
bounded={false}
/>
)}
</div>
</div>
</div>
{/* The channelScoped Feed portals its toolbar into this node — the bottom of the fixed chrome,
just under the tabs. */}
<div ref={setChromeSlot} />
</PageToolbar>
{/* Tab content — the feed's toolbar portals up into the fixed chrome via this slot override;
only the grid (and About) render here, in the scroller. */}
<PageToolbarSlot.Provider value={chromeSlot}>
{tab === "videos" ? (
<>
<Feed
filters={filters}
setFilters={setFilters}
view={view}
setView={setView}
canRead={me.can_read}
isDemo={me.is_demo}
ytSearch={null}
onYtSearch={() => {}}
onExitYtSearch={() => {}}
channelScoped
/>
{!ch?.subscribed && nextToken && (
<div className="flex justify-center pb-6">
<button
onClick={() => runExplore(nextToken)}
disabled={exploring}
className="inline-flex items-center gap-2 px-4 py-2 rounded-lg text-sm border border-border hover:bg-surface disabled:opacity-50"
>
{exploring ? (
<Loader2 className="w-4 h-4 animate-spin" />
) : (
<RefreshCw className="w-4 h-4" />
)}
{t("channel.loadMore")}
</button>
</div>
)}
</>
) : (
<div className="px-4 sm:px-6 py-4 max-w-3xl">
{ch ? (
<ChannelAboutContent ch={ch} />
) : (
<p className="text-sm text-muted">{t("channel.noDescription")}</p>
)}
</div>
)}
</PageToolbarSlot.Provider>
</>
);
}