diff --git a/VERSION b/VERSION index 1dd1c715..aa4ceb30 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.64.0 \ No newline at end of file +0.65.0 \ No newline at end of file diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index f3626d55..70e62c69 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -4,7 +4,7 @@ hidden, tags) and per-channel sync state. Read + local-write only — YouTube-si import logging from datetime import datetime, timezone -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from pydantic import BaseModel from sqlalchemy import func, select, update from sqlalchemy.orm import Session @@ -28,7 +28,7 @@ from app.models import ( from app.routes._common import owned_or_404 from app.routes.admin import admin_user from app.sync.explore import explore_ingest_page -from app.sync.runner import run_recent_backfill +from app.sync.runner import backfill_channel_now, run_recent_backfill from app.sync.subscriptions import apply_channel_details from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError @@ -50,20 +50,19 @@ def list_channels( ).all() # Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last - # upload, total duration, and a Normal/Shorts/Live breakdown. Cheap — a single grouped scan - # (~35 ms over the full 233k-row catalog on the dev DB), so computed on read rather than + # upload, total duration. Cheap — a single grouped scan, so computed on read rather than # denormalised (measure-first: reads aren't frequent/hot enough to warrant cached columns). + # (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer + # computed here — the manager table dropped its N/S/L column.) channel_ids = [c.id for _, c in rows] agg: dict[str, dict] = {} if channel_ids: - for cid, n, last_up, total_dur, shorts, live in db.execute( + for cid, n, last_up, total_dur in db.execute( select( Video.channel_id, func.count(Video.id), func.max(Video.published_at), func.coalesce(func.sum(Video.duration_seconds), 0), - func.count(Video.id).filter(Video.is_short.is_(True)), - func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)), ) .where(Video.channel_id.in_(channel_ids)) .group_by(Video.channel_id) @@ -72,9 +71,6 @@ def list_channels( "stored": n, "last_video_at": iso(last_up), "total_duration_seconds": int(total_dur or 0), - "count_short": int(shorts), - "count_live": int(live), - "count_normal": max(0, int(n) - int(shorts) - int(live)), } # Channels already in the *global* deep-backfill queue — i.e. at least one user (any @@ -108,9 +104,6 @@ def list_channels( "stored_videos": (agg.get(ch.id) or {}).get("stored", 0), "last_video_at": (agg.get(ch.id) or {}).get("last_video_at"), "total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0), - "count_normal": (agg.get(ch.id) or {}).get("count_normal", 0), - "count_short": (agg.get(ch.id) or {}).get("count_short", 0), - "count_live": (agg.get(ch.id) or {}).get("count_live", 0), "priority": sub.priority, "hidden": sub.hidden, "deep_requested": sub.deep_requested, @@ -229,6 +222,9 @@ def _channel_detail_dict( explored: bool, blocked: bool, stored: int, + count_normal: int, + count_short: int, + count_live: int, deep_requested: bool, priority: int, tag_ids: list[int], @@ -253,6 +249,11 @@ def _channel_detail_dict( "explored": explored, "blocked": blocked, "stored_videos": stored, + # Per-type breakdown (Normal / Shorts / Live) — R14 moved it off the manager table onto the + # channel page. Same aggregation the manager list uses (count_normal = stored − shorts − live). + "count_normal": count_normal, + "count_short": count_short, + "count_live": count_live, # Deep-backfill state, so the channel page can offer "get full history" when the catalog # holds only the recent-backfill/RSS subset (video_count is YouTube's raw total). "deep_requested": deep_requested, @@ -304,7 +305,19 @@ def channel_detail( ) ).first() is not None blocked = _is_blocked(db, user, channel_id) - stored = db.scalar(select(func.count(Video.id)).where(Video.channel_id == channel_id)) or 0 + # Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager + # list), so the channel page can show the per-type breakdown R14 moved off the table. + n_all, n_short, n_live = db.execute( + select( + func.count(Video.id), + func.count(Video.id).filter(Video.is_short.is_(True)), + func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)), + ).where(Video.channel_id == channel_id) + ).one() + stored = int(n_all or 0) + count_short = int(n_short or 0) + count_live = int(n_live or 0) + count_normal = max(0, stored - count_short - count_live) tag_ids = list( db.scalars( select(ChannelTag.tag_id).where( @@ -318,6 +331,9 @@ def channel_detail( explored=explored, blocked=blocked, stored=int(stored), + count_normal=count_normal, + count_short=count_short, + count_live=count_live, deep_requested=bool(sub.deep_requested) if sub else False, priority=int(sub.priority) if sub else 0, tag_ids=tag_ids, @@ -426,6 +442,7 @@ class ChannelUpdateIn(BaseModel): def update_channel( channel_id: str, body: ChannelUpdateIn, + background: BackgroundTasks, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: @@ -450,12 +467,18 @@ def update_channel( # Opting a channel into full history should give an immediate feed: if we haven't even # fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for - # the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep). - # (deep_turned_on is only ever True for a non-demo user — see the guard above.) + # the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.) if deep_turned_on and channel is not None and channel.recent_synced_at is None: with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT): run_recent_backfill(db, [channel], max_channels=1) + # …and kick the FULL deep backfill for this one channel immediately in the background, so the + # whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The + # request returns right away; the channel page's live poll shows the count climb. The scheduler + # still finishes a very large channel's tail on its normal runs (deep_requested stays set). + if deep_turned_on: + background.add_task(backfill_channel_now, channel_id, user.id) + return { "id": channel_id, "priority": sub.priority, diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index dca35641..8221a5fd 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu public reads. """ import logging +import threading from concurrent.futures import ThreadPoolExecutor, as_completed from sqlalchemy import select @@ -13,6 +14,7 @@ from sqlalchemy.orm import Session from app import progress, quota, sysconfig log = logging.getLogger("siftlode.sync") +from app.db import SessionLocal from app.models import Channel, OAuthToken, Subscription, User from app.sync.subscriptions import import_subscriptions from app.sync.videos import ( @@ -229,6 +231,12 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - for channel in channels: if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): break + # Claim the channel (or skip it if an interactive backfill already owns it) so the two + # never advance the same cursor concurrently — symmetric with backfill_channel_now. + with _backfill_lock: + if channel.id in _backfill_in_progress: + continue + _backfill_in_progress.add(channel.id) try: videos_added += backfill_channel_deep(db, yt, channel, max_pages) processed += 1 @@ -236,6 +244,9 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - except Exception: db.rollback() log.exception("Deep backfill failed for channel %s", channel.id) + finally: + with _backfill_lock: + _backfill_in_progress.discard(channel.id) reconciled = reconcile_full_history(db) return { "channels_processed": processed, @@ -244,6 +255,58 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - } +# Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a +# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread). +# BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page +# the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running` +# counts only the interactive path, capped so one user's burst can't drain the shared budget (the +# scheduler is already quota-paced). Single uvicorn process → one shared module state, lock suffices. +_backfill_in_progress: set[str] = set() +_immediate_running = 0 +_backfill_lock = threading.Lock() +_MAX_CONCURRENT_IMMEDIATE = 2 + + +def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 60) -> None: + """User-initiated "get full history": deep-backfill ONE channel to completion right now, in a + background task with its OWN session, so the full catalogue lands in seconds instead of over the + paced scheduler's 10-minute ticks (max_pages=5/channel/tick). Quota-guarded, capped to + _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the shared backfill budget by + opting many channels in at once (the overflow, and any very large channel's tail, is finished by + the scheduler since deep_requested stays set), and mutually excluded per-channel with the scheduler + via _backfill_in_progress. Best-effort — logs and swallows failures.""" + global _immediate_running + with _backfill_lock: + if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE: + return # already backfilling this channel, or too many interactive runs — leave it to the scheduler + _backfill_in_progress.add(channel_id) + _immediate_running += 1 + try: + with SessionLocal() as db: + user = get_service_user(db) + if user is None: + return + channel = db.get(Channel, channel_id) + if channel is None or channel.backfill_done or not channel.uploads_playlist_id: + return + if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"): + return + try: + # Attribute the spend to the requesting user under the full-backfill action, so it + # shows up correctly in the usage breakdown (not the default OTHER/system bucket). + with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL): + with YouTubeClient(db, user) as yt: + backfill_channel_deep(db, yt, channel, max_pages) + reconcile_full_history(db) + except Exception: + db.rollback() + log.exception("Immediate backfill failed for channel %s", channel_id) + finally: + with _backfill_lock: + _backfill_in_progress.discard(channel_id) + _immediate_running -= 1 + + def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict: """Rough ETA to finish full-history backfill of `channels`. Quota-bound: total estimated API units / the daily budget left for backfill. Deliberately approximate — diff --git a/backend/tests/test_channel_backfill.py b/backend/tests/test_channel_backfill.py new file mode 100644 index 00000000..8da7c72d --- /dev/null +++ b/backend/tests/test_channel_backfill.py @@ -0,0 +1,178 @@ +"""R14 backend coverage: the channel-detail Normal/Shorts/Live counts (moved off the manager table), +the update_channel -> immediate-backfill wiring (incl. the demo guard), and the interactive +"get full history" guard that keeps it from racing the scheduler's cursor or draining the budget.""" +from datetime import datetime, timezone + +from fastapi.testclient import TestClient + +import app.routes.channels as channels_route +import app.state as state +from app.auth import current_user +from app.db import get_db +from app.main import app +from app.models import Channel, Subscription, User, Video +from app.sync import runner + +_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc) + + +def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch): + # published_at set so channel_detail skips the YouTube-enrich path (no quota / network in tests). + ch = Channel(id="UC_nsl", title="N/S/L Channel", published_at=_PUB) + db.add(ch) + me = User(email="nsl@example.com") + db.add(me) + db.flush() + # DISTINCT counts (2 shorts, 1 live) so a short<->live filter/field swap can't pass: + # stored 5, short 2, live 1, normal = 5 - 2 - 1 = 2. + db.add_all( + [ + Video(id="v_norm1", channel_id=ch.id), + Video(id="v_norm2", channel_id=ch.id), + Video(id="v_short1", channel_id=ch.id, is_short=True), + Video(id="v_short2", channel_id=ch.id, is_short=True), + Video(id="v_live1", channel_id=ch.id, live_status="live"), + ] + ) + db.commit() + + # The request gate 503s an unconfigured instance; flip the process-level flag (auto-restored). + monkeypatch.setattr(state, "_configured_cache", True) + app.dependency_overrides[get_db] = lambda: db + app.dependency_overrides[current_user] = lambda: me + try: + resp = TestClient(app).get(f"/api/channels/{ch.id}") + finally: + app.dependency_overrides.clear() + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["stored_videos"] == 5 + assert body["count_short"] == 2 + assert body["count_live"] == 1 + assert body["count_normal"] == 2 + + +def _setup_channel_and_sub(db, *, email, is_demo=False): + ch = Channel( + id="UC_wire", + title="Wire", + published_at=_PUB, + recent_synced_at=_PUB, # so update_channel skips the in-request recent backfill (no network) + uploads_playlist_id="UU_wire", + ) + db.add(ch) + user = User(email=email, is_demo=is_demo) + db.add(user) + db.flush() + db.add(Subscription(user_id=user.id, channel_id=ch.id, deep_requested=False)) + db.commit() + return ch, user + + +def _patch_deep_requested(db, monkeypatch, user): + calls: list[tuple] = [] + # background.add_task runs after the response in TestClient; record what would be scheduled. + monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a)) + monkeypatch.setattr(state, "_configured_cache", True) + app.dependency_overrides[get_db] = lambda: db + app.dependency_overrides[current_user] = lambda: user + try: + resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True}) + finally: + app.dependency_overrides.clear() + return resp, calls + + +def test_deep_request_schedules_immediate_backfill_for_non_demo(db, monkeypatch): + ch, me = _setup_channel_and_sub(db, email="wire@example.com") + resp, calls = _patch_deep_requested(db, monkeypatch, me) + assert resp.status_code == 200, resp.text + assert resp.json()["deep_requested"] is True + # scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs. + assert calls == [(ch.id, me.id)] + + +def test_deep_request_does_not_schedule_immediate_backfill_for_demo(db, monkeypatch): + # The SHARED demo account must not be able to kick immediate full backfills (budget drain). + _ch, demo = _setup_channel_and_sub(db, email="demo@example.com", is_demo=True) + resp, calls = _patch_deep_requested(db, monkeypatch, demo) + assert resp.status_code == 200, resp.text + assert calls == [], "demo must not schedule an immediate backfill" + # …and deep_requested is not persisted on the shared demo subscription either. + assert resp.json()["deep_requested"] is False + + +class TestImmediateBackfillGuard: + """backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per- + channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a + burst can't drain the shared budget).""" + + def setup_method(self): + runner._backfill_in_progress.clear() + runner._immediate_running = 0 + + def teardown_method(self): + runner._backfill_in_progress.clear() + runner._immediate_running = 0 + + @staticmethod + def _boom_session(): + # Reached only AFTER the guard admits the channel; if the guard short-circuits (as it must + # in these tests) this never runs, so raising proves the short-circuit. + raise AssertionError("guard must short-circuit before opening a session") + + def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch): + runner._backfill_in_progress.add("UC_busy") + monkeypatch.setattr(runner, "SessionLocal", self._boom_session) + runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs + assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched + + def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch): + runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE + monkeypatch.setattr(runner, "SessionLocal", self._boom_session) + runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it + assert "UC_new" not in runner._backfill_in_progress + + def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch): + # A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)… + ch = Channel( + id="UC_inflight", + title="In-flight", + recent_synced_at=_PUB, + uploads_playlist_id="UU_inflight", + backfill_done=False, + ) + db.add(ch) + me = User(email="svc@example.com") + db.add(me) + db.flush() + db.add(Subscription(user_id=me.id, channel_id=ch.id, deep_requested=True)) + db.commit() + + paged: list[str] = [] + monkeypatch.setattr(runner, "get_service_user", lambda _db: me) + monkeypatch.setattr(runner.quota, "remaining_today", lambda _db: 10_000) + monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt()) + monkeypatch.setattr( + runner, "backfill_channel_deep", lambda _db, _yt, channel, _mp: paged.append(channel.id) or 0 + ) + monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 0) + + # …is skipped by run_deep_backfill while an interactive backfill owns it (no cursor race). + runner._backfill_in_progress.add(ch.id) + runner.run_deep_backfill(db) + assert paged == [], "the scheduler must not page a channel an immediate backfill is handling" + + # …and IS paged once the interactive backfill releases it. + runner._backfill_in_progress.discard(ch.id) + runner.run_deep_backfill(db) + assert paged == [ch.id] + + +class _NullYt: + def __enter__(self): + return self + + def __exit__(self, *a): + return False diff --git a/frontend/src/components/ChannelLink.tsx b/frontend/src/components/ChannelLink.tsx index 08b78dbc..2fc67126 100644 --- a/frontend/src/components/ChannelLink.tsx +++ b/frontend/src/components/ChannelLink.tsx @@ -41,12 +41,16 @@ export default function ChannelLink({ onMouseDown={(e) => { if (e.button === 1) e.preventDefault(); // suppress middle-click autoscroll }} + // title = the full name, so a truncated fluid-column name is still readable on hover. + title={name} className="text-sm font-medium truncate hover:text-accent text-left min-w-0" > {name} ) : ( - {name} + + {name} + )} api.channelDetail(channelId), + // A LIVE query: while THIS user's full-history backfill is running, poll so the stored count (and + // the N/S/L breakdown) climb on their own — the immediate backfill fills it in seconds, and the + // scheduler tops up any tail on its ~10-minute ticks — then stop polling once the back-catalogue is + // in. Same queryKey as before, so mutation invalidations still hit. + const detail = useLiveQuery(qk.channel(channelId), () => api.channelDetail(channelId), { + intervalMs: channelDetailPollMs, }); const ch = detail.data; useEffect(() => setBannerAttempt(0), [ch?.banner_url]); // fresh channel/banner → try again @@ -264,6 +269,15 @@ export default function ChannelPage({ ch?.total_view_count != null ? t("channel.totalViews", { formatted: formatViews(ch.total_view_count) }) : null, + // Per-type breakdown (R14 moved this off the manager table's N/S/L column). Only meaningful + // once we actually hold some of the channel's videos. + ch && ch.stored_videos > 0 + ? t("channel.videoTypes", { + normal: ch.count_normal, + short: ch.count_short, + live: ch.count_live, + }) + : null, joined ? t("channel.joined", { date: joined }) : null, ].filter(Boolean) as string[]; // The channel page owns no scroller of its own (PageShell provides the one app scroller). Its diff --git a/frontend/src/components/Channels.tsx b/frontend/src/components/Channels.tsx index c7566b86..366e6e71 100644 --- a/frontend/src/components/Channels.tsx +++ b/frontend/src/components/Channels.tsx @@ -7,7 +7,7 @@ import ExcludedChannelList from "./ExcludedChannelList"; import { api, type ManagedChannel } from "../lib/api"; import { notifyYouTubeActionError } from "../lib/youtubeErrors"; import { accountKey, LS } from "../lib/storage"; -import { buildChannelColumns } from "./channelColumns"; +import { buildChannelColumns, applySyncFilter } from "./channelColumns"; import { notify } from "../lib/notifications"; import Tooltip from "./Tooltip"; import DataTable from "./DataTable"; @@ -236,17 +236,22 @@ export default function Channels() { setSearch(""); }, [filtersResetToken]); - // The sync-status AND tag filters are now controlled column filters (`filters.sync` / `filters.tags`) - // shared by the top chips, the column funnels, and the card layout — no separate pre-filter. Hidden - // channels have their own tab, so they're excluded from the subscribed table (no more dimmed rows). - // Only the name search filters here; the column filters (sync/tags/priority/text funnels) are applied - // by DataTable in table mode and by filterRows below in card mode, over the same `columns`. + // Tag/priority/text filters stay controlled COLUMN filters (`filters.tags` etc.) shared by the top + // chips, the column funnels and the cards. The sync-status filter (`filters.sync`) is no longer a + // column — R14 retired the Sync column and folded its status into a ✓ charm on Stored — so it's + // applied as a pre-filter (`applySyncFilter`) over the rows fed to BOTH the table and the cards, + // still driven by the same top chips. Hidden channels have their own tab (excluded here). Only the + // name search filters at this step; the remaining column funnels run in DataTable / filterRows below. const hiddenChannels = (channelsQuery.data ?? []).filter((c) => c.hidden); const channels = (channelsQuery.data ?? []).filter((c) => !c.hidden); const q = search.trim().toLowerCase(); const visibleChannels = channels.filter( (c) => !q || `${c.title ?? ""} ${c.handle ?? ""}`.toLowerCase().includes(q), ); + // Sync-status pre-filter (top chips → `filters.sync`), applied before the table/cards see the rows. + // (Inline, not memoised: `visibleChannels` is a fresh array each render, so a useMemo keyed on it + // could never hit its cache — and applySyncFilter is a cheap single pass anyway.) + const syncedChannels = applySyncFilter(visibleChannels, filters.sync); const s = statusQuery.data; const onView = (c: ManagedChannel) => @@ -371,8 +376,8 @@ export default function Channels() { // Cards have no column headers, so apply the shared column filters (sync + any funnel set in table // mode) here via the exported predicate — the same filterRows DataTable runs internally — then sort. const sortedChannels = isCards - ? sortRows(filterRows(visibleChannels, columns, filters), columns, sort) - : visibleChannels; + ? sortRows(filterRows(syncedChannels, columns, filters), columns, sort) + : syncedChannels; const { paged: pagedCards, pagerProps, @@ -527,7 +532,7 @@ export default function Channels() { ) : layout === "table" ? ( c.id} persistKey={accountKey(LS.channelsTable) ?? undefined} @@ -539,6 +544,7 @@ export default function Channels() { onSortChange={setSort} filters={filters} onFiltersChange={setFilters} + fixedLayout emptyText={t("channels.empty")} /> ) : ( diff --git a/frontend/src/components/DataTable.fixedLayout.dom.test.tsx b/frontend/src/components/DataTable.fixedLayout.dom.test.tsx new file mode 100644 index 00000000..866d5e13 --- /dev/null +++ b/frontend/src/components/DataTable.fixedLayout.dom.test.tsx @@ -0,0 +1,44 @@ +// @vitest-environment jsdom +// +// The R14 zoom-fit foundation: DataTable's opt-in `fixedLayout` renders a and puts the +// table into `table-layout: fixed`, so columns with a width keep it and the rest (fluid text columns) +// split the remainder + truncate instead of pushing the table wider than its container. Off by +// default so existing content-sized tables are untouched. +import { describe, it, expect, afterEach, vi } from "vitest"; +import { render, cleanup } from "@testing-library/react"; +import DataTable, { type Column } from "./DataTable"; + +vi.mock("react-i18next", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, useTranslation: () => ({ t: (key: string) => key }) }; +}); + +afterEach(cleanup); + +type Row = { a: string; b: string }; +const columns: Column[] = [ + { key: "a", header: "A", width: "5rem", render: (r) => r.a }, + { key: "b", header: "B", render: (r) => r.b }, // fluid — no width +]; +const rows: Row[] = [{ a: "1", b: "hello" }]; + +describe("DataTable fixedLayout", () => { + it("renders a width-bearing colgroup and table-fixed only when fixedLayout is set", () => { + const { container } = render( + r.a} fixedLayout />, + ); + const table = container.querySelector("table"); + expect(table?.className).toContain("table-fixed"); + + const cols = container.querySelectorAll("colgroup col"); + expect(cols).toHaveLength(2); + expect((cols[0] as HTMLElement).style.width).toBe("5rem"); // fixed column keeps its width + expect((cols[1] as HTMLElement).style.width).toBe(""); // fluid column: no width, splits remainder + }); + + it("omits the colgroup and stays auto-layout by default", () => { + const { container } = render( r.a} />); + expect(container.querySelector("colgroup")).toBeNull(); + expect(container.querySelector("table")?.className).not.toContain("table-fixed"); + }); +}); diff --git a/frontend/src/components/DataTable.tsx b/frontend/src/components/DataTable.tsx index 3c781b9b..8173ce6e 100644 --- a/frontend/src/components/DataTable.tsx +++ b/frontend/src/components/DataTable.tsx @@ -60,6 +60,7 @@ export default function DataTable({ onSortChange, filters: filtersProp, onFiltersChange, + fixedLayout = false, }: { rows: T[]; columns: Column[]; @@ -96,6 +97,12 @@ export default function DataTable({ // default internal, persistKey-backed filters. filters?: FilterMap; onFiltersChange?: (next: FilterMap) => void; + // Opt into `table-layout: fixed` (default is `auto`). In fixed mode the table width == its + // container at every zoom/text-size — the columns with an explicit `width` keep it, and the rest + // (the fluid text columns) split what remains and TRUNCATE instead of pushing the table wider than + // the viewport. Off by default so existing content-sized tables (playlists, discovery, Plex) are + // untouched; the channel manager turns it on so its row never scrolls horizontally out of view. + fixedLayout?: boolean; }) { const { t } = useTranslation(); const initial = loadPersist(persistKey); @@ -224,7 +231,17 @@ export default function DataTable({ {(controlsPosition === "top" || controlsPosition === "both") && topControls} {/* Wide screens: table. The wrapper isn't clipped so header filter popovers can overflow. */}
- +
+ {/* In fixed-layout mode a colgroup drives the column widths: those with an explicit + `width` keep it, and the unspecified (fluid) columns split the remainder — so the + table never exceeds its container and the fluid cells truncate instead. */} + {fixedLayout && ( + + {columns.map((col) => ( + + ))} + + )} ): ManagedChannel => ({ + id: "c", + title: null, + handle: null, + thumbnail_url: null, + subscriber_count: null, + video_count: null, + description: null, + stored_videos: 0, + last_video_at: null, + total_duration_seconds: 0, + priority: 0, + hidden: false, + deep_requested: false, + deep_in_queue: false, + tag_ids: [], + details_synced: false, + recent_synced: false, + backfill_done: false, + ...over, +}); + +const rows = [ + ch({ id: "fully", recent_synced: true, backfill_done: true }), + ch({ id: "recentOnly", recent_synced: true, backfill_done: false }), + ch({ id: "none", recent_synced: false, backfill_done: false }), +]; + +describe("applySyncFilter", () => { + it("returns every row for the empty/all value (and unknown values)", () => { + expect(applySyncFilter(rows, "").map((r) => r.id)).toEqual(["fully", "recentOnly", "none"]); + expect(applySyncFilter(rows, undefined)).toHaveLength(3); + expect(applySyncFilter(rows, "bogus")).toHaveLength(3); + }); + + it("'fully' keeps only channels with recent uploads AND full history in", () => { + expect(applySyncFilter(rows, "fully").map((r) => r.id)).toEqual(["fully"]); + }); + + it("'needsFull' keeps every channel whose back-catalogue is not done", () => { + expect(applySyncFilter(rows, "needsFull").map((r) => r.id)).toEqual(["recentOnly", "none"]); + }); +}); diff --git a/frontend/src/components/channelColumns.tsx b/frontend/src/components/channelColumns.tsx index 856876c5..fdb4942f 100644 --- a/frontend/src/components/channelColumns.tsx +++ b/frontend/src/components/channelColumns.tsx @@ -28,68 +28,38 @@ export function subsColumn(t: TFu }; } -function SyncBadge({ ok, label, hint }: { ok: boolean; label: string; hint?: string }) { - return ( - - - {ok && } - {label} - - - ); +// A channel is "fully synced" when its recent uploads AND full back-catalogue are both in — the +// predicate the retired Sync column used, now surfaced as a ✓ charm on the Stored count and driven +// by the top sync-filter chips. +const isFullySynced = (c: ManagedChannel) => c.recent_synced && c.backfill_done; + +/** Sync-status pre-filter for the subscribed table. The Sync COLUMN is gone (R14) — its status folded + * into a charm on Stored — so the top chips' `filters.sync` value is applied here instead of via a + * column predicate. Same vocab as the old column filter: "fully" | "needsFull" | "" (all). */ +export function applySyncFilter(rows: ManagedChannel[], value: unknown): ManagedChannel[] { + if (value === "fully") return rows.filter(isFullySynced); + if (value === "needsFull") return rows.filter((c) => !c.backfill_done); + return rows; } -// Status only — the backfill *action* lives in the Actions column. When both recent and -// full history are in, collapse to a single "fully synced" chip; otherwise show what's -// present plus the missing/queued state. -function SyncCell({ c }: { c: ManagedChannel }) { +// The Stored count, with a "fully synced" ✓ charm to the left of the number when recent uploads and +// full history are both in — folding the retired Sync column's at-a-glance status into the value it +// describes. Full detail stays in the hover tooltip. +function StoredCell({ c }: { c: ManagedChannel }) { const { t } = useTranslation(); - if (c.recent_synced && c.backfill_done) { - return ( - - ); - } return ( -
- - {c.backfill_done ? ( - - ) : c.deep_requested ? ( - - ) : c.deep_in_queue ? ( - - ) : ( - + + {isFullySynced(c) && ( + + + )} -
+ {c.stored_videos.toLocaleString()} + ); } @@ -213,7 +183,7 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column c.priority, @@ -231,7 +201,7 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column ( (c.title ?? c.id).toLowerCase(), filter: { kind: "text", get: (c) => `${c.title ?? ""} ${c.handle ?? ""}` }, cardPrimary: true, - // Cap the width so a long-titled channel can't stretch the column (table-layout:auto ignores a - // cell max-width, but a max-w element INSIDE the cell bounds it — same trick AboutCell uses). - // Without this a single long name blows the column ~289→429px and overflows the centered table. + // A FLUID column (R14): no width cap. Under the manager table's fixed layout this cell takes a + // share of the leftover space and ChannelLink truncates the name to fit (full name on hover via + // title), so a long name can't push the table wider than its container at any zoom. render: (c) => ( -
+
, }, { @@ -278,17 +249,18 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column c.stored_videos, - render: (c) => ( - {c.stored_videos.toLocaleString()} - ), + // The stored count carries the "fully synced" ✓ charm (R14 folded the retired Sync column in). + render: (c) => , }, - subsColumn(t), + { ...subsColumn(t), width: "6rem" }, { key: "lastUpload", header: t("channels.cols.lastUpload"), nowrap: true, + width: "6rem", sortable: true, sortValue: (c) => (c.last_video_at ? new Date(c.last_video_at).getTime() : 0), render: (c) => {relativeTime(c.last_video_at) || "—"}, @@ -298,6 +270,7 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column c.total_duration_seconds, render: (c) => ( @@ -306,45 +279,13 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column ), }, - { - key: "types", - header: t("channels.cols.types"), - nowrap: true, - render: (c) => ( - - - {c.count_normal} / {c.count_short} / {c.count_live} - - - ), - }, - { - key: "sync", - header: t("channels.cols.sync"), - nowrap: true, - cardLabel: false, - filter: { - kind: "select", - options: [ - { value: "fully", label: t("channels.filters.fullySynced") }, - { value: "needsFull", label: t("channels.filters.needsFull") }, - ], - test: (c, v) => - v === "fully" - ? c.recent_synced && c.backfill_done - : v === "needsFull" - ? !c.backfill_done - : true, - }, - render: (c) => , - }, { key: "tags", header: t("channels.cols.tags"), cardLabel: false, // Bounded width so the cell can't widen the column and reflow the whole table; the control // fills it and collapses overflow into "+N" (see ChannelTagsControl). - width: "12rem", + width: "9rem", filter: { kind: "multi", options: userTags.map((tg) => ({ value: String(tg.id), label: tg.name })), diff --git a/frontend/src/i18n/locales/en/channel.json b/frontend/src/i18n/locales/en/channel.json index d7d7ec28..b6bab16a 100644 --- a/frontend/src/i18n/locales/en/channel.json +++ b/frontend/src/i18n/locales/en/channel.json @@ -12,6 +12,7 @@ "videoCount_other": "{{count}} videos", "videoCountStored": "{{stored}} / {{total}} videos", "totalViews": "{{formatted}} views", + "videoTypes": "{{normal}} normal / {{short}} shorts / {{live}} live", "joined": "Joined {{date}}", "loadingVideos": "Loading this channel's videos…", "tabVideos": "Videos", diff --git a/frontend/src/i18n/locales/en/channels.json b/frontend/src/i18n/locales/en/channels.json index 57cdcfda..9eb7826e 100644 --- a/frontend/src/i18n/locales/en/channels.json +++ b/frontend/src/i18n/locales/en/channels.json @@ -68,9 +68,6 @@ "subs": "Subscribers", "lastUpload": "Last upload", "length": "Length", - "types": "N/S/L", - "typesHint": "Videos by type: Normal / Shorts / Live", - "sync": "Sync", "tags": "Tags", "actions": "Actions" }, @@ -97,20 +94,13 @@ "raisePriority": "Raise priority", "lowerPriority": "Lower priority", "editPriority": "Click to type a priority (negatives allowed)", - "recent": "recent", - "recentSyncedHint": "Recent uploads synced.", - "recentNotSyncedHint": "Recent uploads not fetched yet.", - "full": "full", "fullHint": "Full history fetched.", - "fullNotFetchedHint": "Full back-catalog not fetched yet — use the backfill action in the Actions column.", "fullySynced": "fully synced", "fullySyncedHint": "Recent uploads and the full back-catalog are both in.", "backfillThis": "Backfill this channel", "backfillThisHint": "Fetch this channel's full back-catalog (older videos + complete search). Click again to cancel the request.", "resetBackfill": "Reset & re-fetch this channel", "resetHint": "Admin: re-fetch this channel from scratch (recent + full back-catalog), regardless of its current sync state.", - "fullHistoryQueued": "queued", - "fullHistoryComing": "coming", "queuedRequestedHint": "Full history requested — this channel's whole back-catalog will backfill as the shared quota allows. Click to cancel your request.", "queuedByOtherHint": "Another subscriber already requested this channel's full history, so its whole back-catalog is on its way to everyone — nothing to do here.", "hiddenHint": "Hidden — this channel's videos are kept out of your feed. Click to show them again.", diff --git a/frontend/src/i18n/locales/hu/channel.json b/frontend/src/i18n/locales/hu/channel.json index 77463ac2..ddaee097 100644 --- a/frontend/src/i18n/locales/hu/channel.json +++ b/frontend/src/i18n/locales/hu/channel.json @@ -12,6 +12,7 @@ "videoCount_other": "{{count}} videó", "videoCountStored": "{{stored}} / {{total}} videó", "totalViews": "{{formatted}} megtekintés", + "videoTypes": "{{normal}} normál / {{short}} short / {{live}} élő", "joined": "Csatlakozott: {{date}}", "loadingVideos": "A csatorna videóinak betöltése…", "tabVideos": "Videók", diff --git a/frontend/src/i18n/locales/hu/channels.json b/frontend/src/i18n/locales/hu/channels.json index 29df7265..6b392cfc 100644 --- a/frontend/src/i18n/locales/hu/channels.json +++ b/frontend/src/i18n/locales/hu/channels.json @@ -68,9 +68,6 @@ "subs": "Feliratkozók", "lastUpload": "Utolsó feltöltés", "length": "Hossz", - "types": "N/S/L", - "typesHint": "Videók típusonként: Normál / Short / Live", - "sync": "Szinkron", "tags": "Címkék", "actions": "Műveletek" }, @@ -97,20 +94,13 @@ "raisePriority": "Prioritás növelése", "lowerPriority": "Prioritás csökkentése", "editPriority": "Kattints a prioritás beírásához (negatív is lehet)", - "recent": "legutóbbi", - "recentSyncedHint": "Legutóbbi feltöltések szinkronizálva.", - "recentNotSyncedHint": "A legutóbbi feltöltések még nincsenek letöltve.", - "full": "teljes", "fullHint": "Teljes előzmény letöltve.", - "fullNotFetchedHint": "A teljes archívum még nincs letöltve — használd a Műveletek oszlop backfill gombját.", "fullySynced": "teljesen szinkronban", "fullySyncedHint": "A legutóbbi feltöltések és a teljes archívum is megvan.", "backfillThis": "Csatorna teljes letöltése", "backfillThisHint": "A csatorna teljes archívumának letöltése (régebbi videók + teljes keresés). Újra kattintva visszavonod a kérést.", "resetBackfill": "Csatorna resetelése és újraletöltése", "resetHint": "Admin: a csatorna teljes újraletöltése a nulláról (legutóbbi + teljes archívum), a jelenlegi szinkron-állapottól függetlenül.", - "fullHistoryQueued": "sorban", - "fullHistoryComing": "úton", "queuedRequestedHint": "Teljes előzmény kérve — a csatorna teljes archívuma letöltődik, ahogy a megosztott kvóta engedi. Kattints a kérés visszavonásához.", "queuedByOtherHint": "Egy másik feliratkozó már kérte ennek a csatornának a teljes előzményét, így a teljes archívuma már úton van mindenkihez — itt nincs teendőd.", "hiddenHint": "Elrejtve — a csatorna videói nem jelennek meg a hírfolyamodban. Kattints, hogy újra láthatóak legyenek.", diff --git a/frontend/src/lib/api/channels.ts b/frontend/src/lib/api/channels.ts index b8417f85..c5113e54 100644 --- a/frontend/src/lib/api/channels.ts +++ b/frontend/src/lib/api/channels.ts @@ -27,6 +27,9 @@ export interface ChannelDetail { explored: boolean; blocked: boolean; stored_videos: number; + count_normal: number; + count_short: number; + count_live: number; deep_requested: boolean; priority: number; backfill_done: boolean; @@ -70,9 +73,6 @@ export interface ManagedChannel { stored_videos: number; last_video_at: string | null; total_duration_seconds: number; - count_normal: number; - count_short: number; - count_live: number; priority: number; hidden: boolean; deep_requested: boolean; diff --git a/frontend/src/lib/channelDetailPoll.test.ts b/frontend/src/lib/channelDetailPoll.test.ts new file mode 100644 index 00000000..03c7a446 --- /dev/null +++ b/frontend/src/lib/channelDetailPoll.test.ts @@ -0,0 +1,24 @@ +import { describe, it, expect } from "vitest"; +import { channelDetailPollMs } from "./channelDetailPoll"; +import { type ChannelDetail } from "./api"; + +// Only deep_requested + backfill_done drive the decision; cast a partial for the pure predicate. +const d = (over: Partial): ChannelDetail => over as ChannelDetail; + +describe("channelDetailPollMs", () => { + it("polls at 4s while THIS user's full-history backfill is running", () => { + expect(channelDetailPollMs(d({ deep_requested: true, backfill_done: false }))).toBe(4000); + }); + + it("stops (false) once the back-catalogue is fully in", () => { + expect(channelDetailPollMs(d({ deep_requested: true, backfill_done: true }))).toBe(false); + }); + + it("stops (false) when the user hasn't requested full history", () => { + expect(channelDetailPollMs(d({ deep_requested: false, backfill_done: false }))).toBe(false); + }); + + it("stops (false) before the detail has loaded", () => { + expect(channelDetailPollMs(undefined)).toBe(false); + }); +}); diff --git a/frontend/src/lib/channelDetailPoll.ts b/frontend/src/lib/channelDetailPoll.ts new file mode 100644 index 00000000..0f796e51 --- /dev/null +++ b/frontend/src/lib/channelDetailPoll.ts @@ -0,0 +1,8 @@ +import { type ChannelDetail } from "./api"; + +/** How often the channel page re-polls its detail (via useLiveQuery): every 4s while THIS user's + * full-history backfill is running (`deep_requested && !backfill_done`) so the stored count + the + * N/S/L breakdown climb on their own, and `false` (stop polling) once the back-catalogue is in or + * the user never requested it. Extracted so the poll-or-stop decision is unit-testable. */ +export const channelDetailPollMs = (d: ChannelDetail | undefined): number | false => + d?.deep_requested && !d.backfill_done ? 4000 : false; diff --git a/frontend/src/lib/releaseNotes.ts b/frontend/src/lib/releaseNotes.ts index 46090461..761ce00f 100644 --- a/frontend/src/lib/releaseNotes.ts +++ b/frontend/src/lib/releaseNotes.ts @@ -14,6 +14,23 @@ export interface ReleaseEntry { } export const RELEASE_NOTES: ReleaseEntry[] = [ + { + version: "0.65.0", + date: "2026-08-15", + summary: + "The channel manager table now fits your screen at any zoom, and asking for a channel's full history fills it in seconds instead of over many minutes.", + features: [ + "Channel manager table redesign: it fits the width at any zoom now (no more sideways scrolling), fully-synced channels get a ✓ next to their stored count, and the crowded N/S/L and Sync columns are gone — the channel-type breakdown moved to the channel page.", + "“Get full history” is now immediate: clicking it fills the whole channel's back-catalogue in seconds rather than waiting for the background schedule, and the channel page's counts climb live as it fills — no need to refresh.", + "The channel page shows the Normal / Shorts / Live breakdown for the videos it holds.", + ], + fixes: [ + "The channel name and description columns now shrink and show “…” (full name on hover) instead of pushing the table off-screen.", + ], + chores: [ + "Backend: the immediate full-history backfill coordinates with the background scheduler so the two never duplicate work, its API-quota use is recorded correctly, and it's capped so one burst can't crowd out others.", + ], + }, { version: "0.64.0", date: "2026-08-15", diff --git a/frontend/src/lib/useLiveQuery.ts b/frontend/src/lib/useLiveQuery.ts index 69b96fbc..22af46c1 100644 --- a/frontend/src/lib/useLiveQuery.ts +++ b/frontend/src/lib/useLiveQuery.ts @@ -9,10 +9,13 @@ export function useLiveQuery( key: QueryKey, queryFn: () => Promise, // intervalMs may be a function of the latest data, so the poll cadence can adapt to it - // (e.g. poll faster while a job is running). react-query re-evaluates it after each fetch - // against fresh data — the right place for this, vs. a React state toggle that can lag or - // stall the live updates. - opts: { intervalMs?: number | ((data: T | undefined) => number); enabled?: boolean } = {}, + // (e.g. poll faster while a job is running, and return `false` to STOP once it's done). + // react-query re-evaluates it after each fetch against fresh data — the right place for this, + // vs. a React state toggle that can lag or stall the live updates. + opts: { + intervalMs?: number | false | ((data: T | undefined) => number | false); + enabled?: boolean; + } = {}, ) { const { intervalMs = 4000, enabled = true } = opts; return useQuery({