fix(reader): the watched-channels list keeps its channels, its shares and its retries

- A channel someone follows: the videos its exploration brings in (the list's dialog, the channel page's explore) come in as the catalog's own, not flagged via_explore, which subscribing would never clear and which kept their watched marks waiting for good. - The explore cleanup keeps the row of a channel a Takeout history shows watched (it still reclaims the videos), so a looked-at channel no longer drops off the list. - The Shorts job takes up only users whose list still has lengths to ask or channels to name, by each user's own follows, blocks and dismissals; a naming that failed is tried again, and the list says it is not complete. - The length lookups have a daily share of their own: a large history no longer leaves the export's Watch later file waiting a day. - Only a channel of the user's own history can be put out of the list. - The dialog's videos have a query root of their own: the list's invalidations no longer re-ask (and re-explore) under an open dialog.
2026-10-24 01:10:29 +02:00
parent b5f9cf14db
commit 33dd9ff054
9 changed files with 233 additions and 47 deletions
@@ -244,6 +244,18 @@ def kept_by_a_reader(video_id: InstrumentedAttribute[str]) -> ColumnElement[bool
    )


+def in_a_watch_history(channel_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
+    """A channel someone's uploaded Takeout history shows watched. The "watched but not followed"
+    list names a channel from its catalog row, so the explore cleanup keeps the row of one (its
+    videos it may still reclaim: opening the list's videos brings them back). As a clause, so the
+    cleanup's DELETE stays one statement."""
+    return (
+        select(TakeoutItem.id)
+        .where(TakeoutItem.channel_id == channel_id, TakeoutItem.kind == TakeoutKind.WATCHED)
+        .exists()
+    )
+
+
 def playlisted_by(db: Session, video_ids: list[str]) -> list[tuple[int, str]]:
    """(user id, video id) for every user who has one of these videos in a playlist."""
    return list(
@@ -592,17 +604,16 @@ def takeout_lengths_to_ask(db: Session, user: User) -> list[str]:


 def watched_channels_pending(db: Session, user: User) -> int:
-    """How many candidate channels still have a watched video whose length is not known: the
-    channels that may yet join the list."""
+    """How many channels a later run may still add to the list: the candidates with a watched video
+    whose length is not known, and the ones that made it but the catalog cannot name yet."""
    cand = _candidate_channels(user)
-    return (
-        db.scalar(
-            select(func.count(func.distinct(TakeoutItem.channel_id)))
-            .join(cand, cand.c.channel_id == TakeoutItem.channel_id)
-            .where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
-        )
-        or 0
-    )
+    unasked = db.execute(
+        select(TakeoutItem.channel_id)
+        .join(cand, cand.c.channel_id == TakeoutItem.channel_id)
+        .where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
+        .distinct()
+    ).scalars()
+    return len(set(unasked) | set(watched_channels_to_describe(db, user)))


 def watched_channels_to_describe(db: Session, user: User) -> list[str]:
@@ -655,6 +666,23 @@ def has_watch_history(db: Session, user: User) -> bool:
    )


+def watched_in_history(db: Session, user: User, channel_id: str) -> bool:
+    """Whether the user's Takeout history shows them watching this channel at all."""
+    return bool(
+        db.scalar(
+            select(
+                select(TakeoutItem.id)
+                .where(
+                    owned_by(TakeoutItem.user_id, user),
+                    TakeoutItem.kind == TakeoutKind.WATCHED,
+                    TakeoutItem.channel_id == channel_id,
+                )
+                .exists()
+            )
+        )
+    )
+
+
 def watched_channel_videos(
    db: Session, user: User, channel_id: str
 ) -> tuple[list[tuple[str, datetime]], int]:
@@ -680,18 +708,19 @@ def watched_channel_videos(
    return counted, gone


-def takeout_users_with_lengths_to_ask(db: Session) -> list[int]:
-    """Every user with a channel of enough watched videos not all asked about yet. It does not ask
-    whether they follow the channel: that is each user's own question, which
-    `takeout_lengths_to_ask` answers (and an empty answer costs a run nothing)."""
+def takeout_users_with_list_work(db: Session) -> list[int]:
+    """Every user whose "watched but not followed" list a run can still add to: lengths to ask, or
+    channels to name. Asked of each user whose history holds a channel of enough watched videos,
+    since what is off a list (followed, blocked, put out, their own) is that user's own — a row of
+    one is never asked about, and would otherwise bring the user back on every run."""
    per_channel = (
        select(TakeoutItem.user_id)
        .where(TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.channel_id.is_not(None))
        .group_by(TakeoutItem.user_id, TakeoutItem.channel_id)
-        .having(
-            func.count() >= WATCHED_CHANNEL_MIN_VIDEOS,
-            func.count().filter(TakeoutItem.looked_up_at.is_(None)) > 0,
-        )
+        .having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
        .subquery()
    )
-    return list(db.execute(select(per_channel.c.user_id).distinct()).scalars())
+    users = db.execute(
+        select(User).where(User.id.in_(select(per_channel.c.user_id))).order_by(User.id)
+    ).scalars()
+    return [user.id for user in users if watched_channels_pending(db, user)]
@@ -322,6 +322,10 @@ def stage(db: Session, user: User, parsed: Parsed) -> None:
 # How many unknown Watch later videos one user may have looked up on the shared API budget in a day:
 # two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs.
 _lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
+# How many watched videos' lengths one user may ask about in a day for the "watched but not followed"
+# list (a unit a fifty). A share of its own: a large history asks for thousands, and on the Watch
+# later share it would leave the export's Watch later file waiting a day behind it.
+_lengths = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)


 def _put_in_watch_later(db: Session, user: User) -> dict:
@@ -356,8 +360,8 @@ def _put_in_watch_later(db: Session, user: User) -> dict:

 def _ask_lengths(db: Session, user: User) -> dict:
    """Ask YouTube how long the user's watched videos are that the "watched but not followed" list
-    still needs, and keep the answer — on the same terms and the same daily share as the Watch
-    later lookups. What is not asked yet (no YouTube-connected account, the reserve reached, the
+    still needs, and keep the answer — on the same terms as the Watch later lookups, on a daily
+    share of its own. What is not asked yet (no YouTube-connected account, the reserve reached, the
    share spent) waits for a later run, and the list says it is not complete. A YouTube failure is
    logged and left for a later run too: it is no reason to fail the import it follows. The caller
    commits."""
@@ -365,7 +369,7 @@ def _ask_lengths(db: Session, user: User) -> dict:
    fetcher = get_service_user(db) if ask else None
    if fetcher is None or quota.backfill_reserve_reached():
        return {}
-    ask = ask[: int(_lookups.room(str(user.id)))]
+    ask = ask[: int(_lengths.room(str(user.id)))]
    if not ask:
        return {}
    try:
@@ -379,7 +383,7 @@ def _ask_lengths(db: Session, user: User) -> dict:
            extra=logs.event("takeout.lengths.failed", user_id=user.id, reason=exc.reason),
        )
        return {}
-    _lookups.allow(str(user.id), cost=len(ask))
+    _lengths.allow(str(user.id), cost=len(ask))
    lengths = {
        it["id"]: parse_iso8601_duration(it.get("contentDetails", {}).get("duration"))
        for it in items
@@ -453,7 +457,7 @@ def apply_waiting(db: Session) -> dict:
    marked = added = asked = failed = 0
    ready = set(queries.takeout_users_ready(db))
    later = set(queries.takeout_users_watch_later_waiting(db))
-    lengths = set(queries.takeout_users_with_lengths_to_ask(db))
+    lengths = set(queries.takeout_users_with_list_work(db))
    for user_id in sorted(ready | later | lengths):
        user = db.get(User, user_id)
        if user is None:
@@ -307,9 +307,9 @@ def explore_channel(
    """Open an un-subscribed channel for browsing. Records the exploration (a per-user,
    grace-clocked row that gates this user's access to the channel's ephemeral videos and
    drives the later cleanup), flags the channel ephemeral unless someone organically follows
-    it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
-    Returns the next page token so the page can "load more". Guarded against the shared
-    backfill quota reserve."""
+    it, then ingests one page of its uploads (newest-first), enriched — marked via_explore while
+    nobody follows it. Returns the next page token so the page can "load more". Guarded against
+    the shared backfill quota reserve."""
    channel = _channel_or_404(db, channel_id)
    if queries.is_blocked(db, user, channel_id):
        raise refused(403, "channel_blocked")
@@ -427,9 +427,9 @@ def takeout_status(user: User = Depends(current_user), db: Session = Depends(get
 def watched_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
    """The channels the user's Takeout history shows them watching often but not following: each
    with how many of its videos over five minutes they watched, and when last. `pending` is how many
-    more channels may join once the import has asked YouTube for their videos' lengths, so a list
-    still being made does not read as the whole of it; `imported` tells "no history uploaded" from
-    "nothing to suggest"."""
+    more channels may join once the import has asked YouTube for their videos' lengths or their
+    names, so a list still being made does not read as the whole of it; `imported` tells "no history
+    uploaded" from "nothing to suggest"."""
    return {
        "imported": queries.has_watch_history(db, user),
        "pending": queries.watched_channels_pending(db, user),
@@ -457,7 +457,8 @@ def watched_channel_videos(
    """The videos of a channel that the user's list counts, ready to play, last watched first. The
    channel is explored for the user (as its page does), and the videos the catalog lacks come in as
    the exploration's: the player needs them in the catalog, the shared Library does not get them,
-    and the cleanup reclaims them once nobody keeps them. `gone`: how many YouTube no longer has."""
+    and the cleanup reclaims them once nobody keeps them. Of a channel someone follows they come in
+    as the catalog's own, as its uploads do. `gone`: how many YouTube no longer has."""
    channel = catalog_queries.channel(db, channel_id)
    if channel is None:
        raise not_found("channel")
@@ -477,7 +478,7 @@ def watched_channel_videos(
                quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE),
                YouTubeClient(db, user) as yt,
            ):
-                ensure_videos(db, yt, missing, explore=True)
+                ensure_videos(db, yt, missing, explore=channel.from_explore)
        except YouTubeError as exc:
            raise youtube_refusal(exc) from exc
    items = queries.feed_rows_by_ids(db, user, ids)
@@ -494,7 +495,10 @@ def dismiss_watched_channel(
    user: User = Depends(current_user),
    db: Session = Depends(get_db),
 ) -> dict:
-    """Put a channel out of the user's list (it stays out until restored)."""
+    """Put a channel out of the user's list (it stays out until restored). Only one their own history
+    shows them watching: a made-up id would only pile up rows every list read then has to skip."""
+    if not queries.watched_in_history(db, user, channel_id):
+        raise not_found("channel")
    service.dismiss_watched_channel(db, user, channel_id)
    db.commit()
    return {"ok": True, "channel_id": channel_id}
@@ -66,11 +66,13 @@ def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
 def explore_ingest_page(
    db: Session, yt: YouTubeClient, channel: Channel, page_token: str | None = None
 ) -> dict:
-    """Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new
-    videos via_explore, then enrich them. Returns the count ingested and the next page token
-    (None when the back-catalogue is exhausted) so the caller can offer "load more". Stateless
-    pagination: the caller round-trips the token, so the channel's own backfill cursor (used by
-    the organic deep backfill) is never touched."""
+    """Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new videos
+    via_explore while the channel is an exploration's (`from_explore`): of a channel someone follows
+    they are its uploads like any other, and a flag there would never be cleared, since subscribing
+    clears only an explored channel's. Then enrich them. Returns the count ingested and the next
+    page token (None when the back-catalogue is exhausted) so the caller can offer "load more".
+    Stateless pagination: the caller round-trips the token, so the channel's own backfill cursor
+    (used by the organic deep backfill) is never touched."""
    if not channel.uploads_playlist_id:
        return {"ingested": 0, "next_page_token": None}
    try:
@@ -86,7 +88,7 @@ def explore_ingest_page(
    for item in data.get("items", []):
        stub = stub_from_playlist_item(item, channel.id)
        if stub is not None:
-            stub["via_explore"] = True
+            stub["via_explore"] = channel.from_explore
            stubs.append(stub)
    inserted = insert_stubs(db, stubs)
    self_enriched = _enrich_new(db, yt, [s["id"] for s in stubs])
@@ -105,7 +107,9 @@ def purge_unkept_explored(db: Session) -> dict:
      - a live ExploredChannel row (someone explored it within the grace window — channel-level), or
      - a video has per-user state: watched/in-progress (VideoState), in a playlist (PlaylistItem),
        or surfaced by the user's own search (SearchFind) — these survive even if the channel goes.
-    A channel row is dropped only once it holds no videos at all. 0 quota."""
+    A channel row is dropped only once it holds no videos at all, and not while someone's Takeout
+    history shows it watched (their "watched but not followed" list names it from that row).
+    0 quota."""
    grace_days = sysconfig.get_int(db, "explore_grace_days")
    cutoff = datetime.now(timezone.utc) - timedelta(days=grace_days)

@@ -135,7 +139,9 @@ def purge_unkept_explored(db: Session) -> dict:

    has_videos = select(Video.id).where(Video.channel_id == Channel.id).exists()
    channels_deleted = db.execute(
-        delete(Channel).where(Channel.id.in_(orphan_ids), ~has_videos)
+        delete(Channel).where(
+            Channel.id.in_(orphan_ids), ~has_videos, ~reader_queries.in_a_watch_history(Channel.id)
+        )
    ).rowcount
    db.commit()
    return {