fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound

- An ad-hoc request made while its target's run is under way is the run after it: the scheduler
  dropped a second fire of a running id, so a second Takeout file was never imported and its key
  stayed queued until a restart.
- Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view
  no longer marks it watched again; un-marks take the user's Takeout lock.
- One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the
  others are settled, and a failed lookup spends none of the daily share.
- The upload reads the user's id before giving the read connection back.
- A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a
  mark never pins it against the cleanup jobs.
- The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and
  two uploads at once across all users (takeout_server_busy).
- A Watch later list goes in as one presence read and one insert.

backend/app/reader/models.py

@@ -85,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin):
     progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
+class VideoUnmark(Base):
+    """When the user last took a video back to "new" here — un-marked it, or reset it. That leaves
+    no `VideoState` (its absence is "new"), so without this row a later import of what they watched
+    on YouTube could not tell a video they un-marked from one never touched, and marked it watched
+    again. Kept by user and video; the video's own deletion takes it."""
+
+    __tablename__ = "video_unmarks"
+    __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),)
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
+    video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
+    at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+
+
 class Playlist(Base, TimestampMixin, UpdatedAtMixin):
     """A per-user named, ordered collection of videos from the shared catalog.
 

backend/app/reader/queries.py

@@ -26,6 +26,7 @@ from app.reader.models import (
     TakeoutItem,
     TakeoutKind,
     VideoState,
+    VideoUnmark,
 )
 from app.userscope import owned_by, owned_row
 
@@ -376,6 +377,16 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
 # --- Google Takeout --------------------------------------------------------------------------
 
 
+def _settled_here() -> ColumnElement[bool]:
+    """A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a
+    discovery video only a search or an explore brought in. A mark is a `VideoState`, which the
+    cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the
+    video, and its explored channel, for good. Such a row waits until the video is the catalog's own
+    (its channel subscribed to) — or is gone."""
+    subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
+    return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed))
+
+
 def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
     """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
     (row id, video id, Takeout time, is Short)."""
@@ -388,6 +399,7 @@ def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datet
                 TakeoutItem.kind == TakeoutKind.WATCHED,
                 TakeoutItem.outcome.is_(None),
                 Video.shorts_probed,
+                _settled_here(),
             )
         ).tuples()
     )
@@ -421,6 +433,35 @@ def known_video_ids(db: Session, ids: list[str]) -> set[str]:
     return known
 
 
+def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]:
+    """When the user last took each of `ids` back to "new" here, for those they ever did."""
+    out: dict[str, datetime] = {}
+    for start in range(0, len(ids), _IN_CHUNK):
+        rows = db.execute(
+            select(VideoUnmark.video_id, VideoUnmark.at).where(
+                owned_by(VideoUnmark.user_id, user),
+                VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]),
+            )
+        ).tuples()
+        out.update((vid, at) for vid, at in rows)
+    return out
+
+
+def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]:
+    """Which of `ids` the playlist already holds."""
+    held: set[str] = set()
+    for start in range(0, len(ids), _IN_CHUNK):
+        chunk = ids[start : start + _IN_CHUNK]
+        held.update(
+            db.execute(
+                select(PlaylistItem.video_id).where(
+                    PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk)
+                )
+            ).scalars()
+        )
+    return held
+
+
 def takeout_rows_besides(
     db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str]
 ) -> int:
@@ -476,6 +517,7 @@ def takeout_users_ready(db: Session) -> list[int]:
                 TakeoutItem.kind == TakeoutKind.WATCHED,
                 TakeoutItem.outcome.is_(None),
                 Video.shorts_probed,
+                _settled_here(),
             )
             .distinct()
         ).scalars()

backend/app/reader/service.py

@@ -25,6 +25,7 @@ from app.reader.models import (
     TakeoutKind,
     TakeoutOutcome,
     VideoState,
+    VideoUnmark,
 )
 from app.sync.playlists import lock_playlists, recompute_dirty
 from app.userscope import owned_by
@@ -206,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None:
     the in-progress state, not where the user left off); otherwise the row goes. The player fires
     this beside a checkpoint that may delete the same row: the caller's commit then raises
     `StaleDataError`, which is that lost race and nothing worse."""
+    lock_takeout(db, user)  # an import settling meanwhile reads the un-mark below, or comes first
     row = queries.video_state(db, user, video_id)
     if row is None:
         return
@@ -214,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None:
         row.watched_at = None
     else:
         db.delete(row)
+    _remember_unmark(db, user, video_id)
     db.flush()
 
 
+def _remember_unmark(db: Session, user: User, video_id: str) -> None:
+    """Note that the user took the video back to "new" now: the state they set leaves no row, and
+    an import of their YouTube history must not mark it watched again (`apply_takeout_watched`)."""
+    now = datetime.now(timezone.utc)
+    db.execute(
+        pg_insert(VideoUnmark)
+        .values(user_id=user.id, video_id=video_id, at=now)
+        .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now})
+    )
+
+
 def mark(db: Session, user: User, video_id: str, status: str) -> None:
     """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
     the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
@@ -241,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None:
 def clear_state(db: Session, user: User, video_id: str) -> None:
     """The video back to pristine for this user: status, resume point and `watched_at` all go.
     Playlist membership lives elsewhere and stays."""
+    lock_takeout(db, user)  # as `unmark`
     row = queries.video_state(db, user, video_id)
     if row is not None:
         db.delete(row)
+        _remember_unmark(db, user, video_id)
         db.flush()
 
 
@@ -517,7 +533,8 @@ _TAKEOUT_LOCK_NS = 0x544F
 def lock_takeout(db: Session, user: User) -> None:
     """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an
     import's settling each lock many of the same rows, one statement after another and in no common
-    order, so without one lock over both the two can deadlock."""
+    order, so without one lock over both the two can deadlock. An un-mark takes it too: the import
+    reads the un-marks before it marks, and one landing between the two would be overwritten."""
     db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id)))
 
 
@@ -558,15 +575,21 @@ def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
 def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
     """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
     for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
-    past is not watching it), and a state the user set here wins: watched or hidden, or anything
-    changed after the Takeout time. The upsert's own condition is that rule, so a state set while
-    the import runs wins too."""
+    past is not watching it), and a state the user set here wins: watched or hidden, anything
+    changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own
+    condition is that rule, so a state set while the import runs wins too."""
     lock_takeout(db, user)
     short: list[int] = []
     todo: dict[str, tuple[int, datetime]] = {}
-    for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user):
+    ready = queries.takeout_watched_ready(db, user)
+    unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready])
+    kept: list[int] = []
+    for row_id, video_id, at, is_short in ready:
         if is_short:
             short.append(row_id)
+        elif video_id in unmarked and unmarked[video_id] >= at:
+            # Un-marked here since they watched it there: theirs, as a state they set is.
+            kept.append(row_id)
         else:
             todo[video_id] = (row_id, at)
     marked: set[str] = set()
@@ -593,7 +616,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
                 ).returning(VideoState.video_id)
             ).scalars()
         )
-    kept = [row_id for vid, (row_id, _) in items if vid not in marked]
+    kept += [row_id for vid, (row_id, _) in items if vid not in marked]
     _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
     _settle(db, kept, TakeoutOutcome.KEPT)
     _settle(db, short, TakeoutOutcome.SHORT)
@@ -610,17 +633,27 @@ def apply_takeout_watch_later(
     if not rows:
         return {}
     lock_takeout(db, user)
-    known = queries.known_video_ids(db, [vid for _, vid in rows])
+    ids = [vid for _, vid in rows]
+    known = queries.known_video_ids(db, ids)
     pl = watch_later(db, user)
+    # What add_item does a video at a time, for the whole list at once: one presence read and one
+    # insert, not four statements a video (a full list is thousands). Locked before the read, as there.
+    next_position = next_item_position(db, pl.id)
+    held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known])
     done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
     for row_id, vid in rows:
         if vid not in known:
             if vid in asked:
                 done[TakeoutOutcome.MISSING].append(row_id)
-        elif add_item(db, pl, vid, mark_dirty=False):
-            done[TakeoutOutcome.ADDED].append(row_id)
-        else:
+        elif vid in held:
             done[TakeoutOutcome.PRESENT].append(row_id)
+        else:
+            db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position))
+            next_position += 1
+            held.add(vid)
+            done[TakeoutOutcome.ADDED].append(row_id)
+    if done[TakeoutOutcome.ADDED]:
+        touch(pl)
     for outcome, ids in done.items():
         _settle(db, ids, outcome)
     db.flush()

backend/app/reader/takeout.py

@@ -15,6 +15,7 @@ Ads". The title is in the account's language, so nothing here reads it. A Short
 header row (in the account's language) and `<id>,<ISO time added>` rows.
 """
 
+import codecs
 import csv
 import io
 import json
@@ -28,7 +29,7 @@ from itertools import chain
 
 from sqlalchemy.orm import Session
 
-from app import quota
+from app import logs, quota
 from app.bodylimit import TAKEOUT_MAX_BYTES
 from app.db import SessionLocal
 from app.models import User
@@ -39,6 +40,8 @@ from app.sync.runner import get_service_user
 from app.sync.videos import ensure_videos
 from app.youtube.client import YouTubeClient
 
+log = logs.get_logger(__name__)
+
 # The largest file the upload takes (its edge cap and the route's own are one number).
 MAX_BYTES = TAKEOUT_MAX_BYTES
 # The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
@@ -150,44 +153,113 @@ def _skip(text: str, at: int) -> int:
     return at
 
 
-def _json_list(text: str) -> Iterator[object]:
-    """The entries of a JSON list, one at a time: the whole tree is never built, so a file of a
-    million tiny entries costs its text and one entry, not a million dicts at once."""
+# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a
+# character (one emoji in a title is enough), several times the file on top of its bytes.
+_WINDOW = 1024 * 1024
+
+
+class _Window:
+    """The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on.
+    The bytes are the caller's, so only the window is this reader's memory."""
+
+    def __init__(self, body: bytes, start: int):
+        self._body, self._next = body, start
+        self._decoder = codecs.getincrementaldecoder("utf-8")()  # strict: bad UTF-8 raises
+        self.text, self.at = "", 0
+
+    @property
+    def done(self) -> bool:
+        return self._next >= len(self._body)
+
+    def more(self, at_least: int = 0) -> None:
+        """Decode the next window (at least `at_least` bytes), dropping what is read already."""
+        self.text, self.at = self.text[self.at :], 0
+        size = max(_WINDOW, at_least)
+        chunk = self._body[self._next : self._next + size]
+        self._next += len(chunk)
+        self.text += self._decoder.decode(chunk, final=self.done)
+
+    def skip(self) -> None:
+        """Past the whitespace, reading on as far as it goes."""
+        while True:
+            self.at = _skip(self.text, self.at)
+            if self.at < len(self.text) or self.done:
+                return
+            self.more()
+
+    def startswith(self, char: str) -> bool:
+        self.skip()
+        return self.text.startswith(char, self.at)
+
+
+def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None:
+    """The JSON value at `at` and where it ends, or None where none reads (yet)."""
+    try:
+        return decoder.raw_decode(text, at)
+    # silent-ok: not a failure here — the caller reads on, or refuses the file once it has all.
+    except json.JSONDecodeError:
+        return None
+
+
+def _json_list(body: bytes, start: int) -> Iterator[object]:
+    """The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never
+    built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not
+    a million dicts at once."""
     decoder = json.JSONDecoder()
-    at = _skip(text, _skip(text, 0) + 1)  # past the "[" the caller saw
-    if text.startswith("]", at):
-        at += 1
+    window = _Window(body, start + 1)  # past the "[" the caller saw
+    if window.startswith("]"):
+        window.at += 1
     else:
         while True:
-            entry, at = decoder.raw_decode(text, at)
+            window.skip()
+            read = _decode(decoder, window.text, window.at)
+            # An entry cut by the window's edge fails, or (a number) reads short: read on and try
+            # again, with a window twice what is left, so one entry larger than any window is read
+            # in a few tries, not one a window.
+            if read is None or (read[1] == len(window.text) and not window.done):
+                if window.done:
+                    raise ValueError("not a JSON list")
+                window.more(2 * (len(window.text) - window.at))
+                continue
+            entry, window.at = read
             yield entry
-            at = _skip(text, at)
-            if text.startswith(",", at):
-                at = _skip(text, at + 1)
-            elif text.startswith("]", at):
-                at += 1
+            if window.startswith(","):
+                window.at += 1
+            elif window.startswith("]"):
+                window.at += 1
                 break
             else:
                 raise ValueError("not a JSON list")
-    if _skip(text, at) != len(text):
+    window.skip()
+    if window.at != len(window.text):
         raise ValueError("text after the JSON list")
 
 
+# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten
+# times that is not one — and, unlike the history, it is decoded whole.
+CSV_MAX_BYTES = 2 * 1024 * 1024
+_BOM = codecs.BOM_UTF8
+
+
 def parse(body: bytes) -> Parsed:
     """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
     apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
     with nothing to import in it."""
-    try:
-        text = body.decode("utf-8-sig")
-    except UnicodeDecodeError:
-        raise TakeoutError("not_takeout") from None
-    stripped = text.lstrip()
-    if stripped.startswith(("{", "<")):
+    start = len(_BOM) if body.startswith(_BOM) else 0
+    while start < len(body) and body[start] in b" \t\n\r\f\v":
+        start += 1
+    first = body[start : start + 1]
+    if first in (b"{", b"<"):
         # Another JSON file of the export, or the history's HTML variant.
         raise TakeoutError("not_takeout")
+    if first != b"[" and len(body) - start > CSV_MAX_BYTES:
+        raise TakeoutError("too_many")
     try:
+        # A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file.
         parsed = (
-            _history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped)
+            _history(_json_list(body, start))
+            if first == b"["
+            else _watch_later(body[start:].decode("utf-8"))
         )
     # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
     # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
@@ -200,10 +272,10 @@ def parse(body: bytes) -> Parsed:
     return parsed
 
 
-# One import per user at a time: a second file uploaded while the first one's run is under way
-# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
-# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
-# so a lock in this module holds it (as the channel backfill's does).
+# One import per user at a time: a second file uploaded while the first one's run is under way runs
+# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a
+# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a
+# lock in this module holds it (as the channel backfill's does).
 _user_locks: dict[int, threading.Lock] = {}
 _user_locks_guard = threading.Lock()
 
@@ -266,11 +338,14 @@ def _put_in_watch_later(db: Session, user: User) -> dict:
     if (
         fetcher is not None
         and not quota.backfill_reserve_reached()
-        and _lookups.allow(str(user.id), cost=len(ask))
+        and _lookups.room(str(user.id)) >= len(ask)
     ):
         with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
             with YouTubeClient(db, fetcher) as yt:
                 ensure_videos(db, yt, ask)
+        # Counted once done: a lookup that failed spent none of the share (one run per user at a
+        # time, so the room asked above is still there).
+        _lookups.allow(str(user.id), cost=len(ask))
         asked = set(ask)
     # Only the rows read above: one a second upload staged meanwhile was never asked about.
     return service.apply_takeout_watch_later(db, user, waiting, asked)
@@ -301,23 +376,36 @@ def apply_waiting(db: Session) -> dict:
     search's), and the Watch later videos a run could not put in yet. Run after each run of the
     `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
     nothing was waiting that could move."""
-    marked = added = 0
+    marked = added = failed = 0
     ready = set(queries.takeout_users_ready(db))
     later = set(queries.takeout_users_watch_later_waiting(db))
     for user_id in sorted(ready | later):
         user = db.get(User, user_id)
         if user is None:
             continue
-        with _one_import(user_id):
-            if user_id in ready:
-                marked += service.apply_takeout_watched(db, user)["watched"]
-                db.commit()
-            if user_id in later:
-                added += _put_in_watch_later(db, user).get("watch_later_added", 0)
-                db.commit()
+        # One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that
+        # runs this nor keeps the users after them waiting. What it did not settle waits.
+        try:
+            with _one_import(user_id):
+                if user_id in ready:
+                    marked += service.apply_takeout_watched(db, user)["watched"]
+                    db.commit()
+                if user_id in later:
+                    added += _put_in_watch_later(db, user).get("watch_later_added", 0)
+                    db.commit()
+        except Exception:
+            db.rollback()
+            failed += 1
+            log.exception(
+                "Takeout settling failed for user %s",
+                user_id,
+                extra=logs.event("takeout.settle.failed", user_id=user_id),
+            )
     out = {}
     if ready:
         out["takeout_watched"] = marked
     if added:
         out["takeout_watch_later_added"] = added
+    if failed:
+        out["takeout_failed"] = failed
     return out