fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound

- An ad-hoc request made while its target's run is under way is the run after it: the scheduler
  dropped a second fire of a running id, so a second Takeout file was never imported and its key
  stayed queued until a restart.
- Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view
  no longer marks it watched again; un-marks take the user's Takeout lock.
- One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the
  others are settled, and a failed lookup spends none of the daily share.
- The upload reads the user's id before giving the read connection back.
- A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a
  mark never pins it against the cleanup jobs.
- The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and
  two uploads at once across all users (takeout_server_busy).
- A Watch later list goes in as one presence read and one insert.

backend/app/reader/service.py

@@ -25,6 +25,7 @@ from app.reader.models import (
     TakeoutKind,
     TakeoutOutcome,
     VideoState,
+    VideoUnmark,
 )
 from app.sync.playlists import lock_playlists, recompute_dirty
 from app.userscope import owned_by
@@ -206,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None:
     the in-progress state, not where the user left off); otherwise the row goes. The player fires
     this beside a checkpoint that may delete the same row: the caller's commit then raises
     `StaleDataError`, which is that lost race and nothing worse."""
+    lock_takeout(db, user)  # an import settling meanwhile reads the un-mark below, or comes first
     row = queries.video_state(db, user, video_id)
     if row is None:
         return
@@ -214,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None:
         row.watched_at = None
     else:
         db.delete(row)
+    _remember_unmark(db, user, video_id)
     db.flush()
 
 
+def _remember_unmark(db: Session, user: User, video_id: str) -> None:
+    """Note that the user took the video back to "new" now: the state they set leaves no row, and
+    an import of their YouTube history must not mark it watched again (`apply_takeout_watched`)."""
+    now = datetime.now(timezone.utc)
+    db.execute(
+        pg_insert(VideoUnmark)
+        .values(user_id=user.id, video_id=video_id, at=now)
+        .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now})
+    )
+
+
 def mark(db: Session, user: User, video_id: str, status: str) -> None:
     """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
     the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
@@ -241,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None:
 def clear_state(db: Session, user: User, video_id: str) -> None:
     """The video back to pristine for this user: status, resume point and `watched_at` all go.
     Playlist membership lives elsewhere and stays."""
+    lock_takeout(db, user)  # as `unmark`
     row = queries.video_state(db, user, video_id)
     if row is not None:
         db.delete(row)
+        _remember_unmark(db, user, video_id)
         db.flush()
 
 
@@ -517,7 +533,8 @@ _TAKEOUT_LOCK_NS = 0x544F
 def lock_takeout(db: Session, user: User) -> None:
     """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an
     import's settling each lock many of the same rows, one statement after another and in no common
-    order, so without one lock over both the two can deadlock."""
+    order, so without one lock over both the two can deadlock. An un-mark takes it too: the import
+    reads the un-marks before it marks, and one landing between the two would be overwritten."""
     db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id)))
 
 
@@ -558,15 +575,21 @@ def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
 def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
     """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
     for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
-    past is not watching it), and a state the user set here wins: watched or hidden, or anything
-    changed after the Takeout time. The upsert's own condition is that rule, so a state set while
-    the import runs wins too."""
+    past is not watching it), and a state the user set here wins: watched or hidden, anything
+    changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own
+    condition is that rule, so a state set while the import runs wins too."""
     lock_takeout(db, user)
     short: list[int] = []
     todo: dict[str, tuple[int, datetime]] = {}
-    for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user):
+    ready = queries.takeout_watched_ready(db, user)
+    unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready])
+    kept: list[int] = []
+    for row_id, video_id, at, is_short in ready:
         if is_short:
             short.append(row_id)
+        elif video_id in unmarked and unmarked[video_id] >= at:
+            # Un-marked here since they watched it there: theirs, as a state they set is.
+            kept.append(row_id)
         else:
             todo[video_id] = (row_id, at)
     marked: set[str] = set()
@@ -593,7 +616,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
                 ).returning(VideoState.video_id)
             ).scalars()
         )
-    kept = [row_id for vid, (row_id, _) in items if vid not in marked]
+    kept += [row_id for vid, (row_id, _) in items if vid not in marked]
     _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
     _settle(db, kept, TakeoutOutcome.KEPT)
     _settle(db, short, TakeoutOutcome.SHORT)
@@ -610,17 +633,27 @@ def apply_takeout_watch_later(
     if not rows:
         return {}
     lock_takeout(db, user)
-    known = queries.known_video_ids(db, [vid for _, vid in rows])
+    ids = [vid for _, vid in rows]
+    known = queries.known_video_ids(db, ids)
     pl = watch_later(db, user)
+    # What add_item does a video at a time, for the whole list at once: one presence read and one
+    # insert, not four statements a video (a full list is thousands). Locked before the read, as there.
+    next_position = next_item_position(db, pl.id)
+    held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known])
     done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
     for row_id, vid in rows:
         if vid not in known:
             if vid in asked:
                 done[TakeoutOutcome.MISSING].append(row_id)
-        elif add_item(db, pl, vid, mark_dirty=False):
-            done[TakeoutOutcome.ADDED].append(row_id)
-        else:
+        elif vid in held:
             done[TakeoutOutcome.PRESENT].append(row_id)
+        else:
+            db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position))
+            next_position += 1
+            held.add(vid)
+            done[TakeoutOutcome.ADDED].append(row_id)
+    if done[TakeoutOutcome.ADDED]:
+        touch(pl)
     for outcome, ids in done.items():
         _settle(db, ids, outcome)
     db.flush()