fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound

- An ad-hoc request made while its target's run is under way is the run after it: the scheduler
  dropped a second fire of a running id, so a second Takeout file was never imported and its key
  stayed queued until a restart.
- Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view
  no longer marks it watched again; un-marks take the user's Takeout lock.
- One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the
  others are settled, and a failed lookup spends none of the daily share.
- The upload reads the user's id before giving the read connection back.
- A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a
  mark never pins it against the cleanup jobs.
- The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and
  two uploads at once across all users (takeout_server_busy).
- A Watch later list goes in as one presence read and one insert.

backend/app/bodylimit.py

@@ -75,7 +75,8 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024
 # A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390
 # bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an
 # account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The
-# file is read an entry at a time (`takeout._json_list`), so it costs its own size, not a tree of it.
+# file's text is decoded a window at a time and read an entry at a time (`takeout._json_list`), so
+# it costs its bytes and a window, not a tree of it; the route takes two at once across every user.
 TAKEOUT_MAX_BYTES = 32 * 1024 * 1024
 
 ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {

backend/app/ratelimit.py

@@ -42,6 +42,14 @@ class RateLimiter:
         with self._lock:
             return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
 
+    def room(self, key: str) -> float:
+        """How much cost ``key`` may still spend in the current window, WITHOUT counting anything —
+        for work counted only once it has succeeded: ask it first, ``allow`` what was done."""
+        cutoff = time.monotonic() - self.window
+        with self._lock:
+            used = sum(c for t, c in self._hits.get(key, []) if t > cutoff)
+        return max(0.0, self.max_events - used)
+
     def allow(self, key: str, cost: float = 1) -> bool:
         """Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
 

backend/app/reader/models.py

@@ -85,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin):
     progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
+class VideoUnmark(Base):
+    """When the user last took a video back to "new" here — un-marked it, or reset it. That leaves
+    no `VideoState` (its absence is "new"), so without this row a later import of what they watched
+    on YouTube could not tell a video they un-marked from one never touched, and marked it watched
+    again. Kept by user and video; the video's own deletion takes it."""
+
+    __tablename__ = "video_unmarks"
+    __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),)
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
+    video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
+    at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+
+
 class Playlist(Base, TimestampMixin, UpdatedAtMixin):
     """A per-user named, ordered collection of videos from the shared catalog.
 

backend/app/reader/queries.py

@@ -26,6 +26,7 @@ from app.reader.models import (
     TakeoutItem,
     TakeoutKind,
     VideoState,
+    VideoUnmark,
 )
 from app.userscope import owned_by, owned_row
 
@@ -376,6 +377,16 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
 # --- Google Takeout --------------------------------------------------------------------------
 
 
+def _settled_here() -> ColumnElement[bool]:
+    """A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a
+    discovery video only a search or an explore brought in. A mark is a `VideoState`, which the
+    cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the
+    video, and its explored channel, for good. Such a row waits until the video is the catalog's own
+    (its channel subscribed to) — or is gone."""
+    subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
+    return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed))
+
+
 def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
     """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
     (row id, video id, Takeout time, is Short)."""
@@ -388,6 +399,7 @@ def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datet
                 TakeoutItem.kind == TakeoutKind.WATCHED,
                 TakeoutItem.outcome.is_(None),
                 Video.shorts_probed,
+                _settled_here(),
             )
         ).tuples()
     )
@@ -421,6 +433,35 @@ def known_video_ids(db: Session, ids: list[str]) -> set[str]:
     return known
 
 
+def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]:
+    """When the user last took each of `ids` back to "new" here, for those they ever did."""
+    out: dict[str, datetime] = {}
+    for start in range(0, len(ids), _IN_CHUNK):
+        rows = db.execute(
+            select(VideoUnmark.video_id, VideoUnmark.at).where(
+                owned_by(VideoUnmark.user_id, user),
+                VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]),
+            )
+        ).tuples()
+        out.update((vid, at) for vid, at in rows)
+    return out
+
+
+def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]:
+    """Which of `ids` the playlist already holds."""
+    held: set[str] = set()
+    for start in range(0, len(ids), _IN_CHUNK):
+        chunk = ids[start : start + _IN_CHUNK]
+        held.update(
+            db.execute(
+                select(PlaylistItem.video_id).where(
+                    PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk)
+                )
+            ).scalars()
+        )
+    return held
+
+
 def takeout_rows_besides(
     db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str]
 ) -> int:
@@ -476,6 +517,7 @@ def takeout_users_ready(db: Session) -> list[int]:
                 TakeoutItem.kind == TakeoutKind.WATCHED,
                 TakeoutItem.outcome.is_(None),
                 Video.shorts_probed,
+                _settled_here(),
             )
             .distinct()
         ).scalars()

backend/app/reader/service.py

@@ -25,6 +25,7 @@ from app.reader.models import (
     TakeoutKind,
     TakeoutOutcome,
     VideoState,
+    VideoUnmark,
 )
 from app.sync.playlists import lock_playlists, recompute_dirty
 from app.userscope import owned_by
@@ -206,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None:
     the in-progress state, not where the user left off); otherwise the row goes. The player fires
     this beside a checkpoint that may delete the same row: the caller's commit then raises
     `StaleDataError`, which is that lost race and nothing worse."""
+    lock_takeout(db, user)  # an import settling meanwhile reads the un-mark below, or comes first
     row = queries.video_state(db, user, video_id)
     if row is None:
         return
@@ -214,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None:
         row.watched_at = None
     else:
         db.delete(row)
+    _remember_unmark(db, user, video_id)
     db.flush()
 
 
+def _remember_unmark(db: Session, user: User, video_id: str) -> None:
+    """Note that the user took the video back to "new" now: the state they set leaves no row, and
+    an import of their YouTube history must not mark it watched again (`apply_takeout_watched`)."""
+    now = datetime.now(timezone.utc)
+    db.execute(
+        pg_insert(VideoUnmark)
+        .values(user_id=user.id, video_id=video_id, at=now)
+        .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now})
+    )
+
+
 def mark(db: Session, user: User, video_id: str, status: str) -> None:
     """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
     the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
@@ -241,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None:
 def clear_state(db: Session, user: User, video_id: str) -> None:
     """The video back to pristine for this user: status, resume point and `watched_at` all go.
     Playlist membership lives elsewhere and stays."""
+    lock_takeout(db, user)  # as `unmark`
     row = queries.video_state(db, user, video_id)
     if row is not None:
         db.delete(row)
+        _remember_unmark(db, user, video_id)
         db.flush()
 
 
@@ -517,7 +533,8 @@ _TAKEOUT_LOCK_NS = 0x544F
 def lock_takeout(db: Session, user: User) -> None:
     """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an
     import's settling each lock many of the same rows, one statement after another and in no common
-    order, so without one lock over both the two can deadlock."""
+    order, so without one lock over both the two can deadlock. An un-mark takes it too: the import
+    reads the un-marks before it marks, and one landing between the two would be overwritten."""
     db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id)))
 
 
@@ -558,15 +575,21 @@ def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
 def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
     """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
     for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
-    past is not watching it), and a state the user set here wins: watched or hidden, or anything
-    changed after the Takeout time. The upsert's own condition is that rule, so a state set while
-    the import runs wins too."""
+    past is not watching it), and a state the user set here wins: watched or hidden, anything
+    changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own
+    condition is that rule, so a state set while the import runs wins too."""
     lock_takeout(db, user)
     short: list[int] = []
     todo: dict[str, tuple[int, datetime]] = {}
-    for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user):
+    ready = queries.takeout_watched_ready(db, user)
+    unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready])
+    kept: list[int] = []
+    for row_id, video_id, at, is_short in ready:
         if is_short:
             short.append(row_id)
+        elif video_id in unmarked and unmarked[video_id] >= at:
+            # Un-marked here since they watched it there: theirs, as a state they set is.
+            kept.append(row_id)
         else:
             todo[video_id] = (row_id, at)
     marked: set[str] = set()
@@ -593,7 +616,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
                 ).returning(VideoState.video_id)
             ).scalars()
         )
-    kept = [row_id for vid, (row_id, _) in items if vid not in marked]
+    kept += [row_id for vid, (row_id, _) in items if vid not in marked]
     _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
     _settle(db, kept, TakeoutOutcome.KEPT)
     _settle(db, short, TakeoutOutcome.SHORT)
@@ -610,17 +633,27 @@ def apply_takeout_watch_later(
     if not rows:
         return {}
     lock_takeout(db, user)
-    known = queries.known_video_ids(db, [vid for _, vid in rows])
+    ids = [vid for _, vid in rows]
+    known = queries.known_video_ids(db, ids)
     pl = watch_later(db, user)
+    # What add_item does a video at a time, for the whole list at once: one presence read and one
+    # insert, not four statements a video (a full list is thousands). Locked before the read, as there.
+    next_position = next_item_position(db, pl.id)
+    held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known])
     done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
     for row_id, vid in rows:
         if vid not in known:
             if vid in asked:
                 done[TakeoutOutcome.MISSING].append(row_id)
-        elif add_item(db, pl, vid, mark_dirty=False):
-            done[TakeoutOutcome.ADDED].append(row_id)
-        else:
+        elif vid in held:
             done[TakeoutOutcome.PRESENT].append(row_id)
+        else:
+            db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position))
+            next_position += 1
+            held.add(vid)
+            done[TakeoutOutcome.ADDED].append(row_id)
+    if done[TakeoutOutcome.ADDED]:
+        touch(pl)
     for outcome, ids in done.items():
         _settle(db, ids, outcome)
     db.flush()

backend/app/reader/takeout.py

@@ -15,6 +15,7 @@ Ads". The title is in the account's language, so nothing here reads it. A Short
 header row (in the account's language) and `<id>,<ISO time added>` rows.
 """
 
+import codecs
 import csv
 import io
 import json
@@ -28,7 +29,7 @@ from itertools import chain
 
 from sqlalchemy.orm import Session
 
-from app import quota
+from app import logs, quota
 from app.bodylimit import TAKEOUT_MAX_BYTES
 from app.db import SessionLocal
 from app.models import User
@@ -39,6 +40,8 @@ from app.sync.runner import get_service_user
 from app.sync.videos import ensure_videos
 from app.youtube.client import YouTubeClient
 
+log = logs.get_logger(__name__)
+
 # The largest file the upload takes (its edge cap and the route's own are one number).
 MAX_BYTES = TAKEOUT_MAX_BYTES
 # The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
@@ -150,44 +153,113 @@ def _skip(text: str, at: int) -> int:
     return at
 
 
-def _json_list(text: str) -> Iterator[object]:
-    """The entries of a JSON list, one at a time: the whole tree is never built, so a file of a
-    million tiny entries costs its text and one entry, not a million dicts at once."""
+# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a
+# character (one emoji in a title is enough), several times the file on top of its bytes.
+_WINDOW = 1024 * 1024
+
+
+class _Window:
+    """The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on.
+    The bytes are the caller's, so only the window is this reader's memory."""
+
+    def __init__(self, body: bytes, start: int):
+        self._body, self._next = body, start
+        self._decoder = codecs.getincrementaldecoder("utf-8")()  # strict: bad UTF-8 raises
+        self.text, self.at = "", 0
+
+    @property
+    def done(self) -> bool:
+        return self._next >= len(self._body)
+
+    def more(self, at_least: int = 0) -> None:
+        """Decode the next window (at least `at_least` bytes), dropping what is read already."""
+        self.text, self.at = self.text[self.at :], 0
+        size = max(_WINDOW, at_least)
+        chunk = self._body[self._next : self._next + size]
+        self._next += len(chunk)
+        self.text += self._decoder.decode(chunk, final=self.done)
+
+    def skip(self) -> None:
+        """Past the whitespace, reading on as far as it goes."""
+        while True:
+            self.at = _skip(self.text, self.at)
+            if self.at < len(self.text) or self.done:
+                return
+            self.more()
+
+    def startswith(self, char: str) -> bool:
+        self.skip()
+        return self.text.startswith(char, self.at)
+
+
+def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None:
+    """The JSON value at `at` and where it ends, or None where none reads (yet)."""
+    try:
+        return decoder.raw_decode(text, at)
+    # silent-ok: not a failure here — the caller reads on, or refuses the file once it has all.
+    except json.JSONDecodeError:
+        return None
+
+
+def _json_list(body: bytes, start: int) -> Iterator[object]:
+    """The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never
+    built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not
+    a million dicts at once."""
     decoder = json.JSONDecoder()
-    at = _skip(text, _skip(text, 0) + 1)  # past the "[" the caller saw
-    if text.startswith("]", at):
-        at += 1
+    window = _Window(body, start + 1)  # past the "[" the caller saw
+    if window.startswith("]"):
+        window.at += 1
     else:
         while True:
-            entry, at = decoder.raw_decode(text, at)
+            window.skip()
+            read = _decode(decoder, window.text, window.at)
+            # An entry cut by the window's edge fails, or (a number) reads short: read on and try
+            # again, with a window twice what is left, so one entry larger than any window is read
+            # in a few tries, not one a window.
+            if read is None or (read[1] == len(window.text) and not window.done):
+                if window.done:
+                    raise ValueError("not a JSON list")
+                window.more(2 * (len(window.text) - window.at))
+                continue
+            entry, window.at = read
             yield entry
-            at = _skip(text, at)
-            if text.startswith(",", at):
-                at = _skip(text, at + 1)
-            elif text.startswith("]", at):
-                at += 1
+            if window.startswith(","):
+                window.at += 1
+            elif window.startswith("]"):
+                window.at += 1
                 break
             else:
                 raise ValueError("not a JSON list")
-    if _skip(text, at) != len(text):
+    window.skip()
+    if window.at != len(window.text):
         raise ValueError("text after the JSON list")
 
 
+# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten
+# times that is not one — and, unlike the history, it is decoded whole.
+CSV_MAX_BYTES = 2 * 1024 * 1024
+_BOM = codecs.BOM_UTF8
+
+
 def parse(body: bytes) -> Parsed:
     """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
     apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
     with nothing to import in it."""
-    try:
-        text = body.decode("utf-8-sig")
-    except UnicodeDecodeError:
-        raise TakeoutError("not_takeout") from None
-    stripped = text.lstrip()
-    if stripped.startswith(("{", "<")):
+    start = len(_BOM) if body.startswith(_BOM) else 0
+    while start < len(body) and body[start] in b" \t\n\r\f\v":
+        start += 1
+    first = body[start : start + 1]
+    if first in (b"{", b"<"):
         # Another JSON file of the export, or the history's HTML variant.
         raise TakeoutError("not_takeout")
+    if first != b"[" and len(body) - start > CSV_MAX_BYTES:
+        raise TakeoutError("too_many")
     try:
+        # A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file.
         parsed = (
-            _history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped)
+            _history(_json_list(body, start))
+            if first == b"["
+            else _watch_later(body[start:].decode("utf-8"))
         )
     # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
     # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
@@ -200,10 +272,10 @@ def parse(body: bytes) -> Parsed:
     return parsed
 
 
-# One import per user at a time: a second file uploaded while the first one's run is under way
-# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
-# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
-# so a lock in this module holds it (as the channel backfill's does).
+# One import per user at a time: a second file uploaded while the first one's run is under way runs
+# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a
+# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a
+# lock in this module holds it (as the channel backfill's does).
 _user_locks: dict[int, threading.Lock] = {}
 _user_locks_guard = threading.Lock()
 
@@ -266,11 +338,14 @@ def _put_in_watch_later(db: Session, user: User) -> dict:
     if (
         fetcher is not None
         and not quota.backfill_reserve_reached()
-        and _lookups.allow(str(user.id), cost=len(ask))
+        and _lookups.room(str(user.id)) >= len(ask)
     ):
         with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
             with YouTubeClient(db, fetcher) as yt:
                 ensure_videos(db, yt, ask)
+        # Counted once done: a lookup that failed spent none of the share (one run per user at a
+        # time, so the room asked above is still there).
+        _lookups.allow(str(user.id), cost=len(ask))
         asked = set(ask)
     # Only the rows read above: one a second upload staged meanwhile was never asked about.
     return service.apply_takeout_watch_later(db, user, waiting, asked)
@@ -301,23 +376,36 @@ def apply_waiting(db: Session) -> dict:
     search's), and the Watch later videos a run could not put in yet. Run after each run of the
     `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
     nothing was waiting that could move."""
-    marked = added = 0
+    marked = added = failed = 0
     ready = set(queries.takeout_users_ready(db))
     later = set(queries.takeout_users_watch_later_waiting(db))
     for user_id in sorted(ready | later):
         user = db.get(User, user_id)
         if user is None:
             continue
-        with _one_import(user_id):
-            if user_id in ready:
-                marked += service.apply_takeout_watched(db, user)["watched"]
-                db.commit()
-            if user_id in later:
-                added += _put_in_watch_later(db, user).get("watch_later_added", 0)
-                db.commit()
+        # One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that
+        # runs this nor keeps the users after them waiting. What it did not settle waits.
+        try:
+            with _one_import(user_id):
+                if user_id in ready:
+                    marked += service.apply_takeout_watched(db, user)["watched"]
+                    db.commit()
+                if user_id in later:
+                    added += _put_in_watch_later(db, user).get("watch_later_added", 0)
+                    db.commit()
+        except Exception:
+            db.rollback()
+            failed += 1
+            log.exception(
+                "Takeout settling failed for user %s",
+                user_id,
+                extra=logs.event("takeout.settle.failed", user_id=user_id),
+            )
     out = {}
     if ready:
         out["takeout_watched"] = marked
     if added:
         out["takeout_watch_later_added"] = added
+    if failed:
+        out["takeout_failed"] = failed
     return out

backend/app/routes/feed.py

@@ -309,38 +309,47 @@ def get_video_detail(
 # --- Google Takeout ------------------------------------------------------------------------------
 
 
-# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these
-# bound what one account can make the process hold. The section uploads its files one after another.
+# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are
+# held whole while it is read (its text a window at a time), so these bound what the accounts can
+# make the process hold. The section uploads its files one after another.
 _takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
 _takeout_uploading: set[int] = set()
 _takeout_uploading_guard = threading.Lock()
+_TAKEOUT_AT_ONCE = 2
 
 
 async def _takeout_body(
     request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
 ) -> AsyncIterator[bytes]:
     # The sign-in check has run, and it was a SELECT: hand the connection back before a file of
-    # megabytes arrives, so a slow upload does not hold one `idle in transaction`.
+    # megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is
+    # taken first: the release expires the user, and reading it after would reload it, taking a
+    # connection straight back out.
+    uid = user.id
     release_read_connection(db)
     with _takeout_uploading_guard:
-        busy = user.id in _takeout_uploading
-        if not busy:
-            _takeout_uploading.add(user.id)
-    if busy or not _takeout_limiter.allow(str(user.id)):
-        if not busy:
+        busy = uid in _takeout_uploading
+        full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE
+        taken = not busy and not full
+        if taken:
+            _takeout_uploading.add(uid)
+    if not taken or not _takeout_limiter.allow(str(uid)):
+        if taken:
             with _takeout_uploading_guard:
-                _takeout_uploading.discard(user.id)
+                _takeout_uploading.discard(uid)
         # Read to the end before answering: refused mid-upload, the browser reports a network
         # failure instead of the refusal (capped_stream_to says why).
         await drain_body(request)
         if busy:
             raise refused(429, "takeout_busy")
+        if full:
+            raise refused(429, "takeout_server_busy")
         raise refused(429, "takeout_cooldown")
     try:
         yield await capped_body(request, takeout.MAX_BYTES)
     finally:
         with _takeout_uploading_guard:
-            _takeout_uploading.discard(user.id)
+            _takeout_uploading.discard(uid)
 
 
 @router.post("/takeout")

backend/app/scheduler.py

@@ -637,6 +637,15 @@ def ad_hoc_target_title(db, job: str, target: str) -> str | None:
 # run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too.
 _queued: set[str] = set()
 _queued_guard = threading.Lock()
+# What the scheduler is told to run is only the key: the request itself waits here, the newest one
+# per key, until a fire of that key takes it. A request made while its key's run is under way is so
+# kept for the run after it, which that run asks for as it ends (`_running`): the scheduler would
+# drop a second fire of an id whose run has not returned (one instance per job id).
+_pending: dict[str, tuple[Callable[[], dict], int, bool]] = {}
+_running: set[str] = set()
+# Two at once, for the moment between a run letting go of its key and its thread returning: a fire
+# then is what takes the next request, and is not dropped.
+_AD_HOC_INSTANCES = 2
 
 
 def ad_hoc_queued(job: str, target: str) -> bool:
@@ -650,37 +659,77 @@ def run_ad_hoc(
 ) -> None:
     """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
     own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
-    a second request before the first has even started is that same run. `notify` posts its outcome
-    to the user's inbox, for work whose page does not watch it run."""
+    a second request before the first has even started is that same run, and one made while it runs
+    is the run after it. `notify` posts its outcome to the user's inbox, for work whose page does not
+    watch it run."""
     if job not in AD_HOC_JOBS:
         raise KeyError(job)  # an unnamed job would show on the page as its bare id
-    sched = _scheduler
-    if sched is None:
+    if _scheduler is None:
         # Only between shutdown and exit: start_scheduler runs at every API startup.
         raise RuntimeError("the scheduler is not running")
     key = f"{job}@{target}"
     with _queued_guard:
+        _pending[key] = (fn, actor_id, notify)
         _queued.add(key)
+        running = key in _running
+    if running:
+        log.info("ad-hoc %s for %s runs again once the run under way ends", job, target)
+        return
+    _fire(job, target)
+
+
+def _fire(job: str, target: str) -> None:
+    """Add the one-shot that takes `job@target`'s pending request. Raises (the request dropped) only
+    when the scheduler cannot take it."""
+    key = f"{job}@{target}"
+    sched = _scheduler
     try:
+        if sched is None:
+            raise RuntimeError("the scheduler is not running")
         sched.add_job(
-            _ad_hoc,
+            _ad_hoc_fired,
             "date",
-            args=(job, fn, actor_id, target, notify),
+            args=(job, target),
             id=key,
             misfire_grace_time=None,
+            max_instances=_AD_HOC_INSTANCES,
         )
-    # silent-ok: not a failure — the run for this target is already queued, and it does this work.
+    # silent-ok: not a failure — a fire of this key is already queued, and it takes this request.
     except ConflictingIdError:
         log.info("ad-hoc %s for %s already queued", job, target)
     except Exception:
         with _queued_guard:
+            _pending.pop(key, None)
             _queued.discard(key)
         raise
 
 
+def _ad_hoc_fired(job: str, target: str) -> None:
+    key = f"{job}@{target}"
+    with _queued_guard:
+        request = _pending.pop(key, None)
+        if request is None:
+            return  # an earlier fire took it
+        _running.add(key)
+    fn, actor_id, notify = request
+    try:
+        _ad_hoc(job, fn, actor_id, target, notify)
+    finally:
+        with _queued_guard:
+            _running.discard(key)
+            again = key in _pending
+        if again:
+            try:
+                _fire(job, target)
+            except Exception:
+                # The request made meanwhile is lost with the scheduler: shut down, nothing runs.
+                log.exception("ad-hoc %s for %s not run again", job, target)
+
+
 def _ad_hoc(
     job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False
 ) -> None:
+    key = f"{job}@{target}"
     try:
         run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
     except Exception:
@@ -688,9 +737,11 @@ def _ad_hoc(
         log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
         return
     finally:
-        # Recorded (or never will be): the run row now speaks for it.
+        # Recorded (or never will be): the run row now speaks for it — unless another request came
+        # meanwhile, which is still queued.
         with _queued_guard:
-            _queued.discard(f"{job}@{target}")
+            if key not in _pending:
+                _queued.discard(key)
     _opened(run_id, job)
     try:
         _ad_hoc_work(job, fn, run_id, actor_id if notify else None)
@@ -765,4 +816,6 @@ def shutdown_scheduler() -> None:
         _scheduler.shutdown(wait=False)
         _scheduler = None
     with _queued_guard:
-        _queued.clear()  # what it held will not run
+        # What it held will not run.
+        _queued.clear()
+        _pending.clear()