fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound

- An ad-hoc request made while its target's run is under way is the run after it: the scheduler
  dropped a second fire of a running id, so a second Takeout file was never imported and its key
  stayed queued until a restart.
- Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view
  no longer marks it watched again; un-marks take the user's Takeout lock.
- One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the
  others are settled, and a failed lookup spends none of the daily share.
- The upload reads the user's id before giving the read connection back.
- A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a
  mark never pins it against the cleanup jobs.
- The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and
  two uploads at once across all users (takeout_server_busy).
- A Watch later list goes in as one presence read and one insert.
This commit is contained in:
2026-10-23 22:07:11 +02:00
parent 9f580fd8c8
commit 9c5f81fd94
17 changed files with 667 additions and 70 deletions
+19 -10
View File
@@ -309,38 +309,47 @@ def get_video_detail(
# --- Google Takeout ------------------------------------------------------------------------------
# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these
# bound what one account can make the process hold. The section uploads its files one after another.
# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are
# held whole while it is read (its text a window at a time), so these bound what the accounts can
# make the process hold. The section uploads its files one after another.
_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
_takeout_uploading: set[int] = set()
_takeout_uploading_guard = threading.Lock()
_TAKEOUT_AT_ONCE = 2
async def _takeout_body(
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> AsyncIterator[bytes]:
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
# megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is
# taken first: the release expires the user, and reading it after would reload it, taking a
# connection straight back out.
uid = user.id
release_read_connection(db)
with _takeout_uploading_guard:
busy = user.id in _takeout_uploading
if not busy:
_takeout_uploading.add(user.id)
if busy or not _takeout_limiter.allow(str(user.id)):
if not busy:
busy = uid in _takeout_uploading
full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE
taken = not busy and not full
if taken:
_takeout_uploading.add(uid)
if not taken or not _takeout_limiter.allow(str(uid)):
if taken:
with _takeout_uploading_guard:
_takeout_uploading.discard(user.id)
_takeout_uploading.discard(uid)
# Read to the end before answering: refused mid-upload, the browser reports a network
# failure instead of the refusal (capped_stream_to says why).
await drain_body(request)
if busy:
raise refused(429, "takeout_busy")
if full:
raise refused(429, "takeout_server_busy")
raise refused(429, "takeout_cooldown")
try:
yield await capped_body(request, takeout.MAX_BYTES)
finally:
with _takeout_uploading_guard:
_takeout_uploading.discard(user.id)
_takeout_uploading.discard(uid)
@router.post("/takeout")