fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+48 -6
View File
@@ -1,3 +1,5 @@
import threading
from collections.abc import AsyncIterator
from datetime import date
from functools import partial
@@ -10,12 +12,14 @@ from app import jobruns, quota, scheduler
from app.auth import current_user
from app.db import get_db, release_read_connection
from app.mailtext import lang_for_user
from app.ratelimit import RateLimiter
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
from app.models import User
from app.reader import queries, service, takeout
from app.routes._common import (
ProgressIn,
capped_body,
drain_body,
kept_position,
not_found,
youtube_refusal,
@@ -305,17 +309,44 @@ def get_video_detail(
# --- Google Takeout ------------------------------------------------------------------------------
async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes:
# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these
# bound what one account can make the process hold. The section uploads its files one after another.
_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
_takeout_uploading: set[int] = set()
_takeout_uploading_guard = threading.Lock()
async def _takeout_body(
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> AsyncIterator[bytes]:
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
release_read_connection(db)
return await capped_body(request, takeout.MAX_BYTES)
with _takeout_uploading_guard:
busy = user.id in _takeout_uploading
if not busy:
_takeout_uploading.add(user.id)
if busy or not _takeout_limiter.allow(str(user.id)):
if not busy:
with _takeout_uploading_guard:
_takeout_uploading.discard(user.id)
# Read to the end before answering: refused mid-upload, the browser reports a network
# failure instead of the refusal (capped_stream_to says why).
await drain_body(request)
if busy:
raise refused(429, "takeout_busy")
raise refused(429, "takeout_cooldown")
try:
yield await capped_body(request, takeout.MAX_BYTES)
finally:
with _takeout_uploading_guard:
_takeout_uploading.discard(user.id)
@router.post("/takeout")
def upload_takeout(
# The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before
# a byte of it is read.
# The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's
# file is refused before a byte of it is read.
user: User = Depends(current_user),
body: bytes = Depends(_takeout_body),
db: Session = Depends(get_db),
@@ -327,8 +358,14 @@ def upload_takeout(
except takeout.TakeoutError as exc:
if exc.reason == "no_videos":
raise refused(422, "takeout_no_videos") from None
if exc.reason == "too_many":
raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None
raise refused(422, "takeout_unreadable") from None
service.stage_takeout(db, user, parsed.kind, parsed.videos)
try:
takeout.stage(db, user, parsed)
except takeout.TooMany:
db.rollback()
raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None
db.commit()
scheduler.run_ad_hoc(
scheduler.TAKEOUT_IMPORT,
@@ -353,8 +390,13 @@ def takeout_status(user: User = Depends(current_user), db: Session = Depends(get
"""What the user's Takeout import has done: its last run (none before the first upload), and
every video kept from the export, by kind and outcome — a video the catalog did not know yet
counts as "waiting" until it does, and is marked then."""
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, takeout.run_target(user.id))
target = takeout.run_target(user.id)
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target)
return {
# Queued and not started: its run row is not written yet, and the last run is still the
# previous one, so the page waits for it rather than taking that one for this upload's.
"queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target),
"max_bytes": takeout.MAX_BYTES,
"run": None
if run is None
else {