fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
import threading
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import date
|
||||
from functools import partial
|
||||
|
||||
@@ -10,12 +12,14 @@ from app import jobruns, quota, scheduler
|
||||
from app.auth import current_user
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.mailtext import lang_for_user
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
||||
from app.models import User
|
||||
from app.reader import queries, service, takeout
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
capped_body,
|
||||
drain_body,
|
||||
kept_position,
|
||||
not_found,
|
||||
youtube_refusal,
|
||||
@@ -305,17 +309,44 @@ def get_video_detail(
|
||||
# --- Google Takeout ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
||||
# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these
|
||||
# bound what one account can make the process hold. The section uploads its files one after another.
|
||||
_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
|
||||
_takeout_uploading: set[int] = set()
|
||||
_takeout_uploading_guard = threading.Lock()
|
||||
|
||||
|
||||
async def _takeout_body(
|
||||
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> AsyncIterator[bytes]:
|
||||
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
|
||||
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
|
||||
release_read_connection(db)
|
||||
return await capped_body(request, takeout.MAX_BYTES)
|
||||
with _takeout_uploading_guard:
|
||||
busy = user.id in _takeout_uploading
|
||||
if not busy:
|
||||
_takeout_uploading.add(user.id)
|
||||
if busy or not _takeout_limiter.allow(str(user.id)):
|
||||
if not busy:
|
||||
with _takeout_uploading_guard:
|
||||
_takeout_uploading.discard(user.id)
|
||||
# Read to the end before answering: refused mid-upload, the browser reports a network
|
||||
# failure instead of the refusal (capped_stream_to says why).
|
||||
await drain_body(request)
|
||||
if busy:
|
||||
raise refused(429, "takeout_busy")
|
||||
raise refused(429, "takeout_cooldown")
|
||||
try:
|
||||
yield await capped_body(request, takeout.MAX_BYTES)
|
||||
finally:
|
||||
with _takeout_uploading_guard:
|
||||
_takeout_uploading.discard(user.id)
|
||||
|
||||
|
||||
@router.post("/takeout")
|
||||
def upload_takeout(
|
||||
# The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before
|
||||
# a byte of it is read.
|
||||
# The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's
|
||||
# file is refused before a byte of it is read.
|
||||
user: User = Depends(current_user),
|
||||
body: bytes = Depends(_takeout_body),
|
||||
db: Session = Depends(get_db),
|
||||
@@ -327,8 +358,14 @@ def upload_takeout(
|
||||
except takeout.TakeoutError as exc:
|
||||
if exc.reason == "no_videos":
|
||||
raise refused(422, "takeout_no_videos") from None
|
||||
if exc.reason == "too_many":
|
||||
raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None
|
||||
raise refused(422, "takeout_unreadable") from None
|
||||
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
||||
try:
|
||||
takeout.stage(db, user, parsed)
|
||||
except takeout.TooMany:
|
||||
db.rollback()
|
||||
raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None
|
||||
db.commit()
|
||||
scheduler.run_ad_hoc(
|
||||
scheduler.TAKEOUT_IMPORT,
|
||||
@@ -353,8 +390,13 @@ def takeout_status(user: User = Depends(current_user), db: Session = Depends(get
|
||||
"""What the user's Takeout import has done: its last run (none before the first upload), and
|
||||
every video kept from the export, by kind and outcome — a video the catalog did not know yet
|
||||
counts as "waiting" until it does, and is marked then."""
|
||||
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, takeout.run_target(user.id))
|
||||
target = takeout.run_target(user.id)
|
||||
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target)
|
||||
return {
|
||||
# Queued and not started: its run row is not written yet, and the last run is still the
|
||||
# previous one, so the page waits for it rather than taking that one for this upload's.
|
||||
"queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target),
|
||||
"max_bytes": takeout.MAX_BYTES,
|
||||
"run": None
|
||||
if run is None
|
||||
else {
|
||||
|
||||
Reference in New Issue
Block a user