fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
+136
-37
@@ -24,6 +24,7 @@ from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -31,6 +32,7 @@ from app import quota
|
||||
from app.bodylimit import TAKEOUT_MAX_BYTES
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.reader import queries, service
|
||||
from app.reader.models import TakeoutKind
|
||||
from app.sync.runner import get_service_user
|
||||
@@ -39,6 +41,12 @@ from app.youtube.client import YouTubeClient
|
||||
|
||||
# The largest file the upload takes (its edge cap and the route's own are one number).
|
||||
MAX_BYTES = TAKEOUT_MAX_BYTES
|
||||
# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
|
||||
# looked up on the shared API budget, so a longer list is not one YouTube made.
|
||||
WATCH_LATER_MAX = 5000
|
||||
# The most Takeout videos one user keeps: twenty times the real export measured here (~9,000 videos
|
||||
# across both files), so it only stops uploads of made-up ones piling up for good.
|
||||
ROWS_MAX = 200_000
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
_WATCH_URL = re.compile(
|
||||
@@ -84,7 +92,7 @@ def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
||||
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
||||
|
||||
|
||||
def _history(data: list) -> Parsed:
|
||||
def _history(data: Iterator[object]) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCHED)
|
||||
for entry in data:
|
||||
parsed.entries += 1
|
||||
@@ -117,22 +125,54 @@ def _history(data: list) -> Parsed:
|
||||
|
||||
def _watch_later(text: str) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
||||
rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
|
||||
if not rows or len(rows[0]) < 2:
|
||||
rows = (row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row))
|
||||
first = next(rows, None)
|
||||
if first is None or len(first) < 2:
|
||||
raise TakeoutError("not_takeout")
|
||||
if not _VIDEO_ID.match(rows[0][0].strip()):
|
||||
rows = rows[1:] # the header, in the account's language
|
||||
for row in rows:
|
||||
# The first row is the header, in the account's language, unless it is already a video.
|
||||
for row in chain([first], rows) if _VIDEO_ID.match(first[0].strip()) else rows:
|
||||
parsed.entries += 1
|
||||
vid = row[0].strip() if row else ""
|
||||
vid = row[0].strip()
|
||||
at = _time(row[1].strip()) if len(row) > 1 else None
|
||||
if not _VIDEO_ID.match(vid) or at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
_keep(parsed, vid, at, None)
|
||||
if len(parsed.videos) > WATCH_LATER_MAX:
|
||||
raise TakeoutError("too_many")
|
||||
return parsed
|
||||
|
||||
|
||||
def _skip(text: str, at: int) -> int:
|
||||
"""The first index at or after `at` that is not JSON whitespace."""
|
||||
while at < len(text) and text[at] in " \t\n\r":
|
||||
at += 1
|
||||
return at
|
||||
|
||||
|
||||
def _json_list(text: str) -> Iterator[object]:
|
||||
"""The entries of a JSON list, one at a time: the whole tree is never built, so a file of a
|
||||
million tiny entries costs its text and one entry, not a million dicts at once."""
|
||||
decoder = json.JSONDecoder()
|
||||
at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw
|
||||
if text.startswith("]", at):
|
||||
at += 1
|
||||
else:
|
||||
while True:
|
||||
entry, at = decoder.raw_decode(text, at)
|
||||
yield entry
|
||||
at = _skip(text, at)
|
||||
if text.startswith(",", at):
|
||||
at = _skip(text, at + 1)
|
||||
elif text.startswith("]", at):
|
||||
at += 1
|
||||
break
|
||||
else:
|
||||
raise ValueError("not a JSON list")
|
||||
if _skip(text, at) != len(text):
|
||||
raise ValueError("text after the JSON list")
|
||||
|
||||
|
||||
def parse(body: bytes) -> Parsed:
|
||||
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
||||
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
||||
@@ -142,19 +182,19 @@ def parse(body: bytes) -> Parsed:
|
||||
except UnicodeDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith("["):
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
if not isinstance(data, list):
|
||||
raise TakeoutError("not_takeout")
|
||||
parsed = _history(data)
|
||||
elif stripped.startswith(("{", "<")):
|
||||
if stripped.startswith(("{", "<")):
|
||||
# Another JSON file of the export, or the history's HTML variant.
|
||||
raise TakeoutError("not_takeout")
|
||||
else:
|
||||
parsed = _watch_later(stripped)
|
||||
try:
|
||||
parsed = (
|
||||
_history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped)
|
||||
)
|
||||
# A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
|
||||
# long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
|
||||
except (ValueError, RecursionError, csv.Error) as exc:
|
||||
if isinstance(exc, TakeoutError):
|
||||
raise
|
||||
raise TakeoutError("not_takeout") from None
|
||||
if not parsed.videos:
|
||||
raise TakeoutError("no_videos")
|
||||
return parsed
|
||||
@@ -181,6 +221,61 @@ def run_target(user_id: int) -> str:
|
||||
return f"user:{user_id}"
|
||||
|
||||
|
||||
def target_title(db: Session, target: str) -> str | None:
|
||||
"""Who an import run's target is, for the admin's run lists: the user's name, else their email."""
|
||||
_, _, raw = target.partition("user:")
|
||||
user = db.get(User, int(raw)) if raw.isdigit() else None
|
||||
return (user.display_name or user.email) if user is not None else None
|
||||
|
||||
|
||||
class TooMany(Exception):
|
||||
"""The upload would take the user past `ROWS_MAX` kept videos."""
|
||||
|
||||
|
||||
def stage(db: Session, user: User, parsed: Parsed) -> None:
|
||||
"""Keep an uploaded file's videos for the import, unless that would take the user past
|
||||
`ROWS_MAX` (`TooMany`). The caller commits."""
|
||||
service.lock_takeout(db, user) # the count and the insert are one user's, not two uploads' each
|
||||
if (
|
||||
queries.takeout_rows_besides(db, user, parsed.kind, parsed.videos) + len(parsed.videos)
|
||||
> ROWS_MAX
|
||||
):
|
||||
raise TooMany
|
||||
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
||||
|
||||
|
||||
# How many unknown Watch later videos one user may have looked up on the shared API budget in a day:
|
||||
# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs.
|
||||
_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
||||
|
||||
|
||||
def _put_in_watch_later(db: Session, user: User) -> dict:
|
||||
"""Put the user's waiting Watch later videos into theirs here: the ones the catalog knows at once,
|
||||
the others once YouTube was asked for them — when a YouTube-connected account can ask, today's
|
||||
budget is above the backfill reserve, and the user's own daily share of lookups allows it. A video
|
||||
not asked about yet waits for a later run. The lookup commits what it fetched; the caller commits
|
||||
the rest."""
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if not waiting:
|
||||
return {}
|
||||
ids = [vid for _, vid in waiting]
|
||||
known = queries.known_video_ids(db, ids)
|
||||
ask = [vid for vid in ids if vid not in known][:WATCH_LATER_MAX]
|
||||
asked: set[str] = set()
|
||||
fetcher = get_service_user(db) if ask else None
|
||||
if (
|
||||
fetcher is not None
|
||||
and not quota.backfill_reserve_reached()
|
||||
and _lookups.allow(str(user.id), cost=len(ask))
|
||||
):
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
ensure_videos(db, yt, ask)
|
||||
asked = set(ask)
|
||||
# Only the rows read above: one a second upload staged meanwhile was never asked about.
|
||||
return service.apply_takeout_watch_later(db, user, waiting, asked)
|
||||
|
||||
|
||||
def run_import(user_id: int) -> dict:
|
||||
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||
@@ -191,15 +286,8 @@ def run_import(user_id: int) -> dict:
|
||||
return {"skipped": "no user"}
|
||||
result = service.apply_takeout_watched(db, user)
|
||||
db.commit()
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if waiting:
|
||||
fetcher = get_service_user(db)
|
||||
if fetcher is not None:
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
ensure_videos(db, yt, [vid for _, vid in waiting])
|
||||
result |= service.apply_takeout_watch_later(db, user)
|
||||
db.commit()
|
||||
result |= _put_in_watch_later(db, user)
|
||||
db.commit()
|
||||
outcomes = queries.takeout_outcomes(db, user)
|
||||
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||
@@ -208,17 +296,28 @@ def run_import(user_id: int) -> dict:
|
||||
|
||||
|
||||
def apply_waiting(db: Session) -> dict:
|
||||
"""Mark watched the waiting Takeout rows whose video the catalog has come to know and the Shorts
|
||||
probe has cleared since the import — every user's, one commit each. Run after each Shorts probe
|
||||
(the scheduler's `shorts` job), the only place a video's Short-ness becomes known. Returns its
|
||||
counts, or an empty dict when nothing was waiting on a probed video."""
|
||||
users = queries.takeout_users_ready(db)
|
||||
marked = 0
|
||||
for user_id in users:
|
||||
"""Settle what every user's import left waiting and can be settled now, one commit each: a
|
||||
watched video the catalog has come to know and the Shorts probe has cleared (by its job or a
|
||||
search's), and the Watch later videos a run could not put in yet. Run after each run of the
|
||||
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
|
||||
nothing was waiting that could move."""
|
||||
marked = added = 0
|
||||
ready = set(queries.takeout_users_ready(db))
|
||||
later = set(queries.takeout_users_watch_later_waiting(db))
|
||||
for user_id in sorted(ready | later):
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
continue
|
||||
with _one_import(user_id):
|
||||
marked += service.apply_takeout_watched(db, user)["watched"]
|
||||
db.commit()
|
||||
return {"takeout_watched": marked} if users else {}
|
||||
if user_id in ready:
|
||||
marked += service.apply_takeout_watched(db, user)["watched"]
|
||||
db.commit()
|
||||
if user_id in later:
|
||||
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
|
||||
db.commit()
|
||||
out = {}
|
||||
if ready:
|
||||
out["takeout_watched"] = marked
|
||||
if added:
|
||||
out["takeout_watch_later_added"] = added
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user