fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+45 -3
View File
@@ -7,6 +7,7 @@ length, whether a video exists); another unit asks the questions at the end of t
than joining Reader's tables itself.
"""
from collections.abc import Iterable
from datetime import datetime
from sqlalchemy import ColumnElement, and_, func, or_, select
@@ -407,11 +408,41 @@ def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[in
)
# Ids per IN list: a list is bound one parameter an id, and a statement takes at most 65,535.
_IN_CHUNK = 5000
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
"""Which of `ids` the catalog holds."""
if not ids:
return set()
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
known: set[str] = set()
for start in range(0, len(ids), _IN_CHUNK):
chunk = ids[start : start + _IN_CHUNK]
known.update(db.execute(select(Video.id).where(Video.id.in_(chunk))).scalars())
return known
def takeout_rows_besides(
db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str]
) -> int:
"""How many Takeout rows the user keeps other than `video_ids` of `kind`: what an upload of
those videos would be added to."""
total = (
db.scalar(select(func.count(TakeoutItem.id)).where(owned_by(TakeoutItem.user_id, user)))
or 0
)
ids = list(video_ids)
for start in range(0, len(ids), _IN_CHUNK):
total -= (
db.scalar(
select(func.count(TakeoutItem.id)).where(
owned_by(TakeoutItem.user_id, user),
TakeoutItem.kind == kind,
TakeoutItem.video_id.in_(ids[start : start + _IN_CHUNK]),
)
)
or 0
)
return total
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
@@ -449,3 +480,14 @@ def takeout_users_ready(db: Session) -> list[int]:
.distinct()
).scalars()
)
def takeout_users_watch_later_waiting(db: Session) -> list[int]:
"""Every user with a Watch later row a run could not put in yet."""
return list(
db.execute(
select(TakeoutItem.user_id)
.where(TakeoutItem.kind == TakeoutKind.WATCH_LATER, TakeoutItem.outcome.is_(None))
.distinct()
).scalars()
)