fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -7,6 +7,7 @@ length, whether a video exists); another unit asks the questions at the end of t
|
||||
than joining Reader's tables itself.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
@@ -407,11 +408,41 @@ def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[in
|
||||
)
|
||||
|
||||
|
||||
# Ids per IN list: a list is bound one parameter an id, and a statement takes at most 65,535.
|
||||
_IN_CHUNK = 5000
|
||||
|
||||
|
||||
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
|
||||
"""Which of `ids` the catalog holds."""
|
||||
if not ids:
|
||||
return set()
|
||||
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
|
||||
known: set[str] = set()
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
chunk = ids[start : start + _IN_CHUNK]
|
||||
known.update(db.execute(select(Video.id).where(Video.id.in_(chunk))).scalars())
|
||||
return known
|
||||
|
||||
|
||||
def takeout_rows_besides(
|
||||
db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str]
|
||||
) -> int:
|
||||
"""How many Takeout rows the user keeps other than `video_ids` of `kind`: what an upload of
|
||||
those videos would be added to."""
|
||||
total = (
|
||||
db.scalar(select(func.count(TakeoutItem.id)).where(owned_by(TakeoutItem.user_id, user)))
|
||||
or 0
|
||||
)
|
||||
ids = list(video_ids)
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
total -= (
|
||||
db.scalar(
|
||||
select(func.count(TakeoutItem.id)).where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == kind,
|
||||
TakeoutItem.video_id.in_(ids[start : start + _IN_CHUNK]),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
|
||||
@@ -449,3 +480,14 @@ def takeout_users_ready(db: Session) -> list[int]:
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def takeout_users_watch_later_waiting(db: Session) -> list[int]:
|
||||
"""Every user with a Watch later row a run could not put in yet."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.user_id)
|
||||
.where(TakeoutItem.kind == TakeoutKind.WATCH_LATER, TakeoutItem.outcome.is_(None))
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user