peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound

Browse Source 
- An ad-hoc request made while its target's run is under way is the run after it: the scheduler
  dropped a second fire of a running id, so a second Takeout file was never imported and its key
  stayed queued until a restart.
- Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view
  no longer marks it watched again; un-marks take the user's Takeout lock.
- One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the
  others are settled, and a failed lookup spends none of the daily share.
- The upload reads the user's id before giving the read connection back.
- A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a
  mark never pins it against the cleanup jobs.
- The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and
  two uploads at once across all users (takeout_server_busy).
- A Watch later list goes in as one presence read and one insert.
This commit is contained in:
peter
2026-10-23 22:07:11 +02:00
parent 9f580fd8c8
commit 9c5f81fd94
17 changed files with 667 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/reader/takeout.py
+122 -34
View File
@@ -15,6 +15,7 @@ Ads". The title is in the account's language, so nothing here reads it. A Short
header row (in the account's language) and `<id>,<ISO time added>` rows.
"""
import codecs
import csv
import io
import json
@@ -28,7 +29,7 @@ from itertools import chain
from sqlalchemy.orm import Session
from app import quota
from app import logs, quota
from app.bodylimit import TAKEOUT_MAX_BYTES
from app.db import SessionLocal
from app.models import User
@@ -39,6 +40,8 @@ from app.sync.runner import get_service_user
from app.sync.videos import ensure_videos
from app.youtube.client import YouTubeClient
log = logs.get_logger(__name__)
# The largest file the upload takes (its edge cap and the route's own are one number).
MAX_BYTES = TAKEOUT_MAX_BYTES
# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
@@ -150,44 +153,113 @@ def _skip(text: str, at: int) -> int:
return at
def _json_list(text: str) -> Iterator[object]:
"""The entries of a JSON list, one at a time: the whole tree is never built, so a file of a
million tiny entries costs its text and one entry, not a million dicts at once."""
# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a
# character (one emoji in a title is enough), several times the file on top of its bytes.
_WINDOW = 1024 * 1024
class _Window:
"""The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on.
The bytes are the caller's, so only the window is this reader's memory."""
def __init__(self, body: bytes, start: int):
self._body, self._next = body, start
self._decoder = codecs.getincrementaldecoder("utf-8")() # strict: bad UTF-8 raises
self.text, self.at = "", 0
@property
def done(self) -> bool:
return self._next >= len(self._body)
def more(self, at_least: int = 0) -> None:
"""Decode the next window (at least `at_least` bytes), dropping what is read already."""
self.text, self.at = self.text[self.at :], 0
size = max(_WINDOW, at_least)
chunk = self._body[self._next : self._next + size]
self._next += len(chunk)
self.text += self._decoder.decode(chunk, final=self.done)
def skip(self) -> None:
"""Past the whitespace, reading on as far as it goes."""
while True:
self.at = _skip(self.text, self.at)
if self.at < len(self.text) or self.done:
return
self.more()
def startswith(self, char: str) -> bool:
self.skip()
return self.text.startswith(char, self.at)
def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None:
"""The JSON value at `at` and where it ends, or None where none reads (yet)."""
try:
return decoder.raw_decode(text, at)
# silent-ok: not a failure here — the caller reads on, or refuses the file once it has all.
except json.JSONDecodeError:
return None
def _json_list(body: bytes, start: int) -> Iterator[object]:
"""The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never
built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not
a million dicts at once."""
decoder = json.JSONDecoder()
at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw
if text.startswith("]", at):
at += 1
window = _Window(body, start + 1) # past the "[" the caller saw
if window.startswith("]"):
window.at += 1
else:
while True:
entry, at = decoder.raw_decode(text, at)
window.skip()
read = _decode(decoder, window.text, window.at)
# An entry cut by the window's edge fails, or (a number) reads short: read on and try
# again, with a window twice what is left, so one entry larger than any window is read
# in a few tries, not one a window.
if read is None or (read[1] == len(window.text) and not window.done):
if window.done:
raise ValueError("not a JSON list")
window.more(2 * (len(window.text) - window.at))
continue
entry, window.at = read
yield entry
at = _skip(text, at)
if text.startswith(",", at):
at = _skip(text, at + 1)
elif text.startswith("]", at):
at += 1
if window.startswith(","):
window.at += 1
elif window.startswith("]"):
window.at += 1
break
else:
raise ValueError("not a JSON list")
if _skip(text, at) != len(text):
window.skip()
if window.at != len(window.text):
raise ValueError("text after the JSON list")
# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten
# times that is not one — and, unlike the history, it is decoded whole.
CSV_MAX_BYTES = 2 * 1024 * 1024
_BOM = codecs.BOM_UTF8
def parse(body: bytes) -> Parsed:
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
with nothing to import in it."""
try:
text = body.decode("utf-8-sig")
except UnicodeDecodeError:
raise TakeoutError("not_takeout") from None
stripped = text.lstrip()
if stripped.startswith(("{", "<")):
start = len(_BOM) if body.startswith(_BOM) else 0
while start < len(body) and body[start] in b" \t\n\r\f\v":
start += 1
first = body[start : start + 1]
if first in (b"{", b"<"):
# Another JSON file of the export, or the history's HTML variant.
raise TakeoutError("not_takeout")
if first != b"[" and len(body) - start > CSV_MAX_BYTES:
raise TakeoutError("too_many")
try:
# A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file.
parsed = (
_history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped)
_history(_json_list(body, start))
if first == b"["
else _watch_later(body[start:].decode("utf-8"))
)
# A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
# long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
@@ -200,10 +272,10 @@ def parse(body: bytes) -> Parsed:
return parsed
# One import per user at a time: a second file uploaded while the first one's run is under way
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
# so a lock in this module holds it (as the channel backfill's does).
# One import per user at a time: a second file uploaded while the first one's run is under way runs
# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a
# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a
# lock in this module holds it (as the channel backfill's does).
_user_locks: dict[int, threading.Lock] = {}
_user_locks_guard = threading.Lock()
@@ -266,11 +338,14 @@ def _put_in_watch_later(db: Session, user: User) -> dict:
if (
fetcher is not None
and not quota.backfill_reserve_reached()
and _lookups.allow(str(user.id), cost=len(ask))
and _lookups.room(str(user.id)) >= len(ask)
):
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
with YouTubeClient(db, fetcher) as yt:
ensure_videos(db, yt, ask)
# Counted once done: a lookup that failed spent none of the share (one run per user at a
# time, so the room asked above is still there).
_lookups.allow(str(user.id), cost=len(ask))
asked = set(ask)
# Only the rows read above: one a second upload staged meanwhile was never asked about.
return service.apply_takeout_watch_later(db, user, waiting, asked)
@@ -301,23 +376,36 @@ def apply_waiting(db: Session) -> dict:
search's), and the Watch later videos a run could not put in yet. Run after each run of the
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
nothing was waiting that could move."""
marked = added = 0
marked = added = failed = 0
ready = set(queries.takeout_users_ready(db))
later = set(queries.takeout_users_watch_later_waiting(db))
for user_id in sorted(ready | later):
user = db.get(User, user_id)
if user is None:
continue
with _one_import(user_id):
if user_id in ready:
marked += service.apply_takeout_watched(db, user)["watched"]
db.commit()
if user_id in later:
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
db.commit()
# One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that
# runs this nor keeps the users after them waiting. What it did not settle waits.
try:
with _one_import(user_id):
if user_id in ready:
marked += service.apply_takeout_watched(db, user)["watched"]
db.commit()
if user_id in later:
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
db.commit()
except Exception:
db.rollback()
failed += 1
log.exception(
"Takeout settling failed for user %s",
user_id,
extra=logs.event("takeout.settle.failed", user_id=user_id),
)
out = {}
if ready:
out["takeout_watched"] = marked
if added:
out["takeout_watch_later_added"] = added
if failed:
out["takeout_failed"] = failed
return out