From 9c5f81fd944193c6249e1438a03259cd0a5f45bf Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 22:07:11 +0200 Subject: [PATCH] fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound - An ad-hoc request made while its target's run is under way is the run after it: the scheduler dropped a second fire of a running id, so a second Takeout file was never imported and its key stayed queued until a restart. - Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view no longer marks it watched again; un-marks take the user's Takeout lock. - One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the others are settled, and a failed lookup spends none of the daily share. - The upload reads the user's id before giving the read connection back. - A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a mark never pins it against the cleanup jobs. - The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and two uploads at once across all users (takeout_server_busy). - A Watch later list goes in as one presence read and one insert. --- .../alembic/versions/0114_video_unmarks.py | 44 +++ backend/app/bodylimit.py | 3 +- backend/app/ratelimit.py | 8 + backend/app/reader/models.py | 15 + backend/app/reader/queries.py | 42 +++ backend/app/reader/service.py | 53 +++- backend/app/reader/takeout.py | 156 ++++++++--- backend/app/routes/feed.py | 29 +- backend/app/scheduler.py | 73 ++++- backend/tests/conftest.py | 12 + backend/tests/test_scheduler.py | 35 +++ backend/tests/test_takeout.py | 258 +++++++++++++++++- backend/tests/test_unit_fences.py | 1 + frontend/src/i18n/locales/en/errors.json | 1 + frontend/src/i18n/locales/en/scheduler.json | 3 +- frontend/src/i18n/locales/hu/errors.json | 1 + frontend/src/i18n/locales/hu/scheduler.json | 3 +- 17 files changed, 667 insertions(+), 70 deletions(-) create mode 100644 backend/alembic/versions/0114_video_unmarks.py diff --git a/backend/alembic/versions/0114_video_unmarks.py b/backend/alembic/versions/0114_video_unmarks.py new file mode 100644 index 00000000..64e15403 --- /dev/null +++ b/backend/alembic/versions/0114_video_unmarks.py @@ -0,0 +1,44 @@ +"""When a user last took a video back to "new" + +Revision ID: 0114_video_unmarks +Revises: 0113_takeout_items +Create Date: 2026-10-23 + +Un-marking or resetting a video leaves no `video_states` row, so this is what tells an import of the +user's YouTube history a video they un-marked here from one never touched. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0114_video_unmarks" +down_revision: str | None = "0113_takeout_items" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "video_unmarks", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "video_id", + sa.String(16), + sa.ForeignKey("videos.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("user_id", "video_id", name="uq_video_unmark"), + ) + op.create_index("ix_video_unmarks_video_id", "video_unmarks", ["video_id"]) + + +def downgrade() -> None: + op.drop_index("ix_video_unmarks_video_id", table_name="video_unmarks") + op.drop_table("video_unmarks") diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index 432c6e1c..43e88f03 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -75,7 +75,8 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390 # bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an # account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The -# file is read an entry at a time (`takeout._json_list`), so it costs its own size, not a tree of it. +# file's text is decoded a window at a time and read an entry at a time (`takeout._json_list`), so +# it costs its bytes and a window, not a tree of it; the route takes two at once across every user. TAKEOUT_MAX_BYTES = 32 * 1024 * 1024 ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { diff --git a/backend/app/ratelimit.py b/backend/app/ratelimit.py index 6dcef515..313409e5 100644 --- a/backend/app/ratelimit.py +++ b/backend/app/ratelimit.py @@ -42,6 +42,14 @@ class RateLimiter: with self._lock: return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events + def room(self, key: str) -> float: + """How much cost ``key`` may still spend in the current window, WITHOUT counting anything — + for work counted only once it has succeeded: ask it first, ``allow`` what was done.""" + cutoff = time.monotonic() - self.window + with self._lock: + used = sum(c for t, c in self._hits.get(key, []) if t > cutoff) + return max(0.0, self.max_events - used) + def allow(self, key: str, cost: float = 1) -> bool: """Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit. diff --git a/backend/app/reader/models.py b/backend/app/reader/models.py index e5e3f031..7ff5d7b1 100644 --- a/backend/app/reader/models.py +++ b/backend/app/reader/models.py @@ -85,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) +class VideoUnmark(Base): + """When the user last took a video back to "new" here — un-marked it, or reset it. That leaves + no `VideoState` (its absence is "new"), so without this row a later import of what they watched + on YouTube could not tell a video they un-marked from one never touched, and marked it watched + again. Kept by user and video; the video's own deletion takes it.""" + + __tablename__ = "video_unmarks" + __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True) + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + class Playlist(Base, TimestampMixin, UpdatedAtMixin): """A per-user named, ordered collection of videos from the shared catalog. diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 9ac01add..b06eb402 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -26,6 +26,7 @@ from app.reader.models import ( TakeoutItem, TakeoutKind, VideoState, + VideoUnmark, ) from app.userscope import owned_by, owned_row @@ -376,6 +377,16 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan # --- Google Takeout -------------------------------------------------------------------------- +def _settled_here() -> ColumnElement[bool]: + """A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a + discovery video only a search or an explore brought in. A mark is a `VideoState`, which the + cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the + video, and its explored channel, for good. Such a row waits until the video is the catalog's own + (its channel subscribed to) — or is gone.""" + subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists() + return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed)) + + def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]: """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts: (row id, video id, Takeout time, is Short).""" @@ -388,6 +399,7 @@ def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datet TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.outcome.is_(None), Video.shorts_probed, + _settled_here(), ) ).tuples() ) @@ -421,6 +433,35 @@ def known_video_ids(db: Session, ids: list[str]) -> set[str]: return known +def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]: + """When the user last took each of `ids` back to "new" here, for those they ever did.""" + out: dict[str, datetime] = {} + for start in range(0, len(ids), _IN_CHUNK): + rows = db.execute( + select(VideoUnmark.video_id, VideoUnmark.at).where( + owned_by(VideoUnmark.user_id, user), + VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]), + ) + ).tuples() + out.update((vid, at) for vid, at in rows) + return out + + +def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]: + """Which of `ids` the playlist already holds.""" + held: set[str] = set() + for start in range(0, len(ids), _IN_CHUNK): + chunk = ids[start : start + _IN_CHUNK] + held.update( + db.execute( + select(PlaylistItem.video_id).where( + PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk) + ) + ).scalars() + ) + return held + + def takeout_rows_besides( db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str] ) -> int: @@ -476,6 +517,7 @@ def takeout_users_ready(db: Session) -> list[int]: TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.outcome.is_(None), Video.shorts_probed, + _settled_here(), ) .distinct() ).scalars() diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index 4d396aee..08bbabc2 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -25,6 +25,7 @@ from app.reader.models import ( TakeoutKind, TakeoutOutcome, VideoState, + VideoUnmark, ) from app.sync.playlists import lock_playlists, recompute_dirty from app.userscope import owned_by @@ -206,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None: the in-progress state, not where the user left off); otherwise the row goes. The player fires this beside a checkpoint that may delete the same row: the caller's commit then raises `StaleDataError`, which is that lost race and nothing worse.""" + lock_takeout(db, user) # an import settling meanwhile reads the un-mark below, or comes first row = queries.video_state(db, user, video_id) if row is None: return @@ -214,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None: row.watched_at = None else: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() +def _remember_unmark(db: Session, user: User, video_id: str) -> None: + """Note that the user took the video back to "new" now: the state they set leaves no row, and + an import of their YouTube history must not mark it watched again (`apply_takeout_watched`).""" + now = datetime.now(timezone.utc) + db.execute( + pg_insert(VideoUnmark) + .values(user_id=user.id, video_id=video_id, at=now) + .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now}) + ) + + def mark(db: Session, user: User, video_id: str, status: str) -> None: """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then @@ -241,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None: def clear_state(db: Session, user: User, video_id: str) -> None: """The video back to pristine for this user: status, resume point and `watched_at` all go. Playlist membership lives elsewhere and stays.""" + lock_takeout(db, user) # as `unmark` row = queries.video_state(db, user, video_id) if row is not None: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() @@ -517,7 +533,8 @@ _TAKEOUT_LOCK_NS = 0x544F def lock_takeout(db: Session, user: User) -> None: """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an import's settling each lock many of the same rows, one statement after another and in no common - order, so without one lock over both the two can deadlock.""" + order, so without one lock over both the two can deadlock. An un-mark takes it too: the import + reads the un-marks before it marks, and one landing between the two would be overwritten.""" db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id))) @@ -558,15 +575,21 @@ def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None: def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one - past is not watching it), and a state the user set here wins: watched or hidden, or anything - changed after the Takeout time. The upsert's own condition is that rule, so a state set while - the import runs wins too.""" + past is not watching it), and a state the user set here wins: watched or hidden, anything + changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own + condition is that rule, so a state set while the import runs wins too.""" lock_takeout(db, user) short: list[int] = [] todo: dict[str, tuple[int, datetime]] = {} - for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user): + ready = queries.takeout_watched_ready(db, user) + unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready]) + kept: list[int] = [] + for row_id, video_id, at, is_short in ready: if is_short: short.append(row_id) + elif video_id in unmarked and unmarked[video_id] >= at: + # Un-marked here since they watched it there: theirs, as a state they set is. + kept.append(row_id) else: todo[video_id] = (row_id, at) marked: set[str] = set() @@ -593,7 +616,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: ).returning(VideoState.video_id) ).scalars() ) - kept = [row_id for vid, (row_id, _) in items if vid not in marked] + kept += [row_id for vid, (row_id, _) in items if vid not in marked] _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED) _settle(db, kept, TakeoutOutcome.KEPT) _settle(db, short, TakeoutOutcome.SHORT) @@ -610,17 +633,27 @@ def apply_takeout_watch_later( if not rows: return {} lock_takeout(db, user) - known = queries.known_video_ids(db, [vid for _, vid in rows]) + ids = [vid for _, vid in rows] + known = queries.known_video_ids(db, ids) pl = watch_later(db, user) + # What add_item does a video at a time, for the whole list at once: one presence read and one + # insert, not four statements a video (a full list is thousands). Locked before the read, as there. + next_position = next_item_position(db, pl.id) + held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known]) done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} for row_id, vid in rows: if vid not in known: if vid in asked: done[TakeoutOutcome.MISSING].append(row_id) - elif add_item(db, pl, vid, mark_dirty=False): - done[TakeoutOutcome.ADDED].append(row_id) - else: + elif vid in held: done[TakeoutOutcome.PRESENT].append(row_id) + else: + db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position)) + next_position += 1 + held.add(vid) + done[TakeoutOutcome.ADDED].append(row_id) + if done[TakeoutOutcome.ADDED]: + touch(pl) for outcome, ids in done.items(): _settle(db, ids, outcome) db.flush() diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py index 67231e33..35d5fc1d 100644 --- a/backend/app/reader/takeout.py +++ b/backend/app/reader/takeout.py @@ -15,6 +15,7 @@ Ads". The title is in the account's language, so nothing here reads it. A Short header row (in the account's language) and `,` rows. """ +import codecs import csv import io import json @@ -28,7 +29,7 @@ from itertools import chain from sqlalchemy.orm import Session -from app import quota +from app import logs, quota from app.bodylimit import TAKEOUT_MAX_BYTES from app.db import SessionLocal from app.models import User @@ -39,6 +40,8 @@ from app.sync.runner import get_service_user from app.sync.videos import ensure_videos from app.youtube.client import YouTubeClient +log = logs.get_logger(__name__) + # The largest file the upload takes (its edge cap and the route's own are one number). MAX_BYTES = TAKEOUT_MAX_BYTES # The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is @@ -150,44 +153,113 @@ def _skip(text: str, at: int) -> int: return at -def _json_list(text: str) -> Iterator[object]: - """The entries of a JSON list, one at a time: the whole tree is never built, so a file of a - million tiny entries costs its text and one entry, not a million dicts at once.""" +# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a +# character (one emoji in a title is enough), several times the file on top of its bytes. +_WINDOW = 1024 * 1024 + + +class _Window: + """The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on. + The bytes are the caller's, so only the window is this reader's memory.""" + + def __init__(self, body: bytes, start: int): + self._body, self._next = body, start + self._decoder = codecs.getincrementaldecoder("utf-8")() # strict: bad UTF-8 raises + self.text, self.at = "", 0 + + @property + def done(self) -> bool: + return self._next >= len(self._body) + + def more(self, at_least: int = 0) -> None: + """Decode the next window (at least `at_least` bytes), dropping what is read already.""" + self.text, self.at = self.text[self.at :], 0 + size = max(_WINDOW, at_least) + chunk = self._body[self._next : self._next + size] + self._next += len(chunk) + self.text += self._decoder.decode(chunk, final=self.done) + + def skip(self) -> None: + """Past the whitespace, reading on as far as it goes.""" + while True: + self.at = _skip(self.text, self.at) + if self.at < len(self.text) or self.done: + return + self.more() + + def startswith(self, char: str) -> bool: + self.skip() + return self.text.startswith(char, self.at) + + +def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None: + """The JSON value at `at` and where it ends, or None where none reads (yet).""" + try: + return decoder.raw_decode(text, at) + # silent-ok: not a failure here — the caller reads on, or refuses the file once it has all. + except json.JSONDecodeError: + return None + + +def _json_list(body: bytes, start: int) -> Iterator[object]: + """The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never + built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not + a million dicts at once.""" decoder = json.JSONDecoder() - at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw - if text.startswith("]", at): - at += 1 + window = _Window(body, start + 1) # past the "[" the caller saw + if window.startswith("]"): + window.at += 1 else: while True: - entry, at = decoder.raw_decode(text, at) + window.skip() + read = _decode(decoder, window.text, window.at) + # An entry cut by the window's edge fails, or (a number) reads short: read on and try + # again, with a window twice what is left, so one entry larger than any window is read + # in a few tries, not one a window. + if read is None or (read[1] == len(window.text) and not window.done): + if window.done: + raise ValueError("not a JSON list") + window.more(2 * (len(window.text) - window.at)) + continue + entry, window.at = read yield entry - at = _skip(text, at) - if text.startswith(",", at): - at = _skip(text, at + 1) - elif text.startswith("]", at): - at += 1 + if window.startswith(","): + window.at += 1 + elif window.startswith("]"): + window.at += 1 break else: raise ValueError("not a JSON list") - if _skip(text, at) != len(text): + window.skip() + if window.at != len(window.text): raise ValueError("text after the JSON list") +# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten +# times that is not one — and, unlike the history, it is decoded whole. +CSV_MAX_BYTES = 2 * 1024 * 1024 +_BOM = codecs.BOM_UTF8 + + def parse(body: bytes) -> Parsed: """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file with nothing to import in it.""" - try: - text = body.decode("utf-8-sig") - except UnicodeDecodeError: - raise TakeoutError("not_takeout") from None - stripped = text.lstrip() - if stripped.startswith(("{", "<")): + start = len(_BOM) if body.startswith(_BOM) else 0 + while start < len(body) and body[start] in b" \t\n\r\f\v": + start += 1 + first = body[start : start + 1] + if first in (b"{", b"<"): # Another JSON file of the export, or the history's HTML variant. raise TakeoutError("not_takeout") + if first != b"[" and len(body) - start > CSV_MAX_BYTES: + raise TakeoutError("too_many") try: + # A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file. parsed = ( - _history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped) + _history(_json_list(body, start)) + if first == b"[" + else _watch_later(body[start:].decode("utf-8")) ) # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit. @@ -200,10 +272,10 @@ def parse(body: bytes) -> Parsed: return parsed -# One import per user at a time: a second file uploaded while the first one's run is under way -# starts a run of its own (the first one's id has left the scheduler), and the two would settle the -# same rows. The second waits, then takes what the first left. One API process runs the scheduler, -# so a lock in this module holds it (as the channel backfill's does). +# One import per user at a time: a second file uploaded while the first one's run is under way runs +# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a +# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a +# lock in this module holds it (as the channel backfill's does). _user_locks: dict[int, threading.Lock] = {} _user_locks_guard = threading.Lock() @@ -266,11 +338,14 @@ def _put_in_watch_later(db: Session, user: User) -> dict: if ( fetcher is not None and not quota.backfill_reserve_reached() - and _lookups.allow(str(user.id), cost=len(ask)) + and _lookups.room(str(user.id)) >= len(ask) ): with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP): with YouTubeClient(db, fetcher) as yt: ensure_videos(db, yt, ask) + # Counted once done: a lookup that failed spent none of the share (one run per user at a + # time, so the room asked above is still there). + _lookups.allow(str(user.id), cost=len(ask)) asked = set(ask) # Only the rows read above: one a second upload staged meanwhile was never asked about. return service.apply_takeout_watch_later(db, user, waiting, asked) @@ -301,23 +376,36 @@ def apply_waiting(db: Session) -> dict: search's), and the Watch later videos a run could not put in yet. Run after each run of the `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when nothing was waiting that could move.""" - marked = added = 0 + marked = added = failed = 0 ready = set(queries.takeout_users_ready(db)) later = set(queries.takeout_users_watch_later_waiting(db)) for user_id in sorted(ready | later): user = db.get(User, user_id) if user is None: continue - with _one_import(user_id): - if user_id in ready: - marked += service.apply_takeout_watched(db, user)["watched"] - db.commit() - if user_id in later: - added += _put_in_watch_later(db, user).get("watch_later_added", 0) - db.commit() + # One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that + # runs this nor keeps the users after them waiting. What it did not settle waits. + try: + with _one_import(user_id): + if user_id in ready: + marked += service.apply_takeout_watched(db, user)["watched"] + db.commit() + if user_id in later: + added += _put_in_watch_later(db, user).get("watch_later_added", 0) + db.commit() + except Exception: + db.rollback() + failed += 1 + log.exception( + "Takeout settling failed for user %s", + user_id, + extra=logs.event("takeout.settle.failed", user_id=user_id), + ) out = {} if ready: out["takeout_watched"] = marked if added: out["takeout_watch_later_added"] = added + if failed: + out["takeout_failed"] = failed return out diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index db16714e..491a104b 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -309,38 +309,47 @@ def get_video_detail( # --- Google Takeout ------------------------------------------------------------------------------ -# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these -# bound what one account can make the process hold. The section uploads its files one after another. +# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are +# held whole while it is read (its text a window at a time), so these bound what the accounts can +# make the process hold. The section uploads its files one after another. _takeout_limiter = RateLimiter(max_events=10, window_seconds=3600) _takeout_uploading: set[int] = set() _takeout_uploading_guard = threading.Lock() +_TAKEOUT_AT_ONCE = 2 async def _takeout_body( request: Request, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> AsyncIterator[bytes]: # The sign-in check has run, and it was a SELECT: hand the connection back before a file of - # megabytes arrives, so a slow upload does not hold one `idle in transaction`. + # megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is + # taken first: the release expires the user, and reading it after would reload it, taking a + # connection straight back out. + uid = user.id release_read_connection(db) with _takeout_uploading_guard: - busy = user.id in _takeout_uploading - if not busy: - _takeout_uploading.add(user.id) - if busy or not _takeout_limiter.allow(str(user.id)): - if not busy: + busy = uid in _takeout_uploading + full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE + taken = not busy and not full + if taken: + _takeout_uploading.add(uid) + if not taken or not _takeout_limiter.allow(str(uid)): + if taken: with _takeout_uploading_guard: - _takeout_uploading.discard(user.id) + _takeout_uploading.discard(uid) # Read to the end before answering: refused mid-upload, the browser reports a network # failure instead of the refusal (capped_stream_to says why). await drain_body(request) if busy: raise refused(429, "takeout_busy") + if full: + raise refused(429, "takeout_server_busy") raise refused(429, "takeout_cooldown") try: yield await capped_body(request, takeout.MAX_BYTES) finally: with _takeout_uploading_guard: - _takeout_uploading.discard(user.id) + _takeout_uploading.discard(uid) @router.post("/takeout") diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 39dd3797..e23eff0b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -637,6 +637,15 @@ def ad_hoc_target_title(db, job: str, target: str) -> str | None: # run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too. _queued: set[str] = set() _queued_guard = threading.Lock() +# What the scheduler is told to run is only the key: the request itself waits here, the newest one +# per key, until a fire of that key takes it. A request made while its key's run is under way is so +# kept for the run after it, which that run asks for as it ends (`_running`): the scheduler would +# drop a second fire of an id whose run has not returned (one instance per job id). +_pending: dict[str, tuple[Callable[[], dict], int, bool]] = {} +_running: set[str] = set() +# Two at once, for the moment between a run letting go of its key and its thread returning: a fire +# then is what takes the next request, and is not dropped. +_AD_HOC_INSTANCES = 2 def ad_hoc_queued(job: str, target: str) -> bool: @@ -650,37 +659,77 @@ def run_ad_hoc( ) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so - a second request before the first has even started is that same run. `notify` posts its outcome - to the user's inbox, for work whose page does not watch it run.""" + a second request before the first has even started is that same run, and one made while it runs + is the run after it. `notify` posts its outcome to the user's inbox, for work whose page does not + watch it run.""" if job not in AD_HOC_JOBS: raise KeyError(job) # an unnamed job would show on the page as its bare id - sched = _scheduler - if sched is None: + if _scheduler is None: # Only between shutdown and exit: start_scheduler runs at every API startup. raise RuntimeError("the scheduler is not running") key = f"{job}@{target}" with _queued_guard: + _pending[key] = (fn, actor_id, notify) _queued.add(key) + running = key in _running + if running: + log.info("ad-hoc %s for %s runs again once the run under way ends", job, target) + return + _fire(job, target) + + +def _fire(job: str, target: str) -> None: + """Add the one-shot that takes `job@target`'s pending request. Raises (the request dropped) only + when the scheduler cannot take it.""" + key = f"{job}@{target}" + sched = _scheduler try: + if sched is None: + raise RuntimeError("the scheduler is not running") sched.add_job( - _ad_hoc, + _ad_hoc_fired, "date", - args=(job, fn, actor_id, target, notify), + args=(job, target), id=key, misfire_grace_time=None, + max_instances=_AD_HOC_INSTANCES, ) - # silent-ok: not a failure — the run for this target is already queued, and it does this work. + # silent-ok: not a failure — a fire of this key is already queued, and it takes this request. except ConflictingIdError: log.info("ad-hoc %s for %s already queued", job, target) except Exception: with _queued_guard: + _pending.pop(key, None) _queued.discard(key) raise +def _ad_hoc_fired(job: str, target: str) -> None: + key = f"{job}@{target}" + with _queued_guard: + request = _pending.pop(key, None) + if request is None: + return # an earlier fire took it + _running.add(key) + fn, actor_id, notify = request + try: + _ad_hoc(job, fn, actor_id, target, notify) + finally: + with _queued_guard: + _running.discard(key) + again = key in _pending + if again: + try: + _fire(job, target) + except Exception: + # The request made meanwhile is lost with the scheduler: shut down, nothing runs. + log.exception("ad-hoc %s for %s not run again", job, target) + + def _ad_hoc( job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False ) -> None: + key = f"{job}@{target}" try: run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) except Exception: @@ -688,9 +737,11 @@ def _ad_hoc( log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return finally: - # Recorded (or never will be): the run row now speaks for it. + # Recorded (or never will be): the run row now speaks for it — unless another request came + # meanwhile, which is still queued. with _queued_guard: - _queued.discard(f"{job}@{target}") + if key not in _pending: + _queued.discard(key) _opened(run_id, job) try: _ad_hoc_work(job, fn, run_id, actor_id if notify else None) @@ -765,4 +816,6 @@ def shutdown_scheduler() -> None: _scheduler.shutdown(wait=False) _scheduler = None with _queued_guard: - _queued.clear() # what it held will not run + # What it held will not run. + _queued.clear() + _pending.clear() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 01b9c1ce..66cb9a8b 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -327,3 +327,15 @@ def _reset_log_erasures(): from app import logs logs._erased.clear() + + +@pytest.fixture(autouse=True) +def _reset_ad_hoc_queue(): + """The scheduler's ad-hoc queue is module-level too: a request a test left pending would read as + a later test's user's import still queued (ids restart from 1).""" + from app import scheduler + + with scheduler._queued_guard: + scheduler._queued.clear() + scheduler._pending.clear() + scheduler._running.clear() diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index d3fa418e..cab4b7bf 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -6,6 +6,7 @@ through (at the top of _job): holding _activity_lock across the check-and-set me caller can hold a job's "running" claim at a time, and it's claimable again once released.""" import logging +import threading import time from datetime import datetime, timedelta, timezone @@ -426,6 +427,40 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next +def test_a_request_made_while_its_run_is_under_way_is_the_run_after_it( + db, make_user, live_scheduler +): + """The scheduler drops a second fire of an id whose run has not returned: the second Takeout file + of a pair, uploaded while the first one's import ran, was never imported, and its key stayed + queued until a restart (the page spun on "Processing…" for good).""" + user = make_user("again@example.com") + first_runs, release = threading.Event(), threading.Event() + calls = [] + + def first(): + first_runs.set() + assert release.wait(10) + calls.append("first") + return {"n": 1} + + def second(): + calls.append("second") + return {"n": 2} + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, first, actor_id=user.id, target="UC_again") + assert first_runs.wait(10) + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, second, actor_id=user.id, target="UC_again") + assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + release.set() + + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2: + time.sleep(0.05) + assert calls == ["first", "second"] + assert sorted(r.result["n"] for r in _finished(db, "channel_backfill")) == [1, 2] + assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + + def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): """The scheduler's pool threads are reused: an actor a manual run left set made the next scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py index 823b5d3e..d63f3475 100644 --- a/backend/tests/test_takeout.py +++ b/backend/tests/test_takeout.py @@ -7,12 +7,15 @@ import tracemalloc from datetime import datetime, timedelta, timezone import pytest +from fastapi import Depends from sqlalchemy import select, text from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session from app import jobruns, quota, scheduler -from app.db import SessionLocal -from app.catalog.models import Channel, Video +from app.auth import current_user +from app.db import SessionLocal, get_db +from app.catalog.models import Channel, Subscription, Video from app.models import JobRun, Notification, User from app.runstatus import RunStatus, RunTrigger from app.reader import queries, service, takeout @@ -145,6 +148,42 @@ def test_the_history_is_read_an_entry_at_a_time_not_as_one_tree(): assert peak < 5 * len(body), f"parsing held {peak / len(body):.1f} times the file" +def test_a_history_with_an_emoji_costs_less_than_its_own_size_to_read(monkeypatch): + """Decoded whole, one emoji makes the text four bytes a character: a 32 MiB history held some + 128 MiB of it on top of its bytes. Read a window at a time, it holds a window (a few times its + bytes as text) whatever the file's size — a small one here, so a 9 MB file can show it.""" + monkeypatch.setattr(takeout, "_WINDOW", 64 * 1024) + view = _view("aaaaaaaaaaa", title="Watched " + "😀" * 40) + body = b"[" + b",".join([json.dumps(view, ensure_ascii=False).encode()] * 20_000) + b"]" + tracemalloc.start() + try: + parsed = takeout.parse(body) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert parsed.entries == 20_000 + assert peak < len(body) / 2, f"parsing held {peak / len(body):.1f} times the file" + + +@pytest.mark.parametrize("split", [1, 7, 100, 4096]) +def test_the_history_reads_the_same_whatever_its_window(monkeypatch, split): + """An entry, a character or a number cut by the window's edge is read whole.""" + monkeypatch.setattr(takeout, "_WINDOW", split) + views = [_view(f"{c * 11}", title="ő😀" * 3) for c in "abcdef"] + body = " [\n" + ",\n".join(json.dumps(v, ensure_ascii=False) for v in views) + "\n]\n" + parsed = takeout.parse(body.encode()) + assert (parsed.entries, sorted(parsed.videos)) == (6, [c * 11 for c in "abcdef"]) + numbers = takeout._json_list(b"[123456789, 2]", 0) + assert list(numbers) == [123456789, 2] + + +def test_a_watch_later_file_far_larger_than_any_list_is_refused_unread(): + body = b"Video ID,Time\n" + b"x" * (takeout.CSV_MAX_BYTES) + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == "too_many" + + def test_the_id_lists_are_asked_in_chunks(db, make_user, monkeypatch): user = make_user("chunks@example.com") ids = [f"chunk{n:06d}" for n in range(5)] @@ -253,6 +292,33 @@ def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_o assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40) +@pytest.mark.parametrize("undo", ["unmark", "clear_state"]) +def test_a_video_the_user_un_marked_here_is_not_marked_again_by_an_older_view(db, make_user, undo): + """Un-marking (or resetting) a video leaves no state row, and the import inserted a watched one + over the user's own choice. A view on YouTube after the un-mark is a new view, and counts.""" + user = make_user(f"{undo}@example.com") + for vid in ("undone00000", "again000000"): + _video(db, vid) + db.commit() + for vid in ("undone00000", "again000000"): + service.mark(db, user, vid, "watched") + getattr(service, undo)(db, user, vid) + db.commit() + assert _states(db, user) == {} + later = datetime.now(timezone.utc) + timedelta(minutes=5) + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"undone00000": (T0, None), "again000000": (later, None)}, + ) + db.commit() + + assert takeout.run_import(user.id)["watched"] == 1 + assert set(_states(db, user)) == {"again000000"} + assert _rows(db, user)["undone00000"].outcome == "kept" + + def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user): user = make_user("again@example.com") _video(db, "plain000000") @@ -305,6 +371,40 @@ def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_hol assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl] +@pytest.mark.parametrize("n", [3, 40]) +def test_a_watch_later_list_goes_in_in_statements_that_do_not_grow_with_it(db, make_user, n): + """A video at a time, a full list of 5000 was some 20,000 statements under the user's lock.""" + from sqlalchemy import event + + user = make_user(f"many{n}@example.com") + ids = [f"wl{i:09d}" for i in range(n)] + for vid in ids: + _video(db, vid) + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + {v: (T0 + timedelta(minutes=i), None) for i, v in enumerate(ids)}, + ) + db.commit() + rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + service.watch_later(db, user) # the list made beforehand: its own insert is not this count's + db.flush() + statements = [] + engine = db.get_bind() + listener = lambda *a: statements.append(a[2]) # noqa: E731 — the event's own signature + event.listen(engine, "before_cursor_execute", listener) + try: + assert service.apply_takeout_watch_later(db, user, rows, set())["watch_later_added"] == n + finally: + event.remove(engine, "before_cursor_execute", listener) + db.commit() + assert len(statements) <= 9, statements + held = [i.video_id for i in queries.items(db, service.watch_later(db, user).id)] + assert held == ids + + def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( db, make_user, monkeypatch ): @@ -407,6 +507,38 @@ def test_the_shorts_job_puts_in_the_watch_later_videos_a_run_could_not(db, make_ assert takeout.apply_waiting(db) == {} +def test_one_users_failure_in_the_shorts_job_leaves_the_others_and_the_probe_alone( + db, make_user, monkeypatch +): + """A Watch later lookup that raised (a revoked token, a refused quota) escaped the Shorts job: + its run was recorded as failed with its own counts lost, and every user after this one waited.""" + first, second = make_user("first@example.com"), make_user("second@example.com") + _video(db, "ready000000") + db.commit() + service.stage_takeout(db, first, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)}) + service.stage_takeout(db, second, TakeoutKind.WATCHED, {"ready000000": (T0, None)}) + db.commit() + + def refused(session, yt, ids): + raise RuntimeError("invalid_grant") + + monkeypatch.setattr(takeout, "ensure_videos", refused) + monkeypatch.setattr(takeout, "get_service_user", lambda session: first) + monkeypatch.setattr(scheduler, "run_shorts", lambda session: {"probed": 4}) + + scheduler._shorts_job() + + db.expire_all() + run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one() + assert run.status == "ok" + assert run.result == {"probed": 4, "takeout_watched": 1, "takeout_failed": 1} + assert _states(db, second)["ready000000"].status == "watched" + assert _rows(db, first)["WL:unknown0000"].outcome is None + # A lookup that failed spent none of the user's daily share. + assert not takeout._lookups.exhausted(str(first.id)) + assert takeout._lookups.room(str(first.id)) == 2 * takeout.WATCH_LATER_MAX + + def test_a_watch_later_row_staged_while_youtube_is_asked_is_left_for_the_next_run( db, make_user, monkeypatch ): @@ -469,6 +601,9 @@ def test_staging_and_settling_take_the_users_one_lock(db, make_user): "watch later": lambda s, u: service.apply_takeout_watch_later( s, u, [(1, "aaaaaaaaaaa")], set() ), + # Between the import's read of the un-marks and its marking, one would be overwritten. + "unmark": lambda s, u: service.unmark(s, u, "aaaaaaaaaaa"), + "reset": lambda s, u: service.clear_state(s, u, "aaaaaaaaaaa"), } with SessionLocal() as holder: service.lock_takeout(holder, holder.get(User, user.id)) @@ -570,6 +705,39 @@ def test_a_signed_in_upload_has_its_body_read(db, make_user, as_user, queued): assert len(pulled) == 4 +def test_the_upload_arrives_with_no_connection_held( + db, make_user, as_user, override_dep, queued, monkeypatch +): + """The read connection is given back before the body: reading the user's id after that reloaded + the user and took one straight back out, `idle in transaction` for the whole upload.""" + user = make_user("idle@example.com") + client = as_user(user) + + def signed_in(session: Session = Depends(get_db)) -> User: + # As the real sign-in check: a SELECT on the request's session, which the release ends — + # and expires what it loaded. (`as_user`'s own resolver reads nothing.) + session.expire_all() + return session.get(User, user.id) + + override_dep(current_user, signed_in) + sessions, held = [], [] + release, capped = feed_routes.release_read_connection, feed_routes.capped_body + + def releasing(session): + sessions.append(session) + release(session) + + async def reading(request, limit): + held.extend(s.in_transaction() for s in sessions) + return await capped(request, limit) + + monkeypatch.setattr(feed_routes, "release_read_connection", releasing) + monkeypatch.setattr(feed_routes, "capped_body", reading) + resp = client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 200, resp.text + assert held == [False] + + def test_uploads_past_the_hourly_limit_are_refused_after_their_body_arrived( db, make_user, as_user, queued ): @@ -600,6 +768,20 @@ def test_a_second_upload_while_one_of_the_users_is_still_arriving_is_refused( assert feed_routes._takeout_uploading == {user.id} +def test_a_third_upload_across_users_while_two_arrive_is_refused_after_its_body( + db, make_user, as_user, queued, monkeypatch +): + """Each upload's bytes are held whole while it arrives: with no bound across users, a few at + once could take the process past its memory.""" + monkeypatch.setattr(feed_routes, "_takeout_uploading", {9001, 9002}) + body, pulled = _counting_body() + resp = as_user(make_user("third@example.com")).post("/api/takeout", content=body) + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "takeout_server_busy" + assert len(pulled) == 4 + assert queued == [] and feed_routes._takeout_uploading == {9001, 9002} + + def test_an_upload_that_would_keep_too_many_videos_is_refused( db, make_user, as_user, queued, monkeypatch ): @@ -679,12 +861,49 @@ def test_a_queued_import_is_told_apart_from_the_last_one_until_its_run_is_record assert scheduler.ad_hoc_queued("takeout_import", target) assert as_user(user).get("/api/takeout").json()["queued"] is True - scheduler._ad_hoc("takeout_import", dict, user.id, target) + scheduler._ad_hoc_fired("takeout_import", target) assert not scheduler.ad_hoc_queued("takeout_import", target) assert as_user(user).get("/api/takeout").json()["queued"] is False assert added == [f"takeout_import@{target}"] +def test_an_upload_while_its_import_starts_stays_queued_for_the_run_after( + db, make_user, monkeypatch +): + """A request landing while the run before it writes its row is the run after that one: still + queued, so the page does not take the run starting now for it.""" + user = make_user("starting@example.com") + added, ran = [], [] + + class Held: + def add_job(self, *args, **kwargs): + added.append(kwargs["id"]) + + monkeypatch.setattr(scheduler, "_scheduler", Held()) + target = takeout.run_target(user.id) + start = jobruns.start + + def starting(*args, **kwargs): + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(2) or {}, actor_id=user.id, target=target + ) + return start(*args, **kwargs) + + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(1) or {}, actor_id=user.id, target=target + ) + monkeypatch.setattr(scheduler.jobruns, "start", starting) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1] + assert scheduler.ad_hoc_queued("takeout_import", target) + assert added == [f"takeout_import@{target}"] * 2 # the run after it, asked for as it ended + + monkeypatch.setattr(scheduler.jobruns, "start", start) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1, 2] + assert not scheduler.ad_hoc_queued("takeout_import", target) + + # --- the videos that wait ---------------------------------------------------------------------- @@ -711,6 +930,39 @@ def test_waiting_videos_are_marked_once_the_shorts_probe_has_cleared_them(db, ma assert takeout.apply_waiting(db) == {} +def test_a_discovery_video_is_not_marked_until_the_catalog_keeps_it(db, make_user): + """A mark is a VideoState, which the cleanup jobs read as a reader keeping the video: on a video + only someone's search or explore brought in, it kept that video (and its channel) for good.""" + user = make_user("discover@example.com") + for vid, flag in (("search00000", "via_search"), ("explore0000", "via_explore")): + _video(db, vid) + db.flush() + setattr(db.get(Video, vid), flag, True) + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {v: (T0, None) for v in ("search00000", "explore0000", "plain000000")}, + ) + db.commit() + + assert takeout.apply_waiting(db) == {"takeout_watched": 1} + assert set(_states(db, user)) == {"plain000000"} + assert {v for v, r in _rows(db, user).items() if r.outcome is None} == { + "search00000", + "explore0000", + } + + # Subscribed to: the catalog keeps the channel's videos of its own, the search one included. + db.add(Subscription(user_id=user.id, channel_id=CHANNEL)) + db.get(Video, "explore0000").via_explore = False # as subscribing does (catalog.service) + db.commit() + assert takeout.apply_waiting(db) == {"takeout_watched": 2} + assert set(_states(db, user)) == {"plain000000", "search00000", "explore0000"} + + def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_user): user = make_user("job-shorts@example.com") _video(db, "ready000000") # probed already: the job's probe has nothing left to do diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index 14f11f33..c189977e 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -45,6 +45,7 @@ UNIT_MODELS = { "SavedView", "ComfortPin", "TakeoutItem", + "VideoUnmark", "Tag", "ChannelTag", }, diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index c9066502..c5d7d4dd 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -254,6 +254,7 @@ "takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.", "takeout_no_videos": "This file has no videos to import", "takeout_busy": "Another of your files is still being uploaded — wait for it to finish", + "takeout_server_busy": "Other imports are being received right now — try again in a minute", "takeout_cooldown": "You have uploaded many files in the last hour — try again later", "takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds", "takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can" diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 1eafde92..f5b559c9 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -188,7 +188,8 @@ "watch_later_missing": "no longer on YouTube", "watch_later_waiting": "Watch later waiting", "takeout_watched": "marked watched from Takeout", - "takeout_watch_later_added": "put into Watch later from Takeout" + "takeout_watch_later_added": "put into Watch later from Takeout", + "takeout_failed": "Takeout users left for the next run after an error" }, "skip": { "disabled": "disabled", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index 6063a8e4..39b735b4 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -254,6 +254,7 @@ "takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.", "takeout_no_videos": "Ebben a fájlban nincs importálható videó", "takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik", + "takeout_server_busy": "Épp más importok érkeznek — próbáld újra egy perc múlva", "takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később", "takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van", "takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet" diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 2506ca42..7fa014aa 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -188,7 +188,8 @@ "watch_later_missing": "már nincs a YouTube-on", "watch_later_waiting": "a „Megnézem később” listára vár", "takeout_watched": "Takeoutból megnézettre jelölve", - "takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve" + "takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve", + "takeout_failed": "hiba miatt a következő futásra maradt Takeout-felhasználók" }, "skip": { "disabled": "kikapcsolva",