From 7b8505af955973609afeb73689840008c4715b74 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 17:04:54 +0200 Subject: [PATCH 1/8] feat(reader): a Google Takeout export becomes watch state and Watch later The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`). --- .../alembic/versions/0113_takeout_items.py | 51 +++ backend/app/bodylimit.py | 5 + backend/app/reader/models.py | 49 +++ backend/app/reader/queries.py | 69 +++- backend/app/reader/service.py | 122 +++++- backend/app/reader/takeout.py | 200 ++++++++++ backend/app/routes/feed.py | 63 ++- backend/app/scheduler.py | 39 +- backend/tests/test_body_limit.py | 1 + backend/tests/test_takeout.py | 362 ++++++++++++++++++ backend/tests/test_unit_fences.py | 4 + frontend/src/i18n/locales/en/errors.json | 4 +- frontend/src/i18n/locales/en/scheduler.json | 17 +- frontend/src/i18n/locales/hu/errors.json | 4 +- frontend/src/i18n/locales/hu/scheduler.json | 17 +- 15 files changed, 978 insertions(+), 29 deletions(-) create mode 100644 backend/alembic/versions/0113_takeout_items.py create mode 100644 backend/app/reader/takeout.py create mode 100644 backend/tests/test_takeout.py diff --git a/backend/alembic/versions/0113_takeout_items.py b/backend/alembic/versions/0113_takeout_items.py new file mode 100644 index 00000000..8a901789 --- /dev/null +++ b/backend/alembic/versions/0113_takeout_items.py @@ -0,0 +1,51 @@ +"""A user's Google Takeout export: what they watched on YouTube, and their Watch later list + +Revision ID: 0113_takeout_items +Revises: 0112_one_watch_later +Create Date: 2026-10-23 + +One row per video and kind. A video the catalog does not know yet waits with no outcome, so the id +is not a foreign key; the partial index is what a newly known video looks its waiting rows up by. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0113_takeout_items" +down_revision: str | None = "0112_one_watch_later" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "takeout_items", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("kind", sa.String(12), nullable=False), + sa.Column("video_id", sa.String(16), nullable=False), + sa.Column("channel_id", sa.String(32), nullable=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.Column("outcome", sa.String(12), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + sa.CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + ) + op.create_index( + "ix_takeout_items_waiting", + "takeout_items", + ["video_id"], + postgresql_where=sa.text("outcome IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_takeout_items_waiting", table_name="takeout_items") + op.drop_table("takeout_items") diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index c0ecab4c..97b67eba 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -72,6 +72,10 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why # `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the # default, so listing it could only widen what it accepts. +# A Google Takeout file (`reader.takeout`): a real watch history of 9500 views, the 18 months Google +# keeps by default, is 3.7 MB; four times that is still well inside one request. +TAKEOUT_MAX_BYTES = 16 * 1024 * 1024 + ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # Ingesting a local file. The body IS the file now — no multipart envelope to allow for — # so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s @@ -95,6 +99,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports. ("POST", "/api/files/upload"): settings.download_upload_max_bytes, ("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES, + ("POST", "/api/takeout"): TAKEOUT_MAX_BYTES, } # The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders diff --git a/backend/app/reader/models.py b/backend/app/reader/models.py index f55bd23a..e5e3f031 100644 --- a/backend/app/reader/models.py +++ b/backend/app/reader/models.py @@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th edge, or asks this unit's service.""" from datetime import datetime +from enum import StrEnum from sqlalchemy import ( Boolean, @@ -213,3 +214,51 @@ class ComfortPin(Base, TimestampMixin): # 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder # endpoint rewrites wholesale beats a fractional index nobody would ever exercise here. position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False) + + +class TakeoutKind(StrEnum): + """What a Google Takeout row says the user did with a video.""" + + WATCHED = "watched" # history/watch-history.json + WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv + + +class TakeoutOutcome(StrEnum): + """What the import did with a row; NULL = still waiting for its video to be known (and, for a + watched one, probed for Shorts).""" + + WATCHED = "watched" # the video is marked watched at the Takeout time + KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays + SHORT = "short" # a Short: swiping one past is not watching it + ADDED = "added" # added to Watch later + PRESENT = "present" # already in Watch later + MISSING = "missing" # YouTube no longer has the video + + +class TakeoutItem(Base, TimestampMixin): + """One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch + later list (with when it was added). The rows are both the import job's input and its memory: a + video the catalog does not know yet waits here, with no outcome, until it enters the catalog. + The id is not a foreign key for that reason. The channel is kept for a later "channels you + watch but do not follow" list.""" + + __tablename__ = "takeout_items" + __table_args__ = ( + UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + # The waiting rows, by video: what a newly known video looks up. + Index( + "ix_takeout_items_waiting", + "video_id", + postgresql_where=text("outcome IS NULL"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + kind: Mapped[str] = mapped_column(String(12)) + video_id: Mapped[str] = mapped_column(String(16)) + channel_id: Mapped[str | None] = mapped_column(String(32)) + # Watched: the last time in the export; Watch later: when it was added. + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + outcome: Mapped[str | None] = mapped_column(String(12)) diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 453b7e88..ffb7302f 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -1,5 +1,5 @@ -"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and -for the other units that ask about them. +"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP +layer and for the other units that ask about them. Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and @@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t than joining Reader's tables itself. """ +from datetime import datetime + from sqlalchemy import ColumnElement, and_, func, or_, select from sqlalchemy.orm import InstrumentedAttribute, Session, aliased @@ -14,7 +16,16 @@ from app.collation import collated from app.feedquery import feed_columns, serialize from app.models import User from app.catalog.models import Channel, Subscription, Video -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + VideoState, +) from app.userscope import owned_by, owned_row # --- catalog, read along the edge ------------------------------------------------------------ @@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan owned_by(ChannelTag.user_id, user), ) ).scalar_one_or_none() + + +# --- Google Takeout -------------------------------------------------------------------------- + + +def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]: + """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts: + (row id, video id, Takeout time, is Short).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short) + .join(Video, Video.id == TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.outcome.is_(None), + Video.shorts_probed, + ) + ).tuples() + ) + + +def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]: + """The user's rows of `kind` the import has not settled, oldest first: (row id, video id).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == kind, + TakeoutItem.outcome.is_(None), + ) + .order_by(TakeoutItem.at, TakeoutItem.id) + ).tuples() + ) + + +def known_video_ids(db: Session, ids: list[str]) -> set[str]: + """Which of `ids` the catalog holds.""" + if not ids: + return set() + return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars()) + + +def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]: + """How many of the user's rows each kind and outcome holds (None = waiting).""" + rows = db.execute( + select(TakeoutItem.kind, TakeoutItem.outcome, func.count()) + .where(owned_by(TakeoutItem.user_id, user)) + .group_by(TakeoutItem.kind, TakeoutItem.outcome) + ).all() + return {(kind, outcome): n for kind, outcome, n in rows} diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index ca16ab49..7113eaae 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -15,7 +15,17 @@ from sqlalchemy.orm import Session from app.models import User from app.reader import queries -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + TakeoutOutcome, + VideoState, +) from app.sync.playlists import lock_playlists, recompute_dirty from app.userscope import owned_by @@ -495,3 +505,113 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No if row is not None: db.delete(row) db.flush() + + +# --- Google Takeout -------------------------------------------------------------------------- + +_TAKEOUT_BATCH = 1000 + + +def stage_takeout( + db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]] +) -> None: + """Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job. + A second upload of the same export, or a newer one, adds what is new: a video already kept keeps + the later time, and a row the import has settled keeps its outcome.""" + values = [ + {"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel} + for vid, (at, channel) in rows.items() + ] + for start in range(0, len(values), _TAKEOUT_BATCH): + stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH]) + db.execute( + stmt.on_conflict_do_update( + constraint="uq_takeout_item", + set_={ + "at": func.greatest(TakeoutItem.at, stmt.excluded.at), + "channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id), + }, + ) + ) + db.flush() + + +def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None: + for start in range(0, len(ids), _TAKEOUT_BATCH): + db.execute( + update(TakeoutItem) + .where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH])) + .values(outcome=outcome) + ) + + +def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: + """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed + for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one + past is not watching it), and a state the user set here wins: watched or hidden, or anything + changed after the Takeout time. The upsert's own condition is that rule, so a state set while + the import runs wins too.""" + short: list[int] = [] + todo: dict[str, tuple[int, datetime]] = {} + for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user): + if is_short: + short.append(row_id) + else: + todo[video_id] = (row_id, at) + marked: set[str] = set() + items = list(todo.items()) + now = datetime.now(timezone.utc) + for start in range(0, len(items), _TAKEOUT_BATCH): + stmt = pg_insert(VideoState).values( + [ + {"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at} + for vid, (_, at) in items[start : start + _TAKEOUT_BATCH] + ] + ) + marked.update( + db.execute( + stmt.on_conflict_do_update( + constraint="uq_user_video", + set_={ + "status": "watched", + "watched_at": stmt.excluded.watched_at, + "updated_at": now, + }, + where=(VideoState.status == "new") + & (VideoState.updated_at < stmt.excluded.watched_at), + ).returning(VideoState.video_id) + ).scalars() + ) + kept = [row_id for vid, (row_id, _) in items if vid not in marked] + _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED) + _settle(db, kept, TakeoutOutcome.KEPT) + _settle(db, short, TakeoutOutcome.SHORT) + db.flush() + return {"watched": len(marked), "kept": len(kept), "shorts": len(short)} + + +def apply_takeout_watch_later(db: Session, user: User) -> dict[str, int]: + """Put the videos of the user's YouTube Watch later list into theirs here, oldest first, after + what it already holds. Called once the catalog has fetched what it could: a video still unknown + is one YouTube no longer has.""" + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if not waiting: + return {} + known = queries.known_video_ids(db, [vid for _, vid in waiting]) + pl = watch_later(db, user) + done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} + for row_id, vid in waiting: + if vid not in known: + done[TakeoutOutcome.MISSING].append(row_id) + elif add_item(db, pl, vid, mark_dirty=False): + done[TakeoutOutcome.ADDED].append(row_id) + else: + done[TakeoutOutcome.PRESENT].append(row_id) + for outcome, ids in done.items(): + _settle(db, ids, outcome) + db.flush() + return { + "watch_later_added": len(done[TakeoutOutcome.ADDED]), + "watch_later_present": len(done[TakeoutOutcome.PRESENT]), + "watch_later_missing": len(done[TakeoutOutcome.MISSING]), + } diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py new file mode 100644 index 00000000..9a752a1e --- /dev/null +++ b/backend/app/reader/takeout.py @@ -0,0 +1,200 @@ +"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their +YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the +only way in. + +Two files of the export are read, each uploaded on its own: `history/watch-history.json` and +`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies +what it can; a video the catalog does not know yet waits for it. + +The format, measured on a real export (2026-10): the history is a JSON list, one entry per view, +newest first. A view of a video has a `titleUrl` with `watch?v=` (on www.youtube.com or +music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles` +entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google +Ads". The title is in the account's language, so nothing here reads it. A Short is a plain +`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a +header row (in the account's language) and `,` rows. +""" + +import csv +import io +import json +import re +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime + +from app import quota +from app.bodylimit import TAKEOUT_MAX_BYTES +from app.db import SessionLocal +from app.models import User +from app.reader import queries, service +from app.reader.models import TakeoutKind +from app.sync.runner import get_service_user +from app.sync.videos import ensure_videos +from app.youtube.client import YouTubeClient + +# The largest file the upload takes (its edge cap and the route's own are one number). +MAX_BYTES = TAKEOUT_MAX_BYTES + +_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") +_WATCH_URL = re.compile( + r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})" +) +_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})") + + +class TakeoutError(ValueError): + """The file is not one the import reads; `reason` is the client's word for why.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +@dataclass +class Parsed: + """One file's videos, `{video id: (time, channel id)}`, and what it left out, counted.""" + + kind: TakeoutKind + videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict) + entries: int = 0 + ads: int = 0 + other: int = 0 # not a view of a video (a post, a story) or a video with no link left + malformed: int = 0 + + +def _time(raw: object) -> datetime | None: + if not isinstance(raw, str): + return None + try: + at = datetime.fromisoformat(raw.replace("Z", "+00:00")) + # silent-ok: the row is counted as malformed, and the count is in the upload's answer. + except ValueError: + return None + return at if at.tzinfo is not None else None + + +def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None: + old = parsed.videos.get(vid) + if old is None or at > old[0]: + parsed.videos[vid] = (at, channel or (old[1] if old else None)) + + +def _history(data: list) -> Parsed: + parsed = Parsed(TakeoutKind.WATCHED) + for entry in data: + parsed.entries += 1 + if not isinstance(entry, dict): + parsed.malformed += 1 + continue + details = entry.get("details") + if isinstance(details, list) and any( + isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details + ): + parsed.ads += 1 + continue + url = entry.get("titleUrl") + match = _WATCH_URL.match(url) if isinstance(url, str) else None + if match is None: + parsed.other += 1 + continue + at = _time(entry.get("time")) + if at is None: + parsed.malformed += 1 + continue + channel = None + subtitles = entry.get("subtitles") + if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict): + found = _CHANNEL_URL.search(str(subtitles[0].get("url", ""))) + channel = found.group(1) if found else None + _keep(parsed, match.group(1), at, channel) + return parsed + + +def _watch_later(text: str) -> Parsed: + parsed = Parsed(TakeoutKind.WATCH_LATER) + rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)] + if not rows or len(rows[0]) < 2: + raise TakeoutError("not_takeout") + if not _VIDEO_ID.match(rows[0][0].strip()): + rows = rows[1:] # the header, in the account's language + for row in rows: + parsed.entries += 1 + vid = row[0].strip() if row else "" + at = _time(row[1].strip()) if len(row) > 1 else None + if not _VIDEO_ID.match(vid) or at is None: + parsed.malformed += 1 + continue + _keep(parsed, vid, at, None) + return parsed + + +def parse(body: bytes) -> Parsed: + """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told + apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file + with nothing to import in it.""" + try: + text = body.decode("utf-8-sig") + except UnicodeDecodeError: + raise TakeoutError("not_takeout") from None + stripped = text.lstrip() + if stripped.startswith("["): + try: + data = json.loads(stripped) + except json.JSONDecodeError: + raise TakeoutError("not_takeout") from None + if not isinstance(data, list): + raise TakeoutError("not_takeout") + parsed = _history(data) + elif stripped.startswith(("{", "<")): + # Another JSON file of the export, or the history's HTML variant. + raise TakeoutError("not_takeout") + else: + parsed = _watch_later(stripped) + if not parsed.videos: + raise TakeoutError("no_videos") + return parsed + + +# One import per user at a time: a second file uploaded while the first one's run is under way +# starts a run of its own (the first one's id has left the scheduler), and the two would settle the +# same rows. The second waits, then takes what the first left. One API process runs the scheduler, +# so a lock in this module holds it (as the channel backfill's does). +_user_locks: dict[int, threading.Lock] = {} +_user_locks_guard = threading.Lock() + + +@contextmanager +def _one_import(user_id: int) -> Iterator[None]: + with _user_locks_guard: + lock = _user_locks.setdefault(user_id, threading.Lock()) + with lock: + yield + + +def run_import(user_id: int) -> dict: + """The import job: apply what the user's kept Takeout rows can say now, as one recorded run. + The watched part commits before the Watch later part asks YouTube for the videos the catalog + lacks, so a YouTube failure there loses nothing already marked. Returns the counts.""" + with _one_import(user_id), SessionLocal() as db: + user = db.get(User, user_id) + if user is None: + return {"skipped": "no user"} + result = service.apply_takeout_watched(db, user) + db.commit() + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if waiting: + fetcher = get_service_user(db) + if fetcher is not None: + with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP): + with YouTubeClient(db, fetcher) as yt: + ensure_videos(db, yt, [vid for _, vid in waiting]) + result |= service.apply_takeout_watch_later(db, user) + db.commit() + outcomes = queries.takeout_outcomes(db, user) + result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0) + if outcomes.get((TakeoutKind.WATCH_LATER, None)): + result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)] + return result diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 001679a0..a8fd811a 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,18 +1,25 @@ from datetime import date +from functools import partial -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Request from pydantic import BaseModel from sqlalchemy.orm import Session from sqlalchemy.orm.exc import StaleDataError -from app import quota +from app import quota, scheduler from app.auth import current_user -from app.db import get_db +from app.db import get_db, release_read_connection from app.mailtext import lang_for_user from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years from app.models import User -from app.reader import queries, service -from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal +from app.reader import queries, service, takeout +from app.routes._common import ( + ProgressIn, + capped_body, + kept_position, + not_found, + youtube_refusal, +) from app.sync.videos import parse_iso8601_duration from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError @@ -293,3 +300,49 @@ def get_video_detail( items[0].get("contentDetails", {}).get("duration") ), } + + +# --- Google Takeout ------------------------------------------------------------------------------ + + +async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes: + # The sign-in check has run, and it was a SELECT: hand the connection back before a file of + # megabytes arrives, so a slow upload does not hold one `idle in transaction`. + release_read_connection(db) + return await capped_body(request, takeout.MAX_BYTES) + + +@router.post("/takeout") +def upload_takeout( + # The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before + # a byte of it is read. + user: User = Depends(current_user), + body: bytes = Depends(_takeout_body), + db: Session = Depends(get_db), +): + """One file of the user's Google Takeout export — the watch history or the Watch later list. + Its videos are kept and the import runs in the background; its outcome reaches the user's inbox.""" + try: + parsed = takeout.parse(body) + except takeout.TakeoutError as exc: + if exc.reason == "no_videos": + raise refused(422, "takeout_no_videos") from None + raise refused(422, "takeout_unreadable") from None + service.stage_takeout(db, user, parsed.kind, parsed.videos) + db.commit() + scheduler.run_ad_hoc( + scheduler.TAKEOUT_IMPORT, + partial(takeout.run_import, user.id), + actor_id=user.id, + target=f"user:{user.id}", + notify=True, + ) + return { + "ok": True, + "kind": parsed.kind, + "videos": len(parsed.videos), + "entries": parsed.entries, + "ads": parsed.ads, + "other": parsed.other, + "malformed": parsed.malformed, + } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index c2a3ae09..dc5e2506 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -118,8 +118,9 @@ def _notify_done( summary: str | None, result: dict | None = None, ) -> None: - """Post a completion notice to the triggering admin's inbox (manual runs only). Best - effort — a notification failure must never mask the job's own outcome.""" + """Post a completion notice to the inbox of who started the run: an admin's manual run, or a + user's ad-hoc run that asks for one. Best effort — a notification failure must never mask the + job's own outcome.""" try: create_notification( db, @@ -608,14 +609,19 @@ def trigger_all(actor_id: int | None = None) -> list[str]: # one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and # spent is in `job_runs` like any job's, not a background task only the log ever heard of. CHANNEL_BACKFILL = "channel_backfill" +# A user's Google Takeout import (`reader.takeout`): its target is the user. +TAKEOUT_IMPORT = "takeout_import" # Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). -AD_HOC_JOBS = (CHANNEL_BACKFILL,) +AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT) -def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: +def run_ad_hoc( + job: str, fn: Callable[[], dict], *, actor_id: int, target: str, notify: bool = False +) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so - a second request before the first has even started is that same run.""" + a second request before the first has even started is that same run. `notify` posts its outcome + to the user's inbox, for work whose page does not watch it run.""" if job not in AD_HOC_JOBS: raise KeyError(job) # an unnamed job would show on the page as its bare id sched = _scheduler @@ -626,7 +632,7 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) sched.add_job( _ad_hoc, "date", - args=(job, fn, actor_id, target), + args=(job, fn, actor_id, target, notify), id=f"{job}@{target}", misfire_grace_time=None, ) @@ -635,7 +641,9 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) log.info("ad-hoc %s for %s already queued", job, target) -def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None: +def _ad_hoc( + job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False +) -> None: try: run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) except Exception: @@ -644,12 +652,12 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non return _opened(run_id, job) try: - _ad_hoc_work(job, fn, run_id) + _ad_hoc_work(job, fn, run_id, actor_id if notify else None) finally: _closed(run_id) -def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: +def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | None) -> None: with logs.bind(run_id=run_id): # As in _run: only the work is inside the try. try: @@ -658,8 +666,19 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) err, failure = _failure(exc) _close(run_id, RunStatus.ERROR, error=err, failure=failure) + _notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure) return - _finish_ok(job, run_id, result) + summary, data = _finish_ok(job, run_id, result) + _notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data) + + +def _notify_ad_hoc( + user_id: int | None, job: str, status: RunStatus, summary: str | None, result: dict | None +) -> None: + if user_id is None: + return + with SessionLocal() as db: + _notify_done(db, user_id, job, status, summary, result) # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py index dc308d01..ad1f72c2 100644 --- a/backend/tests/test_body_limit.py +++ b/backend/tests/test_body_limit.py @@ -221,6 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes(): CAPPED_BODY_SITES = { ("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"), ("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"), + ("feed.py", "_takeout_body", "capped_body", "takeout.MAX_BYTES"), ("files.py", "spool_upload", "capped_stream_to", "cap"), } diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py new file mode 100644 index 00000000..39d65834 --- /dev/null +++ b/backend/tests/test_takeout.py @@ -0,0 +1,362 @@ +"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the +import job makes of them — watched here (never a Short, never over a state the user set), waiting for +a video the catalog does not know yet, and the Watch later list joined to the user's own.""" + +import json +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from app import scheduler +from app.catalog.models import Channel, Video +from app.models import Notification +from app.reader import service, takeout +from app.reader.models import ( + Playlist, + PlaylistItem, + TakeoutItem, + TakeoutKind, + VideoState, +) + +T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc) +CHANNEL = "UC" + "a" * 22 + + +def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra): + entry = { + "header": "YouTube", + "title": "Megnézted: valami", # the account's language: never read + "titleUrl": f"https://{host}.youtube.com/watch?v={vid}", + "time": at.isoformat().replace("+00:00", "Z"), + "products": ["YouTube"], + "activityControls": ["YouTube watch history"], + } + if channel: + entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}] + return entry | extra + + +def _history(*entries) -> bytes: + return json.dumps(list(entries)).encode() + + +# --- reading the files ------------------------------------------------------------------------- + + +def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel(): + later = T0 + timedelta(days=3) + parsed = takeout.parse( + _history( + _view("aaaaaaaaaaa", later), + _view("aaaaaaaaaaa", T0), + _view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel + _view("ccccccccccc", T0, host="music"), + # Out of order: the later view still wins. + _view("ddddddddddd", T0, channel=None), + _view("ddddddddddd", later), + ) + ) + assert parsed.kind == TakeoutKind.WATCHED + assert parsed.entries == 6 + assert parsed.videos == { + "aaaaaaaaaaa": (later, CHANNEL), + "bbbbbbbbbbb": (T0, None), + "ccccccccccc": (T0, CHANNEL), + "ddddddddddd": (later, CHANNEL), + } + + +def test_the_history_counts_what_it_leaves_out(): + ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}]) + post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"} + no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"} + bad_time = _view("ggggggggggg") | {"time": "yesterday"} + no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"} + parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk")) + assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2) + assert parsed.videos == { + "hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL) + } + + +def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none(): + body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n" + parsed = takeout.parse(("" + body).encode()) + assert parsed.kind == TakeoutKind.WATCH_LATER + assert parsed.videos == { + "Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None) + } + assert parsed.malformed == 1 + headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n") + assert list(headless.videos) == ["Ngb04xUK8uU"] + + +@pytest.mark.parametrize( + ("body", "reason"), + [ + (b'{"some": "other", "export": "file"}', "not_takeout"), + (b"watch history, as a page", "not_takeout"), + (b"[not json", "not_takeout"), + (b"\xff\xfe\x00", "not_takeout"), + (b"one column only\n", "not_takeout"), + (b"[]", "no_videos"), + ( + json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(), + "no_videos", + ), + ], +) +def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason): + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == reason + + +# --- keeping and applying ---------------------------------------------------------------------- + + +def _video(db, vid: str, *, probed=True, short=False): + if db.get(Channel, CHANNEL) is None: + db.add(Channel(id=CHANNEL, title="Ch")) + db.flush() + db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short)) + + +def _state(db, user, vid, status, *, changed, position=0, watched_at=None): + db.add( + VideoState( + user_id=user.id, + video_id=vid, + status=status, + position_seconds=position, + watched_at=watched_at, + updated_at=changed, + ) + ) + + +def _rows(db, user) -> dict[str, TakeoutItem]: + db.expire_all() + rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars() + return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows} + + +def _states(db, user) -> dict[str, VideoState]: + db.expire_all() + rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars() + return {r.video_id: r for r in rows} + + +def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept( + db, make_user +): + user = make_user("takeout@example.com") + for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"): + _video(db, vid, probed=vid != "unprobed000", short=vid == "short000000") + for vid in ("newer000000", "older000000"): + _video(db, vid) + earlier = T0 - timedelta(days=30) + _state(db, user, "hidden00000", "hidden", changed=earlier) + _state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier) + # In progress here, touched after the YouTube view: what the user did here is newer. + _state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40) + # In progress here, but last touched before the YouTube view: they finished it there. + _state(db, user, "older000000", "new", changed=earlier, position=40) + db.commit() + ids = [ + "plain000000", + "short000000", + "unprobed000", + "unknown0000", + "hidden00000", + "watched0000", + "newer000000", + "older000000", + ] + service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids}) + + counts = service.apply_takeout_watched(db, user) + db.commit() + + assert counts == {"watched": 2, "kept": 3, "shorts": 1} + rows, states = _rows(db, user), _states(db, user) + assert {v: rows[v].outcome for v in ids} == { + "plain000000": "watched", + "short000000": "short", + "unprobed000": None, + "unknown0000": None, + "hidden00000": "kept", + "watched0000": "kept", + "newer000000": "kept", + "older000000": "watched", + } + assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0) + assert "short000000" not in states + assert states["hidden00000"].status == "hidden" + assert states["watched0000"].watched_at == earlier + assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40) + assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40) + + +def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user): + user = make_user("again@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + service.apply_takeout_watched(db, user) + later = T0 + timedelta(days=1) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)}) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + db.commit() + row = _rows(db, user)["plain000000"] + assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched") + + +def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user): + user = make_user("later@example.com") + for vid in ("held0000000", "old00000000", "new00000000"): + _video(db, vid) + db.commit() + pl = service.watch_later(db, user) + service.add_item(db, pl, "held0000000", mark_dirty=False) + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + { + "new00000000": (T0, None), + "held0000000": (T0, None), + "old00000000": (T0 - timedelta(days=9), None), + "gone0000000": (T0, None), # YouTube no longer has it + }, + ) + + counts = service.apply_takeout_watch_later(db, user) + db.commit() + + assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1} + items = db.execute( + select(PlaylistItem.video_id) + .where(PlaylistItem.playlist_id == pl.id) + .order_by(PlaylistItem.position) + ).scalars() + assert list(items) == ["held0000000", "old00000000", "new00000000"] + assert _rows(db, user)["WL:gone0000000"].outcome == "missing" + assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl] + + +def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( + db, make_user, monkeypatch +): + user = make_user("job@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)}, + ) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)}) + db.commit() + asked = [] + + def fake_ensure(session, yt, ids): + asked.append(ids) + _video(session, "fetched0000") + session.commit() + + monkeypatch.setattr(takeout, "ensure_videos", fake_ensure) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + + result = takeout.run_import(user.id) + + assert asked == [["fetched0000"]] + assert result == { + "watched": 1, + "kept": 0, + "shorts": 0, + "watch_later_added": 1, + "watch_later_present": 0, + "watch_later_missing": 0, + "waiting": 1, + } + + +# --- the upload -------------------------------------------------------------------------------- + + +@pytest.fixture +def queued(monkeypatch): + runs = [] + monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw))) + return runs + + +def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice( + db, make_user, as_user, queued +): + user = make_user("upload@example.com") + body = _history( + _view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}]) + ) + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 200, resp.text + assert resp.json() == { + "ok": True, + "kind": "watched", + "videos": 1, + "entries": 2, + "ads": 1, + "other": 0, + "malformed": 0, + } + assert list(_rows(db, user)) == ["aaaaaaaaaaa"] + assert queued == [ + ( + "takeout_import", + {"actor_id": user.id, "target": f"user:{user.id}", "notify": True}, + ) + ] + + +@pytest.mark.parametrize( + ("body", "reason"), + [(b"", "takeout_unreadable"), (b"[]", "takeout_no_videos")], +) +def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language( + db, make_user, as_user, queued, body, reason +): + user = make_user("refused@example.com") + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "refused", "reason": reason} + assert queued == [] and _rows(db, user) == {} + + +def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch): + user = make_user("big@example.com") + monkeypatch.setattr(takeout, "MAX_BYTES", 64) + resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 413 + assert _rows(db, user) == {} + + +def test_an_anonymous_upload_is_refused(client): + assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401 + + +# --- the notice -------------------------------------------------------------------------------- + + +def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user): + user = make_user("notice@example.com") + scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True) + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x") + db.expire_all() + notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars() + assert [(n.data["job_id"], n.data["result"]) for n in notices] == [ + ("takeout_import", {"watched": 3}) + ] diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index 9bce2f17..f49b3a0d 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -44,6 +44,7 @@ UNIT_MODELS = { "PlaylistItem", "SavedView", "ComfortPin", + "TakeoutItem", "Tag", "ChannelTag", }, @@ -360,6 +361,9 @@ UNIT_COMMITS = { "sync/playlists.py:pull_user_playlists", "sync/playlists.py:merge_playlist", "sync/playlists.py:repull_playlist", + # The Takeout import is a recorded run with its own session: the watched part lands before + # the Watch later part asks YouTube (which commits the videos it fetched). + "reader/takeout.py:run_import", }, # The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page # fetched and stored is not lost when the next call fails, and an enrich that fails rolls back diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index 2c3afd48..d82745cc 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -249,6 +249,8 @@ "api_token_preset_grants": "The Hub token gets its folder by itself — leave the folders out", "api_token_no_grants": "Choose at least one folder the token may use", "api_token_name_required": "Give the token a name", - "api_token_limit": "You already have {{limit}} tokens — revoke one first" + "api_token_limit": "You already have {{limit}} tokens — revoke one first", + "takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.", + "takeout_no_videos": "This file has no videos to import" } } diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index f3a40d0a..7c1a7101 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -80,7 +80,8 @@ "log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.", "tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.", "people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.", - "channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running)." + "channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running).", + "takeout_import": "A user's Google Takeout import: what they watched on YouTube becomes watched here (not Shorts, never over a state they set here), and their YouTube Watch later list joins theirs. A video the catalogue does not know yet waits, and is marked once it arrives." }, "jobs": { "rss_poll": "RSS poll (new uploads)", @@ -100,7 +101,8 @@ "audit_gc": "Activity and quota-history cleanup", "log_gc": "Log cleanup", "people_enrich": "People details and portraits (Plex)", - "channel_backfill": "A channel's full history" + "channel_backfill": "A channel's full history", + "takeout_import": "Google Takeout import" }, "queue": { "recentPending": "Channels to sync", @@ -178,7 +180,13 @@ "deep": "deep", "bios": "biographies", "portraits": "portraits", - "cached": "cached" + "cached": "cached", + "kept": "kept as set here", + "waiting": "waiting for their video", + "watch_later_added": "added to Watch later", + "watch_later_present": "already in Watch later", + "watch_later_missing": "no longer on YouTube", + "watch_later_waiting": "Watch later waiting" }, "skip": { "disabled": "disabled", @@ -191,7 +199,8 @@ "busy": "another sync is running", "channelbusy": "this channel is already being backfilled", "atcapacity": "two full-history backfills are already running — the scheduler finishes this one", - "nothingtodo": "nothing left to fetch" + "nothingtodo": "nothing left to fetch", + "nouser": "the account is gone" }, "failure": { "rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index d0d54d8a..24351fc4 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -249,6 +249,8 @@ "api_token_preset_grants": "A Hub-token magától kapja a mappáját — ne adj meg mappát", "api_token_no_grants": "Válassz legalább egy mappát, amelyet a token használhat", "api_token_name_required": "Adj nevet a tokennek", - "api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet" + "api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet", + "takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.", + "takeout_no_videos": "Ebben a fájlban nincs importálható videó" } } diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index a6649a5f..244cf1b0 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -80,7 +80,8 @@ "log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.", "tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.", "people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.", - "channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be." + "channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be.", + "takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül az övébe. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz." }, "jobs": { "rss_poll": "RSS-lekérdezés (új feltöltések)", @@ -100,7 +101,8 @@ "audit_gc": "Napló- és kvótaelőzmény-takarítás", "log_gc": "Naplótakarítás", "people_enrich": "Személyadatok és portrék (Plex)", - "channel_backfill": "Egy csatorna teljes előzménye" + "channel_backfill": "Egy csatorna teljes előzménye", + "takeout_import": "Google Takeout import" }, "queue": { "recentPending": "Szinkronra váró csatorna", @@ -178,7 +180,13 @@ "deep": "mély", "bios": "életrajzok", "portraits": "portrék", - "cached": "elmentve" + "cached": "elmentve", + "kept": "itteni állapot megtartva", + "waiting": "a videójára vár", + "watch_later_added": "Watch laterbe téve", + "watch_later_present": "már a Watch laterben", + "watch_later_missing": "már nincs a YouTube-on", + "watch_later_waiting": "Watch later vár" }, "skip": { "disabled": "kikapcsolva", @@ -191,7 +199,8 @@ "busy": "egy másik szinkron fut", "channelbusy": "ennek a csatornának a betöltése már fut", "atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be", - "nothingtodo": "nincs több letöltendő" + "nothingtodo": "nincs több letöltendő", + "nouser": "a fiók már nem létezik" }, "failure": { "rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.", From f486a69ef26493a89baa457d515347179d8ec90c Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 17:10:38 +0200 Subject: [PATCH 2/8] fix(deps): brotli 1.2.1 (PYSEC-2026-4322) --- backend/requirements.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/requirements.lock b/backend/requirements.lock index 88f15315..487b8557 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -6,7 +6,7 @@ # bumped in place instead of a full regeneration, which would move every other package too — then # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403); -# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2). +# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322). APScheduler==3.11.3 Authlib==1.7.2 Mako==1.4.1 @@ -21,7 +21,7 @@ anyio==4.14.2 argon2-cffi-bindings==25.1.0 argon2-cffi==25.1.0 bgutil-ytdlp-pot-provider==1.3.1 -brotli==1.2.0 +brotli==1.2.1 certifi==2026.10.19 cffi==2.1.1 charset-normalizer==3.5.1 From 5d9952ab1e6cd83854afcb3b19020e8451b909d9 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 17:30:28 +0200 Subject: [PATCH 3/8] feat(reader): a Takeout video the catalog learns later is marked watched then; the import's status A watched video the catalog did not know at import time waits in `takeout_items`. The Shorts job now settles those rows right after its probe (the only place a video's Short-ness becomes known), every user's, a commit each: `reader.takeout.apply_waiting`, its count in the job's result. `GET /api/takeout` is what the settings page reads: the user's last import run (`jobruns.last_run`) and the kept videos by kind and outcome, "waiting" for one not known yet. --- backend/app/jobruns.py | 10 +++ backend/app/reader/queries.py | 27 +++++++ backend/app/reader/takeout.py | 24 ++++++ backend/app/routes/feed.py | 24 +++++- backend/app/scheduler.py | 8 +- backend/tests/test_takeout.py | 85 ++++++++++++++++++++- backend/tests/test_unit_fences.py | 2 + frontend/src/i18n/locales/en/scheduler.json | 5 +- frontend/src/i18n/locales/hu/scheduler.json | 5 +- 9 files changed, 181 insertions(+), 9 deletions(-) diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index 7beb7572..5596e651 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -119,6 +119,16 @@ def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[Jo return out +def last_run(db: Session, job: str, target: str) -> JobRun | None: + """The newest run of an ad-hoc `job` on `target` (a user's own import), or None.""" + return db.execute( + select(JobRun) + .where(JobRun.job == job, JobRun.target == target) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(1) + ).scalar_one_or_none() + + def gc(db: Session, retention_days: int) -> int: """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is never reaped. The caller commits.""" diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index ffb7302f..88a6550a 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -422,3 +422,30 @@ def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], in .group_by(TakeoutItem.kind, TakeoutItem.outcome) ).all() return {(kind, outcome): n for kind, outcome, n in rows} + + +def takeout_summary(db: Session, user: User) -> dict[str, dict[str, int]]: + """The user's kept Takeout videos by kind and outcome, for their settings page: `{"watched": + {"watched": n, "short": n, "waiting": n, …}, "watch_later": {…}}` — the import's outcome words, + "waiting" for a video not known yet.""" + out: dict[str, dict[str, int]] = {kind.value: {} for kind in TakeoutKind} + for (kind, outcome), n in takeout_outcomes(db, user).items(): + out[kind][outcome or "waiting"] = n + return out + + +def takeout_users_ready(db: Session) -> list[int]: + """Every user with a waiting "watched" row whose video is now known and probed for Shorts: the + rows a later run of the Shorts probe made ready.""" + return list( + db.execute( + select(TakeoutItem.user_id) + .join(Video, Video.id == TakeoutItem.video_id) + .where( + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.outcome.is_(None), + Video.shorts_probed, + ) + .distinct() + ).scalars() + ) diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py index 9a752a1e..7a7b12b9 100644 --- a/backend/app/reader/takeout.py +++ b/backend/app/reader/takeout.py @@ -25,6 +25,8 @@ from contextlib import contextmanager from dataclasses import dataclass, field from datetime import datetime +from sqlalchemy.orm import Session + from app import quota from app.bodylimit import TAKEOUT_MAX_BYTES from app.db import SessionLocal @@ -174,6 +176,11 @@ def _one_import(user_id: int) -> Iterator[None]: yield +def run_target(user_id: int) -> str: + """The import run's target, `job_runs.target`: one user's import.""" + return f"user:{user_id}" + + def run_import(user_id: int) -> dict: """The import job: apply what the user's kept Takeout rows can say now, as one recorded run. The watched part commits before the Watch later part asks YouTube for the videos the catalog @@ -198,3 +205,20 @@ def run_import(user_id: int) -> dict: if outcomes.get((TakeoutKind.WATCH_LATER, None)): result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)] return result + + +def apply_waiting(db: Session) -> dict: + """Mark watched the waiting Takeout rows whose video the catalog has come to know and the Shorts + probe has cleared since the import — every user's, one commit each. Run after each Shorts probe + (the scheduler's `shorts` job), the only place a video's Short-ness becomes known. Returns its + counts, or an empty dict when nothing was waiting on a probed video.""" + users = queries.takeout_users_ready(db) + marked = 0 + for user_id in users: + user = db.get(User, user_id) + if user is None: + continue + with _one_import(user_id): + marked += service.apply_takeout_watched(db, user)["watched"] + db.commit() + return {"takeout_watched": marked} if users else {} diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index a8fd811a..d6a41e24 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -6,7 +6,7 @@ from pydantic import BaseModel from sqlalchemy.orm import Session from sqlalchemy.orm.exc import StaleDataError -from app import quota, scheduler +from app import jobruns, quota, scheduler from app.auth import current_user from app.db import get_db, release_read_connection from app.mailtext import lang_for_user @@ -334,7 +334,7 @@ def upload_takeout( scheduler.TAKEOUT_IMPORT, partial(takeout.run_import, user.id), actor_id=user.id, - target=f"user:{user.id}", + target=takeout.run_target(user.id), notify=True, ) return { @@ -346,3 +346,23 @@ def upload_takeout( "other": parsed.other, "malformed": parsed.malformed, } + + +@router.get("/takeout") +def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)): + """What the user's Takeout import has done: its last run (none before the first upload), and + every video kept from the export, by kind and outcome — a video the catalog did not know yet + counts as "waiting" until it does, and is marked then.""" + run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, takeout.run_target(user.id)) + return { + "run": None + if run is None + else { + "status": run.status, + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), + "result": run.result, + "failure": run.failure, + }, + **queries.takeout_summary(db, user), + } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index dc5e2506..44d395ac 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -22,6 +22,7 @@ from app.plex.tmdb import run_people_enrich from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync from app.notifications import create_notification from app.runstatus import RunStatus, RunTrigger +from app.reader import takeout from app.state import is_sync_paused from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral @@ -403,7 +404,12 @@ def _autotag_job() -> None: def _shorts_job() -> None: - _job("shorts", run_shorts) + def work(db): + # A video waiting in someone's Takeout import becomes markable the moment the probe has + # said it is not a Short, so the import's waiting rows are settled right after it. + return run_shorts(db) | takeout.apply_waiting(db) + + _job("shorts", work) def _subscriptions_job() -> None: diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py index 39d65834..f603c306 100644 --- a/backend/tests/test_takeout.py +++ b/backend/tests/test_takeout.py @@ -8,9 +8,10 @@ from datetime import datetime, timedelta, timezone import pytest from sqlalchemy import select -from app import scheduler +from app import jobruns, scheduler from app.catalog.models import Channel, Video -from app.models import Notification +from app.models import JobRun, Notification +from app.runstatus import RunStatus, RunTrigger from app.reader import service, takeout from app.reader.models import ( Playlist, @@ -360,3 +361,83 @@ def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user assert [(n.data["job_id"], n.data["result"]) for n in notices] == [ ("takeout_import", {"watched": 3}) ] + + +# --- the videos that wait ---------------------------------------------------------------------- + + +def test_waiting_videos_are_marked_once_the_shorts_probe_has_cleared_them(db, make_user): + a, b = make_user("a@example.com"), make_user("b@example.com") + _video(db, "later000000", probed=False) + _video(db, "notyet00000", probed=False) + db.commit() + service.stage_takeout( + db, a, TakeoutKind.WATCHED, {"later000000": (T0, None), "notyet00000": (T0, None)} + ) + service.stage_takeout(db, b, TakeoutKind.WATCHED, {"later000000": (T0, None)}) + db.commit() + assert takeout.apply_waiting(db) == {} + + # The probe clears one of them; the other is still unprobed. + db.get(Video, "later000000").shorts_probed = True + db.commit() + assert takeout.apply_waiting(db) == {"takeout_watched": 2} + + assert _states(db, a)["later000000"].watched_at == T0 + assert _states(db, b)["later000000"].status == "watched" + assert _rows(db, a)["notyet00000"].outcome is None + assert takeout.apply_waiting(db) == {} + + +def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_user): + user = make_user("job-shorts@example.com") + _video(db, "ready000000") # probed already: the job's probe has nothing left to do + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"ready000000": (T0, None)}) + db.commit() + + scheduler._shorts_job() + + db.expire_all() + run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one() + assert run.result == {"probed": 0, "shorts": 0, "takeout_watched": 1} + assert _states(db, user)["ready000000"].status == "watched" + + +# --- what the settings page reads -------------------------------------------------------------- + + +def test_the_status_is_the_users_own_last_run_and_counts(db, make_user, as_user): + user, other = make_user("status@example.com"), make_user("other@example.com") + assert as_user(user).get("/api/takeout").json() == { + "run": None, + "watched": {}, + "watch_later": {}, + } + _video(db, "plain000000") + _video(db, "short000000", short=True) + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {v: (T0, None) for v in ("plain000000", "short000000", "unknown0000")}, + ) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"plain000000": (T0, None)}) + service.stage_takeout(db, other, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + service.apply_takeout_watched(db, user) + db.commit() + older = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}") + jobruns.finish(older, RunStatus.OK, summary=None, result={"watched": 0}) + newest = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}") + jobruns.finish(newest, RunStatus.OK, summary=None, result={"watched": 1}) + theirs = jobruns.start("takeout_import", RunTrigger.AD_HOC, other.id, f"user:{other.id}") + jobruns.finish(theirs, RunStatus.OK, summary=None, result={"watched": 99}) + + body = as_user(user).get("/api/takeout").json() + + assert body["run"]["status"] == "ok" + assert body["run"]["result"] == {"watched": 1} + assert body["run"]["finished_at"] is not None + assert body["watched"] == {"watched": 1, "short": 1, "waiting": 1} + assert body["watch_later"] == {"waiting": 1} diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index f49b3a0d..14f11f33 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -364,6 +364,8 @@ UNIT_COMMITS = { # The Takeout import is a recorded run with its own session: the watched part lands before # the Watch later part asks YouTube (which commits the videos it fetched). "reader/takeout.py:run_import", + # Run inside the Shorts job, on its session: each user's waiting rows land on their own. + "reader/takeout.py:apply_waiting", }, # The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page # fetched and stored is not lost when the next call fails, and an enrich that fails rolls back diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 7c1a7101..fbd6410b 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -67,7 +67,7 @@ "enrich": "Fetches video details (duration, views, live/Shorts flags) and re-checks live videos until they end. If it stops, videos stay without a duration/stats and ended livestreams stay stuck as “live”.", "backfill": "Pulls in recent uploads, then the full back-catalogue for opted-in channels, within the shared quota. If it stops, older videos and full history don't fill in.", "autotag": "Detects each channel's language/topics and applies automatic tags. If it stops, new channels get no auto tags, so language/topic filters miss them.", - "shorts": "Probes youtube.com/shorts to mark which videos are Shorts. If it stops, Shorts aren't separated from normal videos in the Feed.", + "shorts": "Probes youtube.com/shorts to mark which videos are Shorts, then marks watched the videos a Google Takeout import was waiting on. If it stops, Shorts aren't separated from normal videos in the Feed, and those videos keep waiting.", "subscriptions": "Re-imports every user's YouTube subscriptions. If it stops, channels subscribed to or dropped on YouTube aren't reflected here.", "playlist_sync": "Mirrors each user's YouTube playlists into the app. If it stops, changes to your YouTube playlists don't show up locally.", "maintenance": "Finds videos that can no longer be played (deleted, made private, abandoned premieres), hides them from the Feed and removes them after a grace period; re-checks the least-recently-checked videos. If it stops, dead videos linger in the catalogue.", @@ -186,7 +186,8 @@ "watch_later_added": "added to Watch later", "watch_later_present": "already in Watch later", "watch_later_missing": "no longer on YouTube", - "watch_later_waiting": "Watch later waiting" + "watch_later_waiting": "Watch later waiting", + "takeout_watched": "marked watched from Takeout" }, "skip": { "disabled": "disabled", diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 244cf1b0..8f04643d 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -67,7 +67,7 @@ "enrich": "Lekéri a videók adatait (hossz, megtekintés, élő/Shorts jelzők), és frissíti az élő videókat, amíg véget nem érnek. Ha leáll, a videók hossz/statisztika nélkül maradnak, a véget ért adások pedig „élő” állapotban ragadnak.", "backfill": "Behúzza a friss feltöltéseket, majd a kijelölt csatornák teljes előzményét, a közös kvótán belül. Ha leáll, a régebbi videók és a teljes előzmény nem töltődik fel.", "autotag": "Felismeri a csatornák nyelvét/témáit, és automatikus címkéket ad. Ha leáll, az új csatornák címke nélkül maradnak, így a nyelv/téma szűrők kihagyják őket.", - "shorts": "A youtube.com/shorts próbával jelöli, mely videók Shorts-ok. Ha leáll, a Shorts-ok nem különülnek el a normál videóktól a Feedben.", + "shorts": "A youtube.com/shorts próbával jelöli, mely videók Shorts-ok, utána megnézettre jelöli azokat a videókat, amelyekre egy Google Takeout import várt. Ha leáll, a Shorts-ok nem különülnek el a normál videóktól a Feedben, és ezek a videók tovább várnak.", "subscriptions": "Újraimportálja minden felhasználó YouTube-feliratkozásait. Ha leáll, a YouTube-on felvett vagy törölt feliratkozások nem tükröződnek itt.", "playlist_sync": "Tükrözi a felhasználók YouTube lejátszási listáit az appba. Ha leáll, a YouTube-listák változásai nem jelennek meg lokálisan.", "maintenance": "Megkeresi a már sehol le nem játszható videókat (törölt, priváttá tett, elhagyott premier), elrejti őket a Feedből, és türelmi idő után eltávolítja; újraellenőrzi a legrégebben ellenőrzött videókat. Ha leáll, a halott videók a katalógusban maradnak.", @@ -186,7 +186,8 @@ "watch_later_added": "Watch laterbe téve", "watch_later_present": "már a Watch laterben", "watch_later_missing": "már nincs a YouTube-on", - "watch_later_waiting": "Watch later vár" + "watch_later_waiting": "Watch later vár", + "takeout_watched": "Takeoutból megnézettre jelölve" }, "skip": { "disabled": "kikapcsolva", From 3553eac8c44c4066bc758bbf0f45dbe0dae93f05 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 18:51:05 +0200 Subject: [PATCH 4/8] =?UTF-8?q?feat(settings):=20upload=20a=20Google=20Tak?= =?UTF-8?q?eout=20export=20in=20Settings=20=E2=80=BA=20Account?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "YouTube history" section under YouTube access: the watch history and the Watch later csv are dropped or picked (one request each, each file's outcome said on its own), and the section then says what the import did - only the counts it produced - or that it is running, failed or was cut off by a restart. While it runs it asks again until it ends; a failed re-read shows the failure, not the previous answer. The drop zone the Files upload had inline is now ui/DropZone, used by both. The e2e seed clears the account's kept Takeout videos. --- frontend/e2e/takeout.spec.ts | 59 +++++ .../src/components/ui/DropZone.dom.test.tsx | 64 ++++++ frontend/src/components/ui/DropZone.tsx | 97 +++++++++ frontend/src/components/ui/README.md | 14 +- .../ui/pressableSelectable.contract.test.ts | 2 +- frontend/src/i18n/locales/en/errors.json | 1 + frontend/src/i18n/locales/en/settings.json | 35 +++ frontend/src/i18n/locales/hu/errors.json | 1 + frontend/src/i18n/locales/hu/settings.json | 35 +++ frontend/src/lib/api/feed.ts | 34 ++- frontend/src/lib/queryKeys.ts | 1 + frontend/src/modules/files/UploadDialog.tsx | 100 +++------ .../src/modules/settings/SettingsPanel.tsx | 3 + .../settings/TakeoutImport.dom.test.tsx | 203 ++++++++++++++++++ .../src/modules/settings/TakeoutImport.tsx | 179 +++++++++++++++ scripts/e2e_seed.py | 5 +- 16 files changed, 749 insertions(+), 84 deletions(-) create mode 100644 frontend/e2e/takeout.spec.ts create mode 100644 frontend/src/components/ui/DropZone.dom.test.tsx create mode 100644 frontend/src/components/ui/DropZone.tsx create mode 100644 frontend/src/modules/settings/TakeoutImport.dom.test.tsx create mode 100644 frontend/src/modules/settings/TakeoutImport.tsx diff --git a/frontend/e2e/takeout.spec.ts b/frontend/e2e/takeout.spec.ts new file mode 100644 index 00000000..1794f171 --- /dev/null +++ b/frontend/e2e/takeout.spec.ts @@ -0,0 +1,59 @@ +import { test, expect } from "@playwright/test"; +import { openApp, gotoPage } from "./helpers"; + +// Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real +// picker, is read, imported in the background, and the section says what it did — a video the +// catalog knows is marked watched, one it does not know is remembered for later. The seed clears +// the account's kept Takeout videos each run, but the projects share one account: a later one's +// feed no longer holds the video an earlier one marked, so what it adds is counted against what was +// there before it. +test("a Takeout watch history uploads, imports, and the section says what it did", async ({ + page, +}) => { + await openApp(page); + // A video the catalog knows: one on the account's feed (not a Short — the feed leaves them out). + const known = await page.getByTestId("video-card").first().getAttribute("data-video-id"); + expect(known).toBeTruthy(); + + await gotoPage(page, "settings"); + await page.getByRole("tab", { name: "Account" }).click(); + await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible(); + // The first project's upload is an import already made: its "New import" opens the drop zone. + const zone = page.getByRole("button", { name: "Drop the files here, or choose them" }); + const again = page.getByRole("button", { name: "New import" }); + await expect(zone.or(again)).toBeVisible(); + if (await again.isVisible()) await again.click(); + await expect(zone).toBeVisible(); + const before = (await (await page.request.get("/api/takeout")).json()) as { + watched?: Record; + }; + const watchedBefore = before.watched?.watched ?? 0; + + // Watched now: later than any state the seed set, which the import would otherwise leave alone. + const now = new Date().toISOString(); + const history = [ + { titleUrl: `https://www.youtube.com/watch?v=${known}`, time: now }, + { titleUrl: "https://www.youtube.com/watch?v=e2eUnknown0", time: now }, + { + titleUrl: "https://www.youtube.com/watch?v=e2eAdvert00", + time: now, + details: [{ name: "From Google Ads" }], + }, + ]; + const uploaded = page.waitForResponse( + (r) => r.url().endsWith("/api/takeout") && r.request().method() === "POST", + ); + await page.locator('input[type="file"][accept=".json,.csv"]').setInputFiles({ + name: "watch-history.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(history)), + }); + expect((await uploaded).ok()).toBeTruthy(); + + await expect(page.getByText("watch-history.json: 2 videos read")).toBeVisible(); + const line = page.getByTestId("takeout-last"); + await expect(line).toContainText("Last import:"); + await expect(line).toContainText(`${watchedBefore + 1} watched`); + await expect(line).toContainText("1 remembered"); + await expect(page.getByRole("button", { name: "New import" })).toBeVisible(); +}); diff --git a/frontend/src/components/ui/DropZone.dom.test.tsx b/frontend/src/components/ui/DropZone.dom.test.tsx new file mode 100644 index 00000000..affa0212 --- /dev/null +++ b/frontend/src/components/ui/DropZone.dom.test.tsx @@ -0,0 +1,64 @@ +// @vitest-environment jsdom +// +// ui/DropZone: a drop or a pick hands over the files (only the first unless `multiple`), the drag +// highlight survives the cursor crossing the zone's own children, and a disabled zone takes nothing. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { DropZone } from "./DropZone"; + +afterEach(cleanup); + +const files = (...names: string[]) => names.map((n) => new File(["x"], n)); +function mount(props: Partial[0]> = {}) { + const onFiles = vi.fn(); + render( + + {(over) => {over ? "over" : "idle"}} + , + ); + return { onFiles, zone: screen.getByRole("button"), inner: screen.getByTestId("inner") }; +} + +describe("DropZone", () => { + it("a drop hands over the first file, or all of them when multiple", () => { + const one = mount(); + fireEvent.drop(one.zone, { dataTransfer: { files: files("a", "b") } }); + expect(one.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a"]); + cleanup(); + const many = mount({ multiple: true }); + fireEvent.drop(many.zone, { dataTransfer: { files: files("a", "b") } }); + expect(many.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a", "b"]); + }); + + it("a pick hands over the picked files; an empty pick hands over nothing", () => { + const { onFiles } = mount({ multiple: true }); + const input = document.querySelector('input[type="file"]')!; + expect(input.multiple).toBe(true); + fireEvent.change(input, { target: { files: [] } }); + expect(onFiles).not.toHaveBeenCalled(); + fireEvent.change(input, { target: { files: files("c") } }); + expect(onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["c"]); + }); + + it("stays highlighted while the cursor crosses its own children, and clears on the last leave", () => { + const { zone, inner } = mount(); + fireEvent.dragEnter(zone); + fireEvent.dragEnter(inner); // into a child: the zone gets its leave after this enter + fireEvent.dragLeave(zone); + expect(inner.textContent).toBe("over"); + fireEvent.dragLeave(inner); + expect(inner.textContent).toBe("idle"); + }); + + it("a disabled zone takes no drop", () => { + const { zone, onFiles } = mount({ disabled: true }); + fireEvent.drop(zone, { dataTransfer: { files: files("a") } }); + expect(onFiles).not.toHaveBeenCalled(); + }); + + it("an invalid zone reads as invalid and points at the reason", () => { + const { zone } = mount({ invalid: true, describedBy: "why" }); + expect(zone.getAttribute("aria-invalid")).toBe("true"); + expect(zone.getAttribute("aria-describedby")).toBe("why"); + }); +}); diff --git a/frontend/src/components/ui/DropZone.tsx b/frontend/src/components/ui/DropZone.tsx new file mode 100644 index 00000000..ada39182 --- /dev/null +++ b/frontend/src/components/ui/DropZone.tsx @@ -0,0 +1,97 @@ +import { useRef, useState, type ReactNode } from "react"; +import clsx from "clsx"; +import { Pressable } from "./Pressable"; + +// A place to drop files onto, which is also the button that opens the native file dialog: the +// dashed surface an upload starts from. It owns what each such surface needed and the first one +// got right only after a flicker: dragenter/dragleave fire as the cursor crosses the zone's OWN +// children (an icon, a line of text), so a plain boolean flickers — enter and leave are counted +// and the highlight clears at depth 0. The caller draws the inside, told whether a drag is over it. +export function DropZone({ + onFiles, + multiple = false, + accept, + disabled = false, + invalid = false, + describedBy, + className, + children, +}: { + /** The files dropped or picked, never empty. One unless `multiple`. */ + onFiles: (files: [File, ...File[]]) => void; + multiple?: boolean; + /** The picker's filter (``); a drop is not filtered by it. */ + accept?: string; + disabled?: boolean; + /** What the zone holds is refused: the border turns danger and `describedBy` says why. */ + invalid?: boolean; + describedBy?: string; + /** Lays the zone out (its padding); the border and the drag paint are the zone's. */ + className?: string; + children: (dragOver: boolean) => ReactNode; +}) { + const [dragOver, setDragOver] = useState(false); + const depth = useRef(0); + const inputRef = useRef(null); + const take = (list: FileList | null | undefined) => { + const [first, ...rest] = [...(list ?? [])]; + if (first) onFiles(multiple ? [first, ...rest] : [first]); + }; + return ( + <> + inputRef.current?.click()} + onDragEnter={(e) => { + e.preventDefault(); + depth.current += 1; + setDragOver(true); + }} + onDragOver={(e) => e.preventDefault()} + onDragLeave={() => { + depth.current = Math.max(0, depth.current - 1); + if (depth.current === 0) setDragOver(false); + }} + onDrop={(e) => { + e.preventDefault(); + depth.current = 0; + setDragOver(false); + if (!disabled) take(e.dataTransfer.files); + }} + disabled={disabled} + // The refusal is tied to the control it invalidates, not left to a one-shot announcement: + // the zone reads as invalid and points at the sentence, so a reader who tabs back to it + // hears WHY. (Measured, because `aria-invalid` is not in `button`'s ARIA-supported list: + // Chromium's accessibility node for a native button does carry `invalid` from it.) The + // signal that is NOT colour is the caller's sentence — the border only reinforces it. + aria-invalid={invalid} + aria-describedby={describedBy} + // `dragOver` outranks `invalid`: a drag is transient and the refusal repaints the instant + // the drop lands, whereas a zone that stays red under the pointer tells the user their drop + // target is dead — in the one state where they are trying to replace the file. + className={clsx( + "w-full flex flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed transition disabled:opacity-60", + dragOver + ? "border-accent bg-accent/5" + : invalid + ? "border-danger" + : "border-border hover:border-fg/40", + className, + )} + > + {children(dragOver)} + + { + take(e.target.files); + e.target.value = ""; // let re-picking the same file fire onChange again + }} + /> + + ); +} diff --git a/frontend/src/components/ui/README.md b/frontend/src/components/ui/README.md index 2253e6a0..8dd13ac4 100644 --- a/frontend/src/components/ui/README.md +++ b/frontend/src/components/ui/README.md @@ -35,13 +35,13 @@ Counter-examples, deliberately NOT here: `PosterTile` (knows about Plex montage ## What lives here -| area | components | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| overlays & anchoring | `AnchoredPopover` + `anchoredPopoverHooks` (the popover engine: arrows, typeahead, Escape, anchor-flip), `ActionMenu`, `MembershipMenu` (the body of a "which sets is it in" menu: checkbox rows + a field naming a new set), `ContextMenu` + `useContextMenu` (the SAME menu, opened by right-click / long-press / Shift+F10 / a row's `MoreButton` "…" via `buttonProps`; `anchor` for a panel a row opens; `PageMenu` for a page's background menu), `Modal` (+ `ModalFooter`; `initialFocus` by purpose, C4), `Overlay`, `SidePanel`, `Tooltip`, `Toaster` (+ `toastLevels`, `overlayOrder`; `lib/toastDock` docks one control under the stack, in every host), `CommandPalette`, `KeyboardHelp`, `ConfirmProvider` + `confirmContext` (`useConfirm` / `useChoice`: the app's promise-based confirm, mounted once at the root; a decision with more than two outcomes is one `useChoice`, never two chained confirms), `dialogFocus` (the focus half of the dialog contract: `Modal`'s, and the wizard's own shell), `menuGroups` (one menu from its groups, a rule only between non-empty ones), `pageMenuSlot` + `pageMenuItems` (a page's background menu, mirroring its own toolbar) | -| actions & inputs | `Button`, `IconButton`, `Pressable` (`selectable` when its text is a title or a name), `Chip` + `chipStyles`, `DragHandle` + `OrderButtons` / `useOrderItems` (the non-drag reorder, WCAG 2.5.7) (one per button family, docs/DESIGN.md; a raw ` + )} + + ) + )} + + ); +} diff --git a/scripts/e2e_seed.py b/scripts/e2e_seed.py index 5d545baf..c4a1be6c 100644 --- a/scripts/e2e_seed.py +++ b/scripts/e2e_seed.py @@ -28,7 +28,7 @@ from app.db import SessionLocal import app.tables # noqa: F401 — every table, before the first query from app.catalog.models import Channel, Subscription, Video from app.models import AppState, User -from app.reader.models import Playlist, PlaylistItem, VideoState +from app.reader.models import Playlist, PlaylistItem, TakeoutItem, VideoState from app.security import hash_password EMAIL = os.environ.get("E2E_EMAIL", "e2e@siftlode.local") @@ -118,6 +118,9 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: # - one WATCHED video from the second channel, so the "watched" show-filter is a # real, non-empty narrowing of the feed (and the "all" vs "unwatched" split exists). db.execute(delete(VideoState).where(VideoState.user_id == user.id)) + # A Takeout import's kept videos too: a settled row keeps its outcome over a re-upload, so the + # import spec's counts would otherwise be the previous run's, not this one's. + db.execute(delete(TakeoutItem).where(TakeoutItem.user_id == user.id)) def _newest(channel_id: str, exclude: set[str]) -> Video | None: conds = [ From e0b02a0c14a4ff58cc9c211fbe47b8b6b3046fe1 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 19:15:06 +0200 Subject: [PATCH 5/8] fix(inbox): a Takeout import's notice opens the user's own import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The import's notice reaches a user who may not see the Scheduler, an admin page, yet its link opened it. It now opens Settings › Account, where the import's status is; any other run still opens the Scheduler. The job id is held to the server's by the backend-ids contract. The notice's Hungarian names the list as the app does elsewhere. --- frontend/src/i18n/backendIds.contract.test.ts | 8 +++ frontend/src/i18n/locales/en/inbox.json | 1 + frontend/src/i18n/locales/hu/inbox.json | 1 + frontend/src/i18n/locales/hu/scheduler.json | 8 +-- frontend/src/lib/api.ts | 1 + frontend/src/lib/api/feed.ts | 4 ++ .../NotificationsPanel.menu.dom.test.tsx | 28 +++++++++ .../notifications/NotificationsPanel.tsx | 60 +++++++++++++------ 8 files changed, 88 insertions(+), 23 deletions(-) diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index 4a23797e..9b2496ed 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -305,6 +305,14 @@ describe("backend ids have their names in both languages", () => { expect([...RUN_STATUSES].sort()).toEqual([...runStatuses].sort()); }); + it("the Takeout import's notice is known by the server's job id", () => { + // Read as text: the slice pulls in the request layer, which needs a browser. + const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8"); + const ours = /^export const TAKEOUT_JOB = "(\w+)";$/m.exec(feedTs)?.[1]; + expect(ours).toBeTruthy(); + expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours); + }); + for (const lang of ["en", "hu"]) { const keys = keysOf(lang); const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k)); diff --git a/frontend/src/i18n/locales/en/inbox.json b/frontend/src/i18n/locales/en/inbox.json index 4a97180a..a34194aa 100644 --- a/frontend/src/i18n/locales/en/inbox.json +++ b/frontend/src/i18n/locales/en/inbox.json @@ -16,6 +16,7 @@ "openScheduler": "Open Scheduler", "openFile": "Open the file", "openTokens": "Open API tokens", + "openTakeout": "Open YouTube history", "revertFailed": "Couldn't undo that — please try again", "downloadDone": { "title": "Download ready" diff --git a/frontend/src/i18n/locales/hu/inbox.json b/frontend/src/i18n/locales/hu/inbox.json index 1eea1a57..5ad64a7e 100644 --- a/frontend/src/i18n/locales/hu/inbox.json +++ b/frontend/src/i18n/locales/hu/inbox.json @@ -16,6 +16,7 @@ "openScheduler": "Ütemező megnyitása", "openFile": "Fájl megnyitása", "openTokens": "API-tokenek megnyitása", + "openTakeout": "YouTube-előzmények megnyitása", "revertFailed": "Nem sikerült visszavonni — próbáld újra", "downloadDone": { "title": "Letöltés kész" diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 8f04643d..029ad15b 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -81,7 +81,7 @@ "tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.", "people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.", "channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be.", - "takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül az övébe. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz." + "takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül a „Megnézem később” listájába. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz." }, "jobs": { "rss_poll": "RSS-lekérdezés (új feltöltések)", @@ -183,10 +183,10 @@ "cached": "elmentve", "kept": "itteni állapot megtartva", "waiting": "a videójára vár", - "watch_later_added": "Watch laterbe téve", - "watch_later_present": "már a Watch laterben", + "watch_later_added": "a „Megnézem később” listába téve", + "watch_later_present": "már a „Megnézem később” listán", "watch_later_missing": "már nincs a YouTube-on", - "watch_later_waiting": "Watch later vár", + "watch_later_waiting": "a „Megnézem később” listára vár", "takeout_watched": "Takeoutból megnézettre jelölve" }, "skip": { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 4cd31714..e957a10b 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -20,6 +20,7 @@ import { setupApi } from "./api/setup"; export type * from "./api/setup"; import { feedApi } from "./api/feed"; export type * from "./api/feed"; +export { TAKEOUT_JOB } from "./api/feed"; import { discoverApi } from "./api/discover"; export type * from "./api/discover"; export { DISCOVER_BUDGETS, HERO_TIERS, discoverCardKey } from "./api/discover"; diff --git a/frontend/src/lib/api/feed.ts b/frontend/src/lib/api/feed.ts index ba4b6e98..0ffbfe83 100644 --- a/frontend/src/lib/api/feed.ts +++ b/frontend/src/lib/api/feed.ts @@ -149,6 +149,10 @@ function feedQuery(f: FeedFilters, cursor: string | null, limit: number): string return p.toString(); } +// The import's job (`scheduler.TAKEOUT_IMPORT`, held to it by backendIds.contract.test): its notice +// reaches a user who may not see the Scheduler, so it opens their own import instead. +export const TAKEOUT_JOB = "takeout_import"; + // A Google Takeout file, uploaded: which one it was, the videos read from it, and its entries left // out (an ad, a view of something that is not a video, a row that could not be read). export interface TakeoutUpload { diff --git a/frontend/src/modules/notifications/NotificationsPanel.menu.dom.test.tsx b/frontend/src/modules/notifications/NotificationsPanel.menu.dom.test.tsx index 28360123..d42adf50 100644 --- a/frontend/src/modules/notifications/NotificationsPanel.menu.dom.test.tsx +++ b/frontend/src/modules/notifications/NotificationsPanel.menu.dom.test.tsx @@ -205,6 +205,34 @@ describe("NotificationsPanel · a server row", () => { expect(menuRows()).toEqual(["inbox.openTokens", "inbox.markRead", "inbox.dismiss"]); }); + it("a Takeout import's run opens the user's own import, any other run the Scheduler", async () => { + const run = (id: number, job: string) => + server(id, { + type: "scheduler", + data: { job_id: job, status: "ok", summary: `${job} done` }, + }); + h.server = [run(1, "takeout_import"), run(2, "channel_backfill")]; + h.client = []; + mount(); + fireEvent.click(screen.getByText("inbox.openTakeout")); + expect(h.focusSettingsTab).toHaveBeenCalledWith("account"); + expect(h.setPage).toHaveBeenLastCalledWith("settings"); + fireEvent.click(screen.getByText("inbox.openScheduler")); + expect(h.setPage).toHaveBeenLastCalledWith("scheduler"); + const takeoutRow = screen + .getByText("inbox.openTakeout") + .closest("[data-nav-index]")!; + await openMenu(() => + fireEvent.click(takeoutRow.querySelector('button[aria-label="inbox.rowMenu"]')!), + ); + expect(menuRows()).toEqual(["inbox.openTakeout", "inbox.markRead", "inbox.dismiss"]); + h.focusSettingsTab.mockClear(); + h.setPage.mockClear(); + choose("inbox.openTakeout"); + await waitFor(() => expect(h.focusSettingsTab).toHaveBeenCalledWith("account")); + expect(h.setPage).toHaveBeenCalledWith("settings"); + }); + it("holds the link, mark read and dismiss", async () => { h.server = [server(1)]; h.client = []; diff --git a/frontend/src/modules/notifications/NotificationsPanel.tsx b/frontend/src/modules/notifications/NotificationsPanel.tsx index 4b220755..e6c9eced 100644 --- a/frontend/src/modules/notifications/NotificationsPanel.tsx +++ b/frontend/src/modules/notifications/NotificationsPanel.tsx @@ -11,6 +11,7 @@ import { ExternalLink, Eye, FolderOpen, + History, KeyRound, RotateCcw, Search, @@ -19,7 +20,7 @@ import { UserPlus, X, } from "lucide-react"; -import { api, type AppNotification } from "../../lib/api"; +import { api, TAKEOUT_JOB, type AppNotification } from "../../lib/api"; import { jobRunText } from "../../lib/jobResult"; import { notifyActionError } from "../../lib/actionErrors"; import { useLiveQuery } from "../../lib/useLiveQuery"; @@ -192,6 +193,10 @@ export default function NotificationsPanel() { focusSettingsTab("tokens"); setPage("settings"); }; + const openTakeout = () => { + focusSettingsTab("account"); + setPage("settings"); + }; // A finished download opens on its file, in its folder; a notice from before it named one, on Files. const openDone = (n: AppNotification) => { const { job_id: id, folder_id: folder } = n.data ?? {}; @@ -213,34 +218,43 @@ export default function NotificationsPanel() { const kind = serverKind(n); return menuGroups( [ - ...(kind === "scheduler" + ...(kind === "takeout" ? [ { id: "open", - label: t("inbox.openScheduler"), - icon: icon(Activity), - onSelect: openScheduler, + label: t("inbox.openTakeout"), + icon: icon(History), + onSelect: openTakeout, }, ] - : kind === "download" + : kind === "scheduler" ? [ { id: "open", - label: t("inbox.openFile"), - icon: icon(FolderOpen), - onSelect: () => openDone(n), + label: t("inbox.openScheduler"), + icon: icon(Activity), + onSelect: openScheduler, }, ] - : kind === "token" + : kind === "download" ? [ { id: "open", - label: t("inbox.openTokens"), - icon: icon(KeyRound), - onSelect: openTokens, + label: t("inbox.openFile"), + icon: icon(FolderOpen), + onSelect: () => openDone(n), }, ] - : []), + : kind === "token" + ? [ + { + id: "open", + label: t("inbox.openTokens"), + icon: icon(KeyRound), + onSelect: openTokens, + }, + ] + : []), ...(!n.read ? [ { @@ -411,6 +425,7 @@ export default function NotificationsPanel() { onRead={() => readMut.mutate(n.id)} onDismiss={() => dismissMut.mutate(n.id)} onOpenScheduler={openScheduler} + onOpenTakeout={openTakeout} onOpenFile={() => openDone(n)} onOpenTokens={openTokens} /> @@ -478,7 +493,9 @@ const DONE_TITLE: Record = { upload: "inbox.uploadDone.title", }; -function serverKind(n: AppNotification): "scheduler" | "download" | "token" | null { +function serverKind(n: AppNotification): "scheduler" | "takeout" | "download" | "token" | null { + // A Takeout import's run is the user's own: told like any run, opened where they started it. + if (n.type === "scheduler" && n.data?.job_id === TAKEOUT_JOB) return "takeout"; if (n.type === "scheduler" && typeof n.data?.job_id === "string") return "scheduler"; if (n.type === "download_done") return "download"; if (n.type === "api_token_expiring" && typeof n.data?.name === "string") return "token"; @@ -527,6 +544,7 @@ function NotificationRow({ onRead, onDismiss, onOpenScheduler, + onOpenTakeout, onOpenFile, onOpenTokens, }: { @@ -538,6 +556,7 @@ function NotificationRow({ onRead: () => void; onDismiss: () => void; onOpenScheduler: () => void; + onOpenTakeout: () => void; onOpenFile: () => void; onOpenTokens: () => void; }) { @@ -549,7 +568,8 @@ function NotificationRow({ // Known notification types are rendered from i18n (so they're trilingual) using the typed // payload; the server-stored title/body are an English fallback for any unknown type. const isMaintenance = n.type === "maintenance" && typeof n.data?.count === "number"; - const isScheduler = serverKind(n) === "scheduler"; + const isTakeout = serverKind(n) === "takeout"; + const isScheduler = serverKind(n) === "scheduler" || isTakeout; const isDownloadDone = serverKind(n) === "download"; const isTokenExpiring = serverKind(n) === "token"; const isEmailUndelivered = n.type === "email_undelivered" && typeof n.data?.reason === "string"; @@ -633,10 +653,12 @@ function NotificationRow({ variant="link" className="text-sm mt-1.5" data-nav-open="" - onClick={onOpenScheduler} - icon={} + onClick={isTakeout ? onOpenTakeout : onOpenScheduler} + icon={ + isTakeout ? : + } > - {t("inbox.openScheduler")} + {t(isTakeout ? "inbox.openTakeout" : "inbox.openScheduler")} )} {isTokenExpiring && ( From 9f580fd8c8175684b04913131c4bdb469919a944 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 20:48:23 +0200 Subject: [PATCH 6/8] fix(reader): the Takeout import holds up under abuse, races and a run it cannot see - Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user. --- backend/app/bodylimit.py | 8 +- backend/app/reader/queries.py | 48 ++- backend/app/reader/service.py | 32 +- backend/app/reader/takeout.py | 173 +++++++-- backend/app/routes/_common.py | 8 + backend/app/routes/feed.py | 54 ++- backend/app/routes/runs.py | 8 +- backend/app/scheduler.py | 95 +++-- backend/tests/test_activity_reads.py | 9 + backend/tests/test_body_limit.py | 4 + backend/tests/test_scheduler.py | 36 +- backend/tests/test_takeout.py | 346 +++++++++++++++++- frontend/e2e/global.setup.ts | 6 +- frontend/e2e/helpers.ts | 10 + frontend/e2e/takeout.spec.ts | 14 +- frontend/src/components/ui/DropZone.tsx | 6 +- frontend/src/i18n/backendIds.contract.test.ts | 23 ++ frontend/src/i18n/locales/en/errors.json | 6 +- frontend/src/i18n/locales/en/scheduler.json | 6 +- frontend/src/i18n/locales/en/settings.json | 3 +- frontend/src/i18n/locales/hu/errors.json | 6 +- frontend/src/i18n/locales/hu/scheduler.json | 6 +- frontend/src/i18n/locales/hu/settings.json | 7 +- frontend/src/lib/api/feed.ts | 24 +- .../settings/TakeoutImport.dom.test.tsx | 156 +++++++- .../src/modules/settings/TakeoutImport.tsx | 207 +++++++---- .../src/modules/settings/takeoutWatch.test.ts | 67 ++++ frontend/src/modules/settings/takeoutWatch.ts | 57 +++ scripts/e2e_seed.py | 23 ++ 29 files changed, 1235 insertions(+), 213 deletions(-) create mode 100644 frontend/src/modules/settings/takeoutWatch.test.ts create mode 100644 frontend/src/modules/settings/takeoutWatch.ts diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index 97b67eba..432c6e1c 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -72,9 +72,11 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why # `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the # default, so listing it could only widen what it accepts. -# A Google Takeout file (`reader.takeout`): a real watch history of 9500 views, the 18 months Google -# keeps by default, is 3.7 MB; four times that is still well inside one request. -TAKEOUT_MAX_BYTES = 16 * 1024 * 1024 +# A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390 +# bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an +# account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The +# file is read an entry at a time (`takeout._json_list`), so it costs its own size, not a tree of it. +TAKEOUT_MAX_BYTES = 32 * 1024 * 1024 ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # Ingesting a local file. The body IS the file now — no multipart envelope to allow for — diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 88a6550a..9ac01add 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -7,6 +7,7 @@ length, whether a video exists); another unit asks the questions at the end of t than joining Reader's tables itself. """ +from collections.abc import Iterable from datetime import datetime from sqlalchemy import ColumnElement, and_, func, or_, select @@ -407,11 +408,41 @@ def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[in ) +# Ids per IN list: a list is bound one parameter an id, and a statement takes at most 65,535. +_IN_CHUNK = 5000 + + def known_video_ids(db: Session, ids: list[str]) -> set[str]: """Which of `ids` the catalog holds.""" - if not ids: - return set() - return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars()) + known: set[str] = set() + for start in range(0, len(ids), _IN_CHUNK): + chunk = ids[start : start + _IN_CHUNK] + known.update(db.execute(select(Video.id).where(Video.id.in_(chunk))).scalars()) + return known + + +def takeout_rows_besides( + db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str] +) -> int: + """How many Takeout rows the user keeps other than `video_ids` of `kind`: what an upload of + those videos would be added to.""" + total = ( + db.scalar(select(func.count(TakeoutItem.id)).where(owned_by(TakeoutItem.user_id, user))) + or 0 + ) + ids = list(video_ids) + for start in range(0, len(ids), _IN_CHUNK): + total -= ( + db.scalar( + select(func.count(TakeoutItem.id)).where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == kind, + TakeoutItem.video_id.in_(ids[start : start + _IN_CHUNK]), + ) + ) + or 0 + ) + return total def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]: @@ -449,3 +480,14 @@ def takeout_users_ready(db: Session) -> list[int]: .distinct() ).scalars() ) + + +def takeout_users_watch_later_waiting(db: Session) -> list[int]: + """Every user with a Watch later row a run could not put in yet.""" + return list( + db.execute( + select(TakeoutItem.user_id) + .where(TakeoutItem.kind == TakeoutKind.WATCH_LATER, TakeoutItem.outcome.is_(None)) + .distinct() + ).scalars() + ) diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index 7113eaae..4d396aee 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -510,6 +510,15 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No # --- Google Takeout -------------------------------------------------------------------------- _TAKEOUT_BATCH = 1000 +# The advisory-lock namespace of one user's Takeout rows ("TO"), so a key cannot meet another lock's. +_TAKEOUT_LOCK_NS = 0x544F + + +def lock_takeout(db: Session, user: User) -> None: + """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an + import's settling each lock many of the same rows, one statement after another and in no common + order, so without one lock over both the two can deadlock.""" + db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id))) def stage_takeout( @@ -518,6 +527,7 @@ def stage_takeout( """Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job. A second upload of the same export, or a newer one, adds what is new: a video already kept keeps the later time, and a row the import has settled keeps its outcome.""" + lock_takeout(db, user) values = [ {"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel} for vid, (at, channel) in rows.items() @@ -551,6 +561,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: past is not watching it), and a state the user set here wins: watched or hidden, or anything changed after the Takeout time. The upsert's own condition is that rule, so a state set while the import runs wins too.""" + lock_takeout(db, user) short: list[int] = [] todo: dict[str, tuple[int, datetime]] = {} for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user): @@ -590,19 +601,22 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: return {"watched": len(marked), "kept": len(kept), "shorts": len(short)} -def apply_takeout_watch_later(db: Session, user: User) -> dict[str, int]: - """Put the videos of the user's YouTube Watch later list into theirs here, oldest first, after - what it already holds. Called once the catalog has fetched what it could: a video still unknown - is one YouTube no longer has.""" - waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) - if not waiting: +def apply_takeout_watch_later( + db: Session, user: User, rows: list[tuple[int, str]], asked: set[str] +) -> dict[str, int]: + """Put the videos of the user's waiting Watch later `rows`, (row id, video id) oldest first, into + their list here, after what it already holds. `asked` is the videos YouTube was just asked for: one + of those the catalog still lacks is gone from YouTube; any other unknown one keeps waiting.""" + if not rows: return {} - known = queries.known_video_ids(db, [vid for _, vid in waiting]) + lock_takeout(db, user) + known = queries.known_video_ids(db, [vid for _, vid in rows]) pl = watch_later(db, user) done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} - for row_id, vid in waiting: + for row_id, vid in rows: if vid not in known: - done[TakeoutOutcome.MISSING].append(row_id) + if vid in asked: + done[TakeoutOutcome.MISSING].append(row_id) elif add_item(db, pl, vid, mark_dirty=False): done[TakeoutOutcome.ADDED].append(row_id) else: diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py index 7a7b12b9..67231e33 100644 --- a/backend/app/reader/takeout.py +++ b/backend/app/reader/takeout.py @@ -24,6 +24,7 @@ from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass, field from datetime import datetime +from itertools import chain from sqlalchemy.orm import Session @@ -31,6 +32,7 @@ from app import quota from app.bodylimit import TAKEOUT_MAX_BYTES from app.db import SessionLocal from app.models import User +from app.ratelimit import RateLimiter from app.reader import queries, service from app.reader.models import TakeoutKind from app.sync.runner import get_service_user @@ -39,6 +41,12 @@ from app.youtube.client import YouTubeClient # The largest file the upload takes (its edge cap and the route's own are one number). MAX_BYTES = TAKEOUT_MAX_BYTES +# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is +# looked up on the shared API budget, so a longer list is not one YouTube made. +WATCH_LATER_MAX = 5000 +# The most Takeout videos one user keeps: twenty times the real export measured here (~9,000 videos +# across both files), so it only stops uploads of made-up ones piling up for good. +ROWS_MAX = 200_000 _VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") _WATCH_URL = re.compile( @@ -84,7 +92,7 @@ def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None: parsed.videos[vid] = (at, channel or (old[1] if old else None)) -def _history(data: list) -> Parsed: +def _history(data: Iterator[object]) -> Parsed: parsed = Parsed(TakeoutKind.WATCHED) for entry in data: parsed.entries += 1 @@ -117,22 +125,54 @@ def _history(data: list) -> Parsed: def _watch_later(text: str) -> Parsed: parsed = Parsed(TakeoutKind.WATCH_LATER) - rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)] - if not rows or len(rows[0]) < 2: + rows = (row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)) + first = next(rows, None) + if first is None or len(first) < 2: raise TakeoutError("not_takeout") - if not _VIDEO_ID.match(rows[0][0].strip()): - rows = rows[1:] # the header, in the account's language - for row in rows: + # The first row is the header, in the account's language, unless it is already a video. + for row in chain([first], rows) if _VIDEO_ID.match(first[0].strip()) else rows: parsed.entries += 1 - vid = row[0].strip() if row else "" + vid = row[0].strip() at = _time(row[1].strip()) if len(row) > 1 else None if not _VIDEO_ID.match(vid) or at is None: parsed.malformed += 1 continue _keep(parsed, vid, at, None) + if len(parsed.videos) > WATCH_LATER_MAX: + raise TakeoutError("too_many") return parsed +def _skip(text: str, at: int) -> int: + """The first index at or after `at` that is not JSON whitespace.""" + while at < len(text) and text[at] in " \t\n\r": + at += 1 + return at + + +def _json_list(text: str) -> Iterator[object]: + """The entries of a JSON list, one at a time: the whole tree is never built, so a file of a + million tiny entries costs its text and one entry, not a million dicts at once.""" + decoder = json.JSONDecoder() + at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw + if text.startswith("]", at): + at += 1 + else: + while True: + entry, at = decoder.raw_decode(text, at) + yield entry + at = _skip(text, at) + if text.startswith(",", at): + at = _skip(text, at + 1) + elif text.startswith("]", at): + at += 1 + break + else: + raise ValueError("not a JSON list") + if _skip(text, at) != len(text): + raise ValueError("text after the JSON list") + + def parse(body: bytes) -> Parsed: """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file @@ -142,19 +182,19 @@ def parse(body: bytes) -> Parsed: except UnicodeDecodeError: raise TakeoutError("not_takeout") from None stripped = text.lstrip() - if stripped.startswith("["): - try: - data = json.loads(stripped) - except json.JSONDecodeError: - raise TakeoutError("not_takeout") from None - if not isinstance(data, list): - raise TakeoutError("not_takeout") - parsed = _history(data) - elif stripped.startswith(("{", "<")): + if stripped.startswith(("{", "<")): # Another JSON file of the export, or the history's HTML variant. raise TakeoutError("not_takeout") - else: - parsed = _watch_later(stripped) + try: + parsed = ( + _history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped) + ) + # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too + # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit. + except (ValueError, RecursionError, csv.Error) as exc: + if isinstance(exc, TakeoutError): + raise + raise TakeoutError("not_takeout") from None if not parsed.videos: raise TakeoutError("no_videos") return parsed @@ -181,6 +221,61 @@ def run_target(user_id: int) -> str: return f"user:{user_id}" +def target_title(db: Session, target: str) -> str | None: + """Who an import run's target is, for the admin's run lists: the user's name, else their email.""" + _, _, raw = target.partition("user:") + user = db.get(User, int(raw)) if raw.isdigit() else None + return (user.display_name or user.email) if user is not None else None + + +class TooMany(Exception): + """The upload would take the user past `ROWS_MAX` kept videos.""" + + +def stage(db: Session, user: User, parsed: Parsed) -> None: + """Keep an uploaded file's videos for the import, unless that would take the user past + `ROWS_MAX` (`TooMany`). The caller commits.""" + service.lock_takeout(db, user) # the count and the insert are one user's, not two uploads' each + if ( + queries.takeout_rows_besides(db, user, parsed.kind, parsed.videos) + len(parsed.videos) + > ROWS_MAX + ): + raise TooMany + service.stage_takeout(db, user, parsed.kind, parsed.videos) + + +# How many unknown Watch later videos one user may have looked up on the shared API budget in a day: +# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs. +_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600) + + +def _put_in_watch_later(db: Session, user: User) -> dict: + """Put the user's waiting Watch later videos into theirs here: the ones the catalog knows at once, + the others once YouTube was asked for them — when a YouTube-connected account can ask, today's + budget is above the backfill reserve, and the user's own daily share of lookups allows it. A video + not asked about yet waits for a later run. The lookup commits what it fetched; the caller commits + the rest.""" + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if not waiting: + return {} + ids = [vid for _, vid in waiting] + known = queries.known_video_ids(db, ids) + ask = [vid for vid in ids if vid not in known][:WATCH_LATER_MAX] + asked: set[str] = set() + fetcher = get_service_user(db) if ask else None + if ( + fetcher is not None + and not quota.backfill_reserve_reached() + and _lookups.allow(str(user.id), cost=len(ask)) + ): + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP): + with YouTubeClient(db, fetcher) as yt: + ensure_videos(db, yt, ask) + asked = set(ask) + # Only the rows read above: one a second upload staged meanwhile was never asked about. + return service.apply_takeout_watch_later(db, user, waiting, asked) + + def run_import(user_id: int) -> dict: """The import job: apply what the user's kept Takeout rows can say now, as one recorded run. The watched part commits before the Watch later part asks YouTube for the videos the catalog @@ -191,15 +286,8 @@ def run_import(user_id: int) -> dict: return {"skipped": "no user"} result = service.apply_takeout_watched(db, user) db.commit() - waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) - if waiting: - fetcher = get_service_user(db) - if fetcher is not None: - with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP): - with YouTubeClient(db, fetcher) as yt: - ensure_videos(db, yt, [vid for _, vid in waiting]) - result |= service.apply_takeout_watch_later(db, user) - db.commit() + result |= _put_in_watch_later(db, user) + db.commit() outcomes = queries.takeout_outcomes(db, user) result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0) if outcomes.get((TakeoutKind.WATCH_LATER, None)): @@ -208,17 +296,28 @@ def run_import(user_id: int) -> dict: def apply_waiting(db: Session) -> dict: - """Mark watched the waiting Takeout rows whose video the catalog has come to know and the Shorts - probe has cleared since the import — every user's, one commit each. Run after each Shorts probe - (the scheduler's `shorts` job), the only place a video's Short-ness becomes known. Returns its - counts, or an empty dict when nothing was waiting on a probed video.""" - users = queries.takeout_users_ready(db) - marked = 0 - for user_id in users: + """Settle what every user's import left waiting and can be settled now, one commit each: a + watched video the catalog has come to know and the Shorts probe has cleared (by its job or a + search's), and the Watch later videos a run could not put in yet. Run after each run of the + `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when + nothing was waiting that could move.""" + marked = added = 0 + ready = set(queries.takeout_users_ready(db)) + later = set(queries.takeout_users_watch_later_waiting(db)) + for user_id in sorted(ready | later): user = db.get(User, user_id) if user is None: continue with _one_import(user_id): - marked += service.apply_takeout_watched(db, user)["watched"] - db.commit() - return {"takeout_watched": marked} if users else {} + if user_id in ready: + marked += service.apply_takeout_watched(db, user)["watched"] + db.commit() + if user_id in later: + added += _put_in_watch_later(db, user).get("watch_later_added", 0) + db.commit() + out = {} + if ready: + out["takeout_watched"] = marked + if added: + out["takeout_watch_later_added"] = added + return out diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py index 32e75e53..6f904dad 100644 --- a/backend/app/routes/_common.py +++ b/backend/app/routes/_common.py @@ -124,6 +124,14 @@ async def capped_body(request: Request, limit: int) -> bytes: return bytes(body) +async def drain_body(request: Request) -> None: + """Read the rest of the body to nowhere, before a refusal is raised: answered mid-upload, the + client reports a network failure, not the refusal (`capped_stream_to` says why). `bodylimit` + bounds how much there can be.""" + async for _ in request.stream(): + pass + + class ByteSink(Protocol): """Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one.""" diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index d6a41e24..db16714e 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,3 +1,5 @@ +import threading +from collections.abc import AsyncIterator from datetime import date from functools import partial @@ -10,12 +12,14 @@ from app import jobruns, quota, scheduler from app.auth import current_user from app.db import get_db, release_read_connection from app.mailtext import lang_for_user +from app.ratelimit import RateLimiter from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years from app.models import User from app.reader import queries, service, takeout from app.routes._common import ( ProgressIn, capped_body, + drain_body, kept_position, not_found, youtube_refusal, @@ -305,17 +309,44 @@ def get_video_detail( # --- Google Takeout ------------------------------------------------------------------------------ -async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes: +# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these +# bound what one account can make the process hold. The section uploads its files one after another. +_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600) +_takeout_uploading: set[int] = set() +_takeout_uploading_guard = threading.Lock() + + +async def _takeout_body( + request: Request, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> AsyncIterator[bytes]: # The sign-in check has run, and it was a SELECT: hand the connection back before a file of # megabytes arrives, so a slow upload does not hold one `idle in transaction`. release_read_connection(db) - return await capped_body(request, takeout.MAX_BYTES) + with _takeout_uploading_guard: + busy = user.id in _takeout_uploading + if not busy: + _takeout_uploading.add(user.id) + if busy or not _takeout_limiter.allow(str(user.id)): + if not busy: + with _takeout_uploading_guard: + _takeout_uploading.discard(user.id) + # Read to the end before answering: refused mid-upload, the browser reports a network + # failure instead of the refusal (capped_stream_to says why). + await drain_body(request) + if busy: + raise refused(429, "takeout_busy") + raise refused(429, "takeout_cooldown") + try: + yield await capped_body(request, takeout.MAX_BYTES) + finally: + with _takeout_uploading_guard: + _takeout_uploading.discard(user.id) @router.post("/takeout") def upload_takeout( - # The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before - # a byte of it is read. + # The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's + # file is refused before a byte of it is read. user: User = Depends(current_user), body: bytes = Depends(_takeout_body), db: Session = Depends(get_db), @@ -327,8 +358,14 @@ def upload_takeout( except takeout.TakeoutError as exc: if exc.reason == "no_videos": raise refused(422, "takeout_no_videos") from None + if exc.reason == "too_many": + raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None raise refused(422, "takeout_unreadable") from None - service.stage_takeout(db, user, parsed.kind, parsed.videos) + try: + takeout.stage(db, user, parsed) + except takeout.TooMany: + db.rollback() + raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None db.commit() scheduler.run_ad_hoc( scheduler.TAKEOUT_IMPORT, @@ -353,8 +390,13 @@ def takeout_status(user: User = Depends(current_user), db: Session = Depends(get """What the user's Takeout import has done: its last run (none before the first upload), and every video kept from the export, by kind and outcome — a video the catalog did not know yet counts as "waiting" until it does, and is marked then.""" - run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, takeout.run_target(user.id)) + target = takeout.run_target(user.id) + run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target) return { + # Queued and not started: its run row is not written yet, and the last run is still the + # previous one, so the page waits for it rather than taking that one for this upload's. + "queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target), + "max_bytes": takeout.MAX_BYTES, "run": None if run is None else { diff --git a/backend/app/routes/runs.py b/backend/app/routes/runs.py index 2672d802..7f882b27 100644 --- a/backend/app/routes/runs.py +++ b/backend/app/routes/runs.py @@ -3,7 +3,7 @@ Admin-only and read-only — rows leave the table through the retention job (`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and the audit log beside it on the activity page. A row names what a reader needs without a second -request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only +request: what a user's ad-hoc run worked on (the channel, the importing user), and how many log lines the run left (only WARNING and up are stored, so most runs have none — the count says so before the click). """ @@ -116,8 +116,12 @@ def list_runs( limit, lambda r: r.started_at, ) - channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target} + channels = {r.target for r in runs if r.job == scheduler.CHANNEL_BACKFILL and r.target} titles = catalog_queries.channel_titles(db, channels) + for target in {r.target for r in runs if r.job == scheduler.TAKEOUT_IMPORT and r.target}: + title = scheduler.ad_hoc_target_title(db, scheduler.TAKEOUT_IMPORT, target) + if title is not None: + titles[target] = title lines: dict[int, int] = {} if runs: counted = db.execute( diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 44d395ac..39dd3797 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -274,7 +274,6 @@ def scheduler_snapshot(db) -> dict: open_runs = _open_run_ids() recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) open_runs |= _open_run_ids() - going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -290,30 +289,11 @@ def scheduler_snapshot(db) -> dict: ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the - # target's own (sync/runner), so "running" is the runs this process has open: two channels can - # backfill at once, so a run still going counts even when a newer one already ended. That run then - # leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the - # newer run's outcome under the older one's channel. - backfills = recent.get(CHANNEL_BACKFILL, []) - going = db.get(JobRun, going_id) if going_id is not None else None - if going is not None and going.status != RunStatus.RUNNING: - going = None # its outcome is written; only the bookkeeping has not caught up - if going is not None: - backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS] - shown = backfills[0] if backfills else None - ad_hoc = [ - { - "id": CHANNEL_BACKFILL, - "running": going is not None, - "target": shown.target if shown else None, - "target_title": ( - catalog_queries.channel_title(db, shown.target) - if shown is not None and shown.target - else None - ), - **_last_runs(backfills, open_runs), - } - ] + # target's own (sync/runner, the import's per-user lock), so "running" is the runs this process + # has open: two targets can run at once, so a run still going counts even when a newer one already + # ended. That run then leads the row, so its target, its "no outcome yet" and its dot are one + # run's, never the newer run's outcome under the older one's target. + ad_hoc = [_ad_hoc_row(db, job, recent.get(job, []), open_runs) for job in AD_HOC_JOBS] return { "running_here": running_here, "enabled": settings.scheduler_enabled, @@ -335,6 +315,27 @@ def _seen(run, open_runs: dict[int, str]) -> RunStatus: return RunStatus(run.status) +def _ad_hoc_row(db, job: str, runs: list[JobRun], open_runs: dict[int, str]) -> dict: + going_id = max((r for r, j in open_runs.items() if j == job), default=None) + going = db.get(JobRun, going_id) if going_id is not None else None + if going is not None and going.status != RunStatus.RUNNING: + going = None # its outcome is written; only the bookkeeping has not caught up + if going is not None: + runs = [going, *(r for r in runs if r.id != going.id)][:RECENT_RUNS] + shown = runs[0] if runs else None + return { + "id": job, + "running": going is not None, + "target": shown.target if shown else None, + "target_title": ( + ad_hoc_target_title(db, job, shown.target) + if shown is not None and shown.target + else None + ), + **_last_runs(runs, open_runs), + } + + def _last_runs(runs: list, open_runs: dict[int, str]) -> dict: """A row's run fields from its newest runs (newest first): how the last one went, and the few before it for the history dots.""" @@ -619,6 +620,29 @@ CHANNEL_BACKFILL = "channel_backfill" TAKEOUT_IMPORT = "takeout_import" # Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT) +# What each one's target is called on the admin's pages: a channel's title, an importing user's name. +_TARGET_TITLES: dict[str, Callable[..., str | None]] = { + CHANNEL_BACKFILL: catalog_queries.channel_title, + TAKEOUT_IMPORT: takeout.target_title, +} + + +def ad_hoc_target_title(db, job: str, target: str) -> str | None: + """The name of an ad-hoc run's target (None for a job with none, or a target since gone).""" + title = _TARGET_TITLES.get(job) + return title(db, target) if title is not None else None + + +# The ad-hoc runs added and not yet recorded, `job@target`: between the two the target's newest +# run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too. +_queued: set[str] = set() +_queued_guard = threading.Lock() + + +def ad_hoc_queued(job: str, target: str) -> bool: + """Whether a run of `job` on `target` is queued and its run row not written yet.""" + with _queued_guard: + return f"{job}@{target}" in _queued def run_ad_hoc( @@ -634,17 +658,24 @@ def run_ad_hoc( if sched is None: # Only between shutdown and exit: start_scheduler runs at every API startup. raise RuntimeError("the scheduler is not running") + key = f"{job}@{target}" + with _queued_guard: + _queued.add(key) try: sched.add_job( _ad_hoc, "date", args=(job, fn, actor_id, target, notify), - id=f"{job}@{target}", + id=key, misfire_grace_time=None, ) # silent-ok: not a failure — the run for this target is already queued, and it does this work. except ConflictingIdError: log.info("ad-hoc %s for %s already queued", job, target) + except Exception: + with _queued_guard: + _queued.discard(key) + raise def _ad_hoc( @@ -656,6 +687,10 @@ def _ad_hoc( # No row to tell it in: the database is what failed. The log is the record. log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return + finally: + # Recorded (or never will be): the run row now speaks for it. + with _queued_guard: + _queued.discard(f"{job}@{target}") _opened(run_id, job) try: _ad_hoc_work(job, fn, run_id, actor_id if notify else None) @@ -672,7 +707,11 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) err, failure = _failure(exc) _close(run_id, RunStatus.ERROR, error=err, failure=failure) - _notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure) + # The user's notice names the failure, never the upstream's own text: that is an English + # diagnostic, for the admin's run row and the log, not a user's inbox. + _notify_ad_hoc( + notify_id, job, RunStatus.ERROR, None, failure or {"failure": "unexpected"} + ) return summary, data = _finish_ok(job, run_id, result) _notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data) @@ -725,3 +764,5 @@ def shutdown_scheduler() -> None: if _scheduler is not None: _scheduler.shutdown(wait=False) _scheduler = None + with _queued_guard: + _queued.clear() # what it held will not run diff --git a/backend/tests/test_activity_reads.py b/backend/tests/test_activity_reads.py index c4df1b02..59efed36 100644 --- a/backend/tests/test_activity_reads.py +++ b/backend/tests/test_activity_reads.py @@ -159,6 +159,15 @@ def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, assert facets["total"] == 4 +def test_the_run_list_names_the_user_a_takeout_import_ran_for(db, make_user, as_user): + admin = make_user(role="admin") + importer = make_user("importer@example.com") + imported = _run(db, "takeout_import", "ok", importer.id, T0, f"user:{importer.id}", "ad_hoc") + db.commit() + (row,) = as_user(admin).get("/api/admin/runs").json()["items"] + assert (row["id"], row["target_title"]) == (imported, "importer@example.com") + + def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user): admin = make_user(role="admin") now = datetime.now(timezone.utc) diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py index ad1f72c2..f2239d2a 100644 --- a/backend/tests/test_body_limit.py +++ b/backend/tests/test_body_limit.py @@ -23,6 +23,7 @@ from app import bodylimit, debugsession, sysconfig from app.config import settings from app.models import SystemConfig from app.bodylimit import DEFAULT_MAX_BODY_BYTES, BodyLimitMiddleware +from app.reader import takeout from app.routes import diagnostics as diagnostics_routes SMALL = b"x" * 16 @@ -369,6 +370,9 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling(): # The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather # than being a constant to compare against — that pairing is its own test below. assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES + # A Takeout file is larger than the default: without its ceiling every real history is a 413. + assert takeout.MAX_BYTES > DEFAULT_MAX_BODY_BYTES + assert bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/takeout")] == takeout.MAX_BYTES def test_release_read_connection_ends_the_transaction_it_was_called_for(db): diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 8f795b46..d3fa418e 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -342,6 +342,29 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure assert (failed.status, failed.error) == ("error", "quota exceeded") +def _ad_hoc_row(db, job: str) -> dict: + rows = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert [r["id"] for r in rows] == list(scheduler.AD_HOC_JOBS) # every one has its row + return next(r for r in rows if r["id"] == job) + + +def test_a_takeout_import_has_its_row_named_by_the_importing_user(db, make_user): + user = make_user("importer@example.com") + user.display_name = "Ida Importer" + db.commit() + assert _ad_hoc_row(db, "takeout_import")["recent"] == [] + scheduler._ad_hoc("takeout_import", lambda: {"watched": 2}, user.id, f"user:{user.id}") + + row = _ad_hoc_row(db, "takeout_import") + assert (row["running"], row["target"], row["target_title"], row["status"]) == ( + False, + f"user:{user.id}", + "Ida Importer", + "ok", + ) + assert _ad_hoc_row(db, "channel_backfill")["recent"] == [] + + def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user): user = make_user("shown@example.com") db.add(Channel(id="UC_shown", title="BBC Earth")) @@ -349,7 +372,7 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none") scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown") - (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + row = _ad_hoc_row(db, "channel_backfill") assert (row["id"], row["running"], row["target"], row["target_title"]) == ( "channel_backfill", False, @@ -367,12 +390,12 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m seen = [] scheduler._ad_hoc( "channel_backfill", - lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {}, + lambda: seen.append(_ad_hoc_row(db, "channel_backfill")["running"]) or {}, user.id, "UC_shown", ) assert seen == [True] - assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False + assert _ad_hoc_row(db, "channel_backfill")["running"] is False def _finished(db, job) -> list[JobRun]: @@ -397,6 +420,9 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( "channel_backfill@UC_x", "channel_backfill@UC_y", ] + # Queued, so its target's newest run row is not this run's yet: a page watching it can tell. + assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_x") + assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_z") live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next @@ -475,7 +501,7 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") assert calls == ["ok"] # Its row is still `running`, but nothing here is: the page does not call it going. - (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + row = _ad_hoc_row(db, "channel_backfill") assert (row["running"], row["status"], row["recent"][0]["status"]) == ( False, "interrupted", @@ -529,7 +555,7 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use return {} scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long") - (row,) = rows + (row,) = [r for r in rows if r["id"] == "channel_backfill"] assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") # The outcome beside it is the long one's (none yet), never the short one's skip. assert (row["status"], row["result"], row["last_error"]) == (None, None, None) diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py index f603c306..823b5d3e 100644 --- a/backend/tests/test_takeout.py +++ b/backend/tests/test_takeout.py @@ -3,16 +3,20 @@ import job makes of them — watched here (never a Short, never over a state the a video the catalog does not know yet, and the Watch later list joined to the user's own.""" import json +import tracemalloc from datetime import datetime, timedelta, timezone import pytest -from sqlalchemy import select +from sqlalchemy import select, text +from sqlalchemy.exc import OperationalError -from app import jobruns, scheduler +from app import jobruns, quota, scheduler +from app.db import SessionLocal from app.catalog.models import Channel, Video -from app.models import JobRun, Notification +from app.models import JobRun, Notification, User from app.runstatus import RunStatus, RunTrigger -from app.reader import service, takeout +from app.reader import queries, service, takeout +from app.routes import feed as feed_routes from app.reader.models import ( Playlist, PlaylistItem, @@ -107,6 +111,18 @@ def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none(): json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(), "no_videos", ), + # Read an entry at a time, the list must still be one: nothing after it, nothing missing. + (_history(_view("aaaaaaaaaaa")) + b" []", "not_takeout"), + (_history(_view("aaaaaaaaaaa"))[:-1], "not_takeout"), + (b"[" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" x]", "not_takeout"), + # However a hostile file fails, it is one the import cannot read, not a server error. + (b"[" * 100_000, "not_takeout"), + (b"[" + b"1" * 5000 + b"]", "not_takeout"), + (b"Ngb04xUK8uU," + b"x" * 200_000 + b"\n", "not_takeout"), + ( + "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode(), + "too_many", + ), ], ) def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason): @@ -115,6 +131,42 @@ def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason): assert caught.value.reason == reason +def test_the_history_is_read_an_entry_at_a_time_not_as_one_tree(): + """A file of a million tiny entries must cost about its own size, not a million dicts at once: + the whole tree built first measured ~25 times the file.""" + body = b"[" + b",".join([b"{}"] * 1_000_000) + b"]" # 3 MB + tracemalloc.start() + try: + with pytest.raises(takeout.TakeoutError): + takeout.parse(body) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < 5 * len(body), f"parsing held {peak / len(body):.1f} times the file" + + +def test_the_id_lists_are_asked_in_chunks(db, make_user, monkeypatch): + user = make_user("chunks@example.com") + ids = [f"chunk{n:06d}" for n in range(5)] + for vid in ids[:3]: + _video(db, vid) + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {vid: (T0, None) for vid in ids[:4]}) + db.commit() + monkeypatch.setattr(queries, "_IN_CHUNK", 2) + assert queries.known_video_ids(db, ids) == set(ids[:3]) + assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids) == 0 + assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids[:1]) == 3 + + +def test_the_history_is_read_whatever_its_whitespace(): + body = b" \n[ \n" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" ,\n\t" + body += json.dumps(_view("bbbbbbbbbbb")).encode() + b"\n] \n" + assert sorted(takeout.parse(body).videos) == ["aaaaaaaaaaa", "bbbbbbbbbbb"] + full = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5000)) + assert len(takeout.parse(full.encode()).videos) == 5000 # YouTube's own cap still fits + + # --- keeping and applying ---------------------------------------------------------------------- @@ -234,10 +286,15 @@ def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_hol }, ) - counts = service.apply_takeout_watch_later(db, user) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unasked0000": (T0, None)}) + rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + + counts = service.apply_takeout_watch_later(db, user, rows, asked={"gone0000000"}) db.commit() assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1} + # Unknown and not asked about: it waits, it is not called gone. + assert _rows(db, user)["WL:unasked0000"].outcome is None items = db.execute( select(PlaylistItem.video_id) .where(PlaylistItem.playlist_id == pl.id) @@ -265,16 +322,17 @@ def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( asked = [] def fake_ensure(session, yt, ids): - asked.append(ids) + # The lookup is the importing user's spend, on the import's own action. + asked.append((ids, quota._actor_id.get(), quota._action.get())) _video(session, "fetched0000") - session.commit() + session.flush() monkeypatch.setattr(takeout, "ensure_videos", fake_ensure) monkeypatch.setattr(takeout, "get_service_user", lambda session: user) result = takeout.run_import(user.id) - assert asked == [["fetched0000"]] + assert asked == [(["fetched0000"], user.id, quota.QuotaAction.VIDEOS_LOOKUP)] assert result == { "watched": 1, "kept": 0, @@ -286,13 +344,152 @@ def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( } +def _wl_job(db, make_user, monkeypatch, email, *, fetcher=True, reserve=False): + user = make_user(email) + _video(db, "known000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + {"known000000": (T0, None), "unknown0000": (T0 + timedelta(minutes=1), None)}, + ) + db.commit() + asked = [] + monkeypatch.setattr(takeout, "ensure_videos", lambda session, yt, ids: asked.append(ids)) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user if fetcher else None) + monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: reserve) + return user, asked + + +@pytest.mark.parametrize( + "why", ["no account to ask with", "the budget's reserve", "the user's share"] +) +def test_a_watch_later_video_not_asked_about_waits_and_the_known_ones_go_in( + db, make_user, monkeypatch, why +): + user, asked = _wl_job( + db, + make_user, + monkeypatch, + "wait@example.com", + fetcher=why != "no account to ask with", + reserve=why == "the budget's reserve", + ) + if why == "the user's share": + monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(0, 3600)) + + result = takeout.run_import(user.id) + + assert asked == [] + assert (result["watch_later_added"], result["watch_later_missing"]) == (1, 0) + assert result["watch_later_waiting"] == 1 + assert _rows(db, user)["WL:unknown0000"].outcome is None + + +def test_the_shorts_job_puts_in_the_watch_later_videos_a_run_could_not(db, make_user, monkeypatch): + user, asked = _wl_job(db, make_user, monkeypatch, "retry@example.com", fetcher=False) + takeout.run_import(user.id) + assert asked == [] + + # A YouTube-connected account exists now; the next probe run asks for what waits. + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + + def fetched(session, yt, ids): + asked.append(ids) + _video(session, "unknown0000") + session.flush() + + monkeypatch.setattr(takeout, "ensure_videos", fetched) + assert takeout.apply_waiting(db) == {"takeout_watch_later_added": 1} + assert asked == [["unknown0000"]] + assert _rows(db, user)["WL:unknown0000"].outcome == "added" + assert takeout.apply_waiting(db) == {} + + +def test_a_watch_later_row_staged_while_youtube_is_asked_is_left_for_the_next_run( + db, make_user, monkeypatch +): + user, _ = _wl_job(db, make_user, monkeypatch, "race@example.com") + + def meanwhile(session, yt, ids): + # A second upload lands while the lookup runs: its video was never asked about. + with SessionLocal() as other: + service.stage_takeout( + other, + other.get(User, user.id), + TakeoutKind.WATCH_LATER, + {"second00000": (T0, None)}, + ) + other.commit() + + monkeypatch.setattr(takeout, "ensure_videos", meanwhile) + takeout.run_import(user.id) + + rows = _rows(db, user) + assert rows["WL:unknown0000"].outcome == "missing" # asked, and YouTube has it no longer + assert rows["WL:second00000"].outcome is None + + +def test_a_youtube_failure_keeps_what_the_watched_part_already_marked(db, make_user, monkeypatch): + user = make_user("fails@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, CHANNEL)}) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)}) + db.commit() + + def refused(session, yt, ids): + raise RuntimeError("quota exceeded") + + monkeypatch.setattr(takeout, "ensure_videos", refused) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + with pytest.raises(RuntimeError): + takeout.run_import(user.id) + + with SessionLocal() as fresh: + state = fresh.execute(select(VideoState).where(VideoState.user_id == user.id)).scalar_one() + assert (state.video_id, state.status) == ("plain000000", "watched") + outcome = fresh.execute( + select(TakeoutItem.outcome).where(TakeoutItem.video_id == "plain000000") + ).scalar_one() + assert outcome == "watched" + + +def test_staging_and_settling_take_the_users_one_lock(db, make_user): + """An upload's staging and an import's settling lock the same rows in no common order: one lock + over both is what keeps the two from deadlocking.""" + user = make_user("lock@example.com") + db.commit() + steps = { + "stage": lambda s, u: service.stage_takeout( + s, u, TakeoutKind.WATCHED, {"aaaaaaaaaaa": (T0, None)} + ), + "watched": lambda s, u: service.apply_takeout_watched(s, u), + "watch later": lambda s, u: service.apply_takeout_watch_later( + s, u, [(1, "aaaaaaaaaaa")], set() + ), + } + with SessionLocal() as holder: + service.lock_takeout(holder, holder.get(User, user.id)) + for name, step in steps.items(): + with SessionLocal() as other: + other.execute(text("SET lock_timeout = '200ms'")) + with pytest.raises(OperationalError, match="lock timeout"): + step(other, other.get(User, user.id)) + pytest.fail(f"{name} did not wait for the user's lock") + holder.rollback() + + # --- the upload -------------------------------------------------------------------------------- @pytest.fixture def queued(monkeypatch): runs = [] - monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw))) + monkeypatch.setattr( + scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, fn.func, fn.args, kw)) + ) return runs @@ -318,6 +515,8 @@ def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice( assert queued == [ ( "takeout_import", + takeout.run_import, + (user.id,), {"actor_id": user.id, "target": f"user:{user.id}", "notify": True}, ) ] @@ -345,8 +544,87 @@ def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_us assert _rows(db, user) == {} -def test_an_anonymous_upload_is_refused(client): - assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401 +def _counting_body(chunks: int = 4, size: int = 1024): + """A body that records how much of itself the server pulled (test_upload.py's probe).""" + pulled: list[int] = [] + + def body(): + for _ in range(chunks): + pulled.append(size) + yield b"x" * size + + return body(), pulled + + +def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client): + body, pulled = _counting_body() + assert client.post("/api/takeout", content=body).status_code == 401 + assert pulled == [], f"the server read {sum(pulled)} bytes from an anonymous caller" + + +def test_a_signed_in_upload_has_its_body_read(db, make_user, as_user, queued): + """The mutation check for the test above: a route reading nobody's body would pass it too.""" + body, pulled = _counting_body() + c = as_user(make_user("read@example.com")) + assert c.post("/api/takeout", content=body).status_code == 422 + assert len(pulled) == 4 + + +def test_uploads_past_the_hourly_limit_are_refused_after_their_body_arrived( + db, make_user, as_user, queued +): + c = as_user(make_user("many@example.com")) + for _ in range(10): + assert c.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200 + body, pulled = _counting_body() + resp = c.post("/api/takeout", content=body) + assert resp.status_code == 429 + assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_cooldown"} + # Read to the end first: refused mid-upload, the browser would say "network error" instead. + assert len(pulled) == 4 + assert len(queued) == 10 + + +def test_a_second_upload_while_one_of_the_users_is_still_arriving_is_refused( + db, make_user, as_user, queued, monkeypatch +): + user = make_user("busy@example.com") + monkeypatch.setattr(feed_routes, "_takeout_uploading", {user.id}) + resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "takeout_busy" + assert queued == [] + # Another user's upload is not held up by it, and lets go of its own place when it is done. + other = as_user(make_user("free@example.com")) + assert other.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200 + assert feed_routes._takeout_uploading == {user.id} + + +def test_an_upload_that_would_keep_too_many_videos_is_refused( + db, make_user, as_user, queued, monkeypatch +): + user = make_user("cap@example.com") + c = as_user(user) + monkeypatch.setattr(takeout, "ROWS_MAX", 3) + two = _history(_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb")) + assert c.post("/api/takeout", content=two).status_code == 200 + assert c.post("/api/takeout", content=two).status_code == 200 # the same videos again: no more + resp = c.post("/api/takeout", content=_history(_view("ccccccccccc"), _view("ddddddddddd"))) + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_too_many", "limit": 3} + assert sorted(_rows(db, user)) == ["aaaaaaaaaaa", "bbbbbbbbbbb"] + assert len(queued) == 2 + + +def test_a_watch_later_file_longer_than_youtube_allows_is_refused(db, make_user, as_user, queued): + body = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode() + resp = as_user(make_user("long@example.com")).post("/api/takeout", content=body) + assert resp.status_code == 422 + assert resp.json()["detail"] == { + "code": "refused", + "reason": "takeout_list_too_long", + "limit": 5000, + } # --- the notice -------------------------------------------------------------------------------- @@ -363,6 +641,50 @@ def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user ] +def test_a_failed_run_tells_its_user_the_failure_never_the_upstreams_text(db, make_user): + user = make_user("failnotice@example.com") + + def fails(): + raise RuntimeError("(psycopg.errors.Something) [SQL: SELECT secret]") + + scheduler._ad_hoc("takeout_import", fails, user.id, "user:1", True) + scheduler._ad_hoc("channel_backfill", fails, user.id, "UC_x") + db.expire_all() + (notice,) = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars() + assert notice.body is None + assert notice.data == { + "job_id": "takeout_import", + "status": "error", + "summary": None, + "result": {"failure": "unexpected"}, + } + # The admin's run row keeps the diagnostic. + run = db.execute(select(JobRun).where(JobRun.job == "takeout_import")).scalar_one() + assert "SELECT secret" in run.error + + +def test_a_queued_import_is_told_apart_from_the_last_one_until_its_run_is_recorded( + db, make_user, as_user, monkeypatch +): + user = make_user("queued@example.com") + added = [] + + class Held: + def add_job(self, *args, **kwargs): + added.append(kwargs["id"]) + + monkeypatch.setattr(scheduler, "_scheduler", Held()) + target = takeout.run_target(user.id) + scheduler.run_ad_hoc("takeout_import", dict, actor_id=user.id, target=target) + assert scheduler.ad_hoc_queued("takeout_import", target) + assert as_user(user).get("/api/takeout").json()["queued"] is True + + scheduler._ad_hoc("takeout_import", dict, user.id, target) + assert not scheduler.ad_hoc_queued("takeout_import", target) + assert as_user(user).get("/api/takeout").json()["queued"] is False + assert added == [f"takeout_import@{target}"] + + # --- the videos that wait ---------------------------------------------------------------------- @@ -410,6 +732,8 @@ def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_u def test_the_status_is_the_users_own_last_run_and_counts(db, make_user, as_user): user, other = make_user("status@example.com"), make_user("other@example.com") assert as_user(user).get("/api/takeout").json() == { + "queued": False, + "max_bytes": takeout.MAX_BYTES, "run": None, "watched": {}, "watch_later": {}, diff --git a/frontend/e2e/global.setup.ts b/frontend/e2e/global.setup.ts index a7942a82..2c68e31a 100644 --- a/frontend/e2e/global.setup.ts +++ b/frontend/e2e/global.setup.ts @@ -1,9 +1,9 @@ import { test as setup, expect, type Page } from "@playwright/test"; import { execFileSync } from "node:child_process"; -import { mkdirSync, readFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { MEMBER_STORAGE } from "./helpers"; +import { MEMBER_STORAGE, SEED_FILE } from "./helpers"; import { retryConnect } from "./retryConnect"; /** @@ -99,6 +99,8 @@ setup("seed + authenticate", async ({ page, browser }) => { { input: readFileSync(SEED_SCRIPT) }, ); console.log(`[e2e seed] ${out.toString().trim()}`); + // What the seed chose, for the specs that need one of its videos (helpers.seeded). + writeFileSync(SEED_FILE, out); } catch (err: unknown) { const e = err as { stderr?: Buffer; message?: string }; throw new Error(`E2E seed failed: ${e.stderr?.toString() || e.message}`); diff --git a/frontend/e2e/helpers.ts b/frontend/e2e/helpers.ts index 8abc629c..e4ddef32 100644 --- a/frontend/e2e/helpers.ts +++ b/frontend/e2e/helpers.ts @@ -1,8 +1,18 @@ +import { readFileSync } from "node:fs"; import { expect, type Page } from "@playwright/test"; /** The plain member's saved session (global.setup.ts signs it in); `member.spec.ts` and * `devices.spec.ts` run from it. */ export const MEMBER_STORAGE = "e2e/.auth/member.json"; +// The seed's own answer (scripts/e2e_seed.py), written by global.setup: the videos it chose. +export const SEED_FILE = "e2e/.auth/seed.json"; + +/** The videos the seed chose for the main account under `key` in its answer. */ +export function seeded(key: string): string[] { + const ids = (JSON.parse(readFileSync(SEED_FILE, "utf-8")) as Record)[key]; + if (!Array.isArray(ids) || !ids.length) throw new Error(`the seed chose no ${key} videos`); + return ids as string[]; +} /** Open the app and wait for the authenticated shell (the nav rail) to be ready. */ export async function openApp(page: Page, path = "/"): Promise { diff --git a/frontend/e2e/takeout.spec.ts b/frontend/e2e/takeout.spec.ts index 1794f171..a42c0d25 100644 --- a/frontend/e2e/takeout.spec.ts +++ b/frontend/e2e/takeout.spec.ts @@ -1,20 +1,16 @@ import { test, expect } from "@playwright/test"; -import { openApp, gotoPage } from "./helpers"; +import { openApp, gotoPage, seeded } from "./helpers"; // Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real // picker, is read, imported in the background, and the section says what it did — a video the // catalog knows is marked watched, one it does not know is remembered for later. The seed clears // the account's kept Takeout videos each run, but the projects share one account: a later one's -// feed no longer holds the video an earlier one marked, so what it adds is counted against what was -// there before it. +// import of the same video adds nothing new, so what each adds is counted against what was there +// before it. test("a Takeout watch history uploads, imports, and the section says what it did", async ({ page, }) => { await openApp(page); - // A video the catalog knows: one on the account's feed (not a Short — the feed leaves them out). - const known = await page.getByTestId("video-card").first().getAttribute("data-video-id"); - expect(known).toBeTruthy(); - await gotoPage(page, "settings"); await page.getByRole("tab", { name: "Account" }).click(); await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible(); @@ -28,6 +24,10 @@ test("a Takeout watch history uploads, imports, and the section says what it did watched?: Record; }; const watchedBefore = before.watched?.watched ?? 0; + // A video the catalog knows and the Shorts probe has cleared, chosen by the seed (one not probed + // yet would wait for the probe): the next one no earlier project of this run has imported. + const known = seeded("takeout_known")[watchedBefore]; + expect(known, "the seed chose too few videos for every project").toBeTruthy(); // Watched now: later than any state the seed set, which the import would otherwise leave alone. const now = new Date().toISOString(); diff --git a/frontend/src/components/ui/DropZone.tsx b/frontend/src/components/ui/DropZone.tsx index ada39182..65a6130c 100644 --- a/frontend/src/components/ui/DropZone.tsx +++ b/frontend/src/components/ui/DropZone.tsx @@ -1,4 +1,4 @@ -import { useRef, useState, type ReactNode } from "react"; +import { useRef, useState, type ReactNode, type Ref } from "react"; import clsx from "clsx"; import { Pressable } from "./Pressable"; @@ -8,6 +8,7 @@ import { Pressable } from "./Pressable"; // children (an icon, a line of text), so a plain boolean flickers — enter and leave are counted // and the highlight clears at depth 0. The caller draws the inside, told whether a drag is over it. export function DropZone({ + ref, onFiles, multiple = false, accept, @@ -17,6 +18,8 @@ export function DropZone({ className, children, }: { + /** The zone's button, for a caller that moves the focus to it. */ + ref?: Ref; /** The files dropped or picked, never empty. One unless `multiple`. */ onFiles: (files: [File, ...File[]]) => void; multiple?: boolean; @@ -40,6 +43,7 @@ export function DropZone({ return ( <> inputRef.current?.click()} onDragEnter={(e) => { diff --git a/frontend/src/i18n/backendIds.contract.test.ts b/frontend/src/i18n/backendIds.contract.test.ts index 9b2496ed..9b08e934 100644 --- a/frontend/src/i18n/backendIds.contract.test.ts +++ b/frontend/src/i18n/backendIds.contract.test.ts @@ -313,6 +313,29 @@ describe("backend ids have their names in both languages", () => { expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours); }); + it("the Takeout import's files and outcomes are the server's", () => { + const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8"); + const ours = (name: string) => + [ + ...( + new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? "" + ).matchAll(/"(\w+)"/g), + ].map((m) => m[1]); + const models = backend("reader/models.py"); + const theirs = (cls: string) => + [ + ...( + new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec( + models, + )?.[1] ?? "" + ).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm), + ].map((m) => m[1]); + expect(theirs("TakeoutKind").length).toBeGreaterThan(0); + expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind")); + expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0); + expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome")); + }); + for (const lang of ["en", "hu"]) { const keys = keysOf(lang); const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k)); diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index be7692c4..c9066502 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -252,6 +252,10 @@ "api_token_name_required": "Give the token a name", "api_token_limit": "You already have {{limit}} tokens — revoke one first", "takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.", - "takeout_no_videos": "This file has no videos to import" + "takeout_no_videos": "This file has no videos to import", + "takeout_busy": "Another of your files is still being uploaded — wait for it to finish", + "takeout_cooldown": "You have uploaded many files in the last hour — try again later", + "takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds", + "takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can" } } diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index fbd6410b..1eafde92 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -187,7 +187,8 @@ "watch_later_present": "already in Watch later", "watch_later_missing": "no longer on YouTube", "watch_later_waiting": "Watch later waiting", - "takeout_watched": "marked watched from Takeout" + "takeout_watched": "marked watched from Takeout", + "takeout_watch_later_added": "put into Watch later from Takeout" }, "skip": { "disabled": "disabled", @@ -205,7 +206,8 @@ }, "failure": { "rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.", - "rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}." + "rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}.", + "unexpected": "The run ran into an unexpected error." }, "rssCause": { "http": "HTTP {{status}}", diff --git a/frontend/src/i18n/locales/en/settings.json b/frontend/src/i18n/locales/en/settings.json index 6126ebbf..3bf5f3d3 100644 --- a/frontend/src/i18n/locales/en/settings.json +++ b/frontend/src/i18n/locales/en/settings.json @@ -52,7 +52,8 @@ "missing_other": "{{n}} no longer on YouTube", "laterWaiting_one": "{{n}} waiting for Watch later", "laterWaiting_other": "{{n}} waiting for Watch later" - } + }, + "tooLarge": "{{name}} is larger than {{limit}} — the import takes files up to that size" }, "save": { "saving": "Saving…", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index cf797edc..6063a8e4 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -252,6 +252,10 @@ "api_token_name_required": "Adj nevet a tokennek", "api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet", "takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.", - "takeout_no_videos": "Ebben a fájlban nincs importálható videó" + "takeout_no_videos": "Ebben a fájlban nincs importálható videó", + "takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik", + "takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később", + "takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van", + "takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet" } } diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 029ad15b..2506ca42 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -187,7 +187,8 @@ "watch_later_present": "már a „Megnézem később” listán", "watch_later_missing": "már nincs a YouTube-on", "watch_later_waiting": "a „Megnézem később” listára vár", - "takeout_watched": "Takeoutból megnézettre jelölve" + "takeout_watched": "Takeoutból megnézettre jelölve", + "takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve" }, "skip": { "disabled": "kikapcsolva", @@ -205,7 +206,8 @@ }, "failure": { "rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.", - "rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}." + "rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}.", + "unexpected": "A futás váratlan hibába ütközött." }, "rssCause": { "http": "HTTP {{status}}", diff --git a/frontend/src/i18n/locales/hu/settings.json b/frontend/src/i18n/locales/hu/settings.json index be0c1039..44498e47 100644 --- a/frontend/src/i18n/locales/hu/settings.json +++ b/frontend/src/i18n/locales/hu/settings.json @@ -44,15 +44,16 @@ "short_other": "{{n}} Short kihagyva", "kept_one": "{{n}} úgy maradt, ahogy itt beállítottad", "kept_other": "{{n}} úgy maradt, ahogy itt beállítottad", - "added_one": "{{n}} a „Megnézem később” listába", - "added_other": "{{n}} a „Megnézem később” listába", + "added_one": "{{n}} a „Megnézem később” listába téve", + "added_other": "{{n}} a „Megnézem később” listába téve", "present_one": "{{n}} már a „Megnézem később” listán", "present_other": "{{n}} már a „Megnézem később” listán", "missing_one": "{{n}} már nincs fent a YouTube-on", "missing_other": "{{n}} már nincs fent a YouTube-on", "laterWaiting_one": "{{n}} vár a „Megnézem később” listára", "laterWaiting_other": "{{n}} vár a „Megnézem később” listára" - } + }, + "tooLarge": "{{name}}: nagyobb, mint {{limit}} — az import legfeljebb ekkora fájlt fogad" }, "save": { "saving": "Mentés…", diff --git a/frontend/src/lib/api/feed.ts b/frontend/src/lib/api/feed.ts index 0ffbfe83..41a89376 100644 --- a/frontend/src/lib/api/feed.ts +++ b/frontend/src/lib/api/feed.ts @@ -164,12 +164,26 @@ export interface TakeoutUpload { other: number; malformed: number; } -export type TakeoutKind = "watched" | "watch_later"; -// A kept video's outcome (`reader.models.TakeoutOutcome`), or "waiting" while the catalog does not -// know it yet: watched / kept (the user's own state here stood) / short for the history; added / -// present / missing for Watch later. -export type TakeoutCounts = Partial>; +// The two files (`reader.models.TakeoutKind`) and a kept video's outcome (`TakeoutOutcome`): watched +// / kept (the user's own state here stood) / short for the history; added / present / missing for +// Watch later. backendIds.contract.test holds both lists to the backend's enums. +export const TAKEOUT_KINDS = ["watched", "watch_later"] as const; +export const TAKEOUT_OUTCOMES = [ + "watched", + "kept", + "short", + "added", + "present", + "missing", +] as const; +export type TakeoutKind = (typeof TAKEOUT_KINDS)[number]; +// Each outcome's count, and "waiting" for the videos the import could not settle yet. +export type TakeoutCounts = Partial>; export interface TakeoutStatus { + /** An import is queued and its run not started: `run` is still the one before it. */ + queued: boolean; + /** The largest file the upload takes. */ + max_bytes: number; run: { status: RunStatus; started_at: string | null; diff --git a/frontend/src/modules/settings/TakeoutImport.dom.test.tsx b/frontend/src/modules/settings/TakeoutImport.dom.test.tsx index 0675226a..88701b94 100644 --- a/frontend/src/modules/settings/TakeoutImport.dom.test.tsx +++ b/frontend/src/modules/settings/TakeoutImport.dom.test.tsx @@ -56,6 +56,14 @@ const uploaded = (videos: number): TakeoutUpload => ({ malformed: 0, }); +// A whole status: nothing queued and room for any file, unless the test says otherwise. +const st = (s: Partial): TakeoutStatus => ({ + queued: false, + max_bytes: 1_000_000, + run: null, + ...s, +}); + function mount(status: TakeoutStatus | Error | Promise) { const spy = vi.spyOn(api, "takeoutStatus"); if (status instanceof Error) spy.mockRejectedValue(status); @@ -77,11 +85,13 @@ const drop = (files: File[]) => act(async () => { fireEvent.drop(zone()!, { dataTransfer: { files } }); }); -const file = (name: string) => new File(["[]"], name); +const file = (name: string, size = 2) => new File(["x".repeat(size)], name); +const unavailable = (el: HTMLElement | null) => + !!el && ((el as HTMLButtonElement).disabled || el.getAttribute("aria-disabled") === "true"); describe("YouTube history import · what it says", () => { it("before the first import: the drop zone, and no line about an import", async () => { - mount({ run: null, watched: {}, watch_later: {} }); + mount(st({ run: null, watched: {}, watch_later: {} })); await waitFor(() => expect(zone()).toBeTruthy()); expect(screen.queryByTestId("takeout-last")).toBeNull(); }); @@ -92,11 +102,13 @@ describe("YouTube history import · what it says", () => { }); it("after one: when, then only the counts it produced, the history's first", async () => { - mount({ - run: run("ok"), - watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 }, - watch_later: { added: 41, waiting: 2 }, - }); + mount( + st({ + run: run("ok"), + watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 }, + watch_later: { added: 41, waiting: 2 }, + }), + ); await waitFor(() => expect(screen.getByTestId("takeout-last")).toBeTruthy()); expect(line()).toBe( [ @@ -114,20 +126,20 @@ describe("YouTube history import · what it says", () => { }); it("a failed run and one a restart cut off say so, not counts", async () => { - mount({ run: run("error"), watched: { watched: 3 } }); + mount(st({ run: run("error"), watched: { watched: 3 } })); await waitFor(() => expect(line()).toBe("settings.takeout.failed(at F)")); cleanup(); - mount({ run: run("interrupted"), watched: { watched: 3 } }); + mount(st({ run: run("interrupted"), watched: { watched: 3 } })); await waitFor(() => expect(line()).toBe("settings.takeout.interrupted(at S)")); }); - it("a running import: the running line, no drop zone, asked again until it ends", async () => { - const spy = mount({ run: run("running"), watched: {} }); + it("a running import: the running line, the zone unavailable, asked again until it ends", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); await waitFor(() => expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"), ); - expect(zone()).toBeNull(); - spy.mockResolvedValue({ run: run("ok"), watched: { watched: 5 } }); + expect(unavailable(zone())).toBe(true); + spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 5 } })); await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(5,5)"), { timeout: 3000, }); @@ -143,7 +155,7 @@ describe("YouTube history import · what it says", () => { }); it("a re-read that fails does not leave the previous answer standing under it", async () => { - const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} }); + const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); await waitFor(() => expect(line()).toContain("OLD")); fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); @@ -158,7 +170,7 @@ describe("YouTube history import · what it says", () => { describe("YouTube history import · the upload", () => { it("sends each dropped file on its own, says each outcome, and reads the new run", async () => { - const status = mount({ run: null, watched: {} }); + const status = mount(st({ run: null, watched: {} })); await waitFor(() => expect(zone()).toBeTruthy()); const refused = new HttpError(422, "refused"); const sent: string[] = []; @@ -167,7 +179,7 @@ describe("YouTube history import · the upload", () => { if (f.name === "bad.html") throw refused; return uploaded(f.name === "watch-history.json" ? 9034 : 56); }); - status.mockResolvedValue({ run: run("ok"), watched: { watched: 7 } }); + status.mockResolvedValue(st({ run: run("ok"), watched: { watched: 7 } })); await drop([file("watch-history.json"), file("bad.html"), file("Watch later-videos.csv")]); await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(7,7)")); @@ -183,7 +195,7 @@ describe("YouTube history import · the upload", () => { }); it("names the file being sent, then never shows the previous import's line before the new one", async () => { - const status = mount({ run: run("ok", { finished_at: "OLD" }), watched: {} }); + const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); await waitFor(() => expect(line()).toContain("OLD")); fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); let finish!: (v: TakeoutUpload) => void; @@ -197,7 +209,115 @@ describe("YouTube history import · the upload", () => { await act(async () => finish(uploaded(1))); // The new run is still being read: the upload's line stays, the old import's does not return. expect(screen.queryByTestId("takeout-last")).toBeNull(); - await act(async () => answer({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await act(async () => answer(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }))); await waitFor(() => expect(line()).toContain("NEW")); }); }); + +describe("YouTube history import · while it runs", () => { + it("a queued import is this upload's, not the last one: the running line until its run reads", async () => { + const spy = mount(st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => + expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"), + ); + expect(screen.queryByTestId("takeout-last")).toBeNull(); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + }); + + it("a re-read that fails mid-run does not leave the spinner claiming progress under it", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); + await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy()); + spy.mockRejectedValue(new HttpError(500, "boom")); + await waitFor( + () => expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"), + { timeout: 3000 }, + ); + expect(screen.queryByTestId("takeout-running")).toBeNull(); + spy.mockResolvedValue(st({ run: run("ok"), watched: {} })); + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 3000 }); + }); + + it("one live region says the run and then its outcome", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); + await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy()); + const region = screen.getByRole("status"); + spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 2 } })); + await waitFor(() => expect(line()).toContain("watched(2,2)"), { timeout: 3000 }); + expect(screen.getByRole("status")).toBe(region); + expect(region.contains(screen.getByTestId("takeout-last"))).toBe(true); + }); + + it("a file dropped beside the zone is kept from the browser, which would open it", async () => { + mount(st({ run: null, watched: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + const missed = (type: string, types: string[]) => { + const e = new Event(type, { bubbles: true, cancelable: true }); + Object.defineProperty(e, "dataTransfer", { value: { types, dropEffect: "copy" } }); + document.body.dispatchEvent(e); + return e; + }; + expect(missed("drop", ["Files"]).defaultPrevented).toBe(true); + expect(missed("dragover", ["Files"]).defaultPrevented).toBe(true); + // A drag of something else (a link, text) is the page's own business. + expect(missed("drop", ["text/plain"]).defaultPrevented).toBe(false); + cleanup(); + expect(missed("drop", ["Files"]).defaultPrevented).toBe(false); + }); + + it("a file larger than the import takes is refused before it is sent, naming the limit", async () => { + mount(st({ max_bytes: 10, run: run("ok"), watched: {} })); + await waitFor(() => expect(line()).toBeTruthy()); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + const send = vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + await drop([file("watch-history.json", 11)]); + expect(send).not.toHaveBeenCalled(); + expect(mocks.notify).toHaveBeenCalledWith({ + level: "error", + message: "settings.takeout.tooLarge(watch-history.json,10 B)", + }); + // Nothing was taken: the zone stays open for the right file, and so after a refusal. + await waitFor(() => expect(zone()).toBeTruthy()); + send.mockRejectedValue(new HttpError(422, "refused")); + await drop([file("bad.html")]); + await waitFor(() => expect(mocks.notifyActionError).toHaveBeenCalled()); + expect(zone()).toBeTruthy(); + }); + + it("the focus follows: New import hands it to the zone, the zone back to New import", async () => { + const spy = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => expect(line()).toContain("OLD")); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + expect(document.activeElement).toBe(zone()); + + vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + spy.mockResolvedValue( + st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }), + ); + await drop([file("watch-history.json")]); + // While it runs the zone stays, unavailable, and keeps the focus. + expect(document.activeElement).toBe(zone()); + expect(unavailable(zone())).toBe(true); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + expect(zone()).toBeNull(); + expect(document.activeElement).toBe( + screen.getByRole("button", { name: "settings.takeout.newImport" }), + ); + }); + + it("a focus the user moved elsewhere meanwhile is not taken back", async () => { + const spy = mount(st({ run: null, watched: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + const elsewhere = document.body.appendChild(document.createElement("button")); + zone()!.focus(); + vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + spy.mockResolvedValue(st({ queued: true, run: null, watched: {} })); + await drop([file("watch-history.json")]); + elsewhere.focus(); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + expect(document.activeElement).toBe(elsewhere); + elsewhere.remove(); + }); +}); diff --git a/frontend/src/modules/settings/TakeoutImport.tsx b/frontend/src/modules/settings/TakeoutImport.tsx index 2c57466a..9bab8821 100644 --- a/frontend/src/modules/settings/TakeoutImport.tsx +++ b/frontend/src/modules/settings/TakeoutImport.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { RefreshCw, UploadCloud } from "lucide-react"; @@ -6,19 +6,19 @@ import { Section } from "../../components/ui/form"; import { Button } from "../../components/ui/Button"; import { DropZone } from "../../components/ui/DropZone"; import { StateMessage } from "../../components/ui/QueryState"; -import { api, type TakeoutStatus } from "../../lib/api"; +import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api"; +import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed"; import { qk } from "../../lib/queryKeys"; import { notify } from "../../lib/notifications"; import { notifyActionError } from "../../lib/actionErrors"; -import { formatDateTime, formatNumber } from "../../lib/format"; +import { formatBytes, formatDateTime, formatNumber } from "../../lib/format"; +import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch"; // Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and // their YouTube Watch later list reach no API, so the export's two files are uploaded here, one // request each; the server tells them apart and runs the import in the background, and its outcome // also reaches the inbox. The import has no steps to count (the whole history is applied in one -// go), so while it runs the section says so and asks again until it is done. - -const POLL_MS = 1500; +// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`). // The outcome counts, in the order they are said: the history's, then Watch later's. A count the // import did not produce is not said at all. @@ -35,6 +35,12 @@ const WATCH_LATER = [ ["waiting", "laterWaiting"], ] as const; +// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it). +type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number]; +type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0]; +const everyOutcomeSaid: [Exclude] extends [never] ? true : never = true; +void everyOutcomeSaid; + function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] { const parts: string[] = []; for (const [outcome, key] of HISTORY) { @@ -52,58 +58,111 @@ export default function TakeoutImport() { const { t, i18n } = useTranslation(); const lang = i18n.language; const qc = useQueryClient(); - const status = useQuery({ - queryKey: qk.takeout(), - queryFn: api.takeoutStatus, - refetchInterval: (q) => (q.state.data?.run?.status === "running" ? POLL_MS : false), - }); + const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus }); const [uploading, setUploading] = useState(null); // "New import" reopens the drop zone over a finished one; before the first import it is open. const [reopened, setReopened] = useState(false); const run = status.data?.run ?? null; - const running = uploading !== null || run?.status === "running"; + // Queued counts too: until its run row is written, `run` is still the import before it. + const busy = importBusy(status.data); + const running = uploading !== null || busy; const finishedAt = run?.finished_at ?? null; - // A run that ended has changed what the feed and Watch later show: those lists are read again - // when the user next opens them, not 30 s later from the cache. + // A run under way is watched to its end, even if the user leaves Settings meanwhile. useEffect(() => { - if (!finishedAt) return; - void qc.invalidateQueries({ queryKey: qk.feed() }); - void qc.invalidateQueries({ queryKey: qk.playlist() }); + if (busy) void watchTakeout(qc); + }, [busy, qc]); + // One that ended while nothing here watched it (a reload, another tab) has changed what the feed + // and Watch later show too: those lists are read again when next opened, not from the cache. + useEffect(() => { + if (finishedAt) invalidateImported(qc); }, [finishedAt, qc]); + // A file dropped beside the zone, or while it is not there, is kept from the browser: it would + // open the file in the tab, unloading the app and the upload under way. The zone takes its own. + useEffect(() => { + const missed = (e: DragEvent) => { + if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return; + e.preventDefault(); + if (e.type === "dragover") e.dataTransfer.dropEffect = "none"; + }; + window.addEventListener("dragover", missed); + window.addEventListener("drop", missed); + return () => { + window.removeEventListener("dragover", missed); + window.removeEventListener("drop", missed); + }; + }, []); + + // Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and + // the zone, gone once the import it started is done, hands it to "New import". + const zone = useRef(null); + const focusNext = useRef<"zone" | "newImport" | null>(null); + const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => { + if (!el || focusNext.current !== which) return; + focusNext.current = null; + // Only focus that was lost with the control that held it: never taken from where the user went. + if (document.activeElement === null || document.activeElement === document.body) el.focus(); + }; + const zoneRef = (el: HTMLButtonElement | null) => { + zone.current = el; + focusIfNext("zone", el); + }; + const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el); + const upload = async (files: File[]) => { - for (const file of files) await uploadOne(file); + if (document.activeElement === zone.current) focusNext.current = "newImport"; + let taken = false; + for (const file of files) if (await uploadOne(file)) taken = true; // The new run read first, then the upload's line let go: the other way round the previous // import's line showed for a moment, as if this upload had done nothing. await qc.invalidateQueries({ queryKey: qk.takeout() }); setUploading(null); - setReopened(false); + // Nothing taken, nothing imported: the zone stays open for the right file. + if (taken) setReopened(false); + else focusNext.current = null; }; - const uploadOne = async (file: File) => { - setUploading(file.name); - try { - const res = await api.takeoutUpload(file); + /** Send one file; whether the server took it. */ + const uploadOne = async (file: File): Promise => { + const max = status.data?.max_bytes ?? Infinity; + if (file.size > max) { + // Refused here, not after minutes of upload, and with the limit named. notify({ - level: "success", - message: t("settings.takeout.read", { - name: file.name, - count: res.videos, - n: formatNumber(res.videos, lang), - }), + level: "error", + message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }), }); + return false; + } + setUploading(file.name); + let res: TakeoutUpload | null = null; + try { + res = await api.takeoutUpload(file); } catch (e) { // A refusal names its reason (not a Takeout file, nothing in it); the next file still goes. notifyActionError(e, t("errors.writeFailed.takeoutUpload")); } + if (res === null) return false; + notify({ + level: "success", + message: t("settings.takeout.read", { + name: file.name, + count: res.videos, + n: formatNumber(res.videos, lang), + }), + }); + return true; }; const say = (key: string, n: number) => t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) }); const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : ""); // Not while the last import is still being asked for: a zone that flashes before the line that - // replaces it would offer a first import to someone who has done one. - const showZone = status.isSuccess && !running && (run === null || reopened); + // replaces it would offer a first import to someone who has done one. While an import runs the + // zone stays, unavailable: the control that started it keeps the focus. + const showZone = status.isSuccess && (running || run === null || reopened); + // A failed re-read keeps the previous answer: it is not shown under the line saying the read + // failed, as if it were current — a run's spinner no more than its outcome. + const current = !status.isError || uploading !== null; return (
@@ -117,7 +176,14 @@ export default function TakeoutImport() { {showZone && ( <> - + {(dragOver) => ( <> @@ -138,42 +204,49 @@ export default function TakeoutImport() { )} - {running ? ( -
- - {uploading !== null - ? t("settings.takeout.uploading", { name: uploading }) - : t("settings.takeout.running")} +
+ {/* One live region for the run and its outcome, mounted throughout: what replaces + "Processing…" is announced, which a region mounted with its text is not. */} +
+ {running && current ? ( + + + {uploading !== null + ? t("settings.takeout.uploading", { name: uploading }) + : t("settings.takeout.running")} + + ) : ( + !running && + run && + status.data && + current && ( +

+ {run.status === "error" + ? t("settings.takeout.failed", { when: when(run.finished_at) }) + : run.status === "interrupted" + ? t("settings.takeout.interrupted", { when: when(run.started_at) }) + : [ + t("settings.takeout.last", { when: when(run.finished_at) }), + ...countParts(status.data, say), + ].join(" · ")} +

+ ) + )}
- ) : ( - // A failed re-read keeps the previous answer: it is not shown under the line saying the - // read failed, as if it were current. - run && - status.data && - !status.isError && ( -
-

- {run.status === "error" - ? t("settings.takeout.failed", { when: when(run.finished_at) }) - : run.status === "interrupted" - ? t("settings.takeout.interrupted", { when: when(run.started_at) }) - : [ - t("settings.takeout.last", { when: when(run.finished_at) }), - ...countParts(status.data, say), - ].join(" · ")} -

- {!reopened && ( - - )} -
- ) - )} + {!running && run && status.data && current && !reopened && ( + + )} +
); } diff --git a/frontend/src/modules/settings/takeoutWatch.test.ts b/frontend/src/modules/settings/takeoutWatch.test.ts new file mode 100644 index 00000000..c2d435d2 --- /dev/null +++ b/frontend/src/modules/settings/takeoutWatch.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment jsdom +// +// A Takeout import is watched to its end whether or not its section is still on screen, and what it +// changed is read afresh then: the feed and its counts, Watch later, and the playlists' sizes. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { QueryClient } from "@tanstack/react-query"; +import { api, type TakeoutStatus } from "../../lib/api"; +import { qk } from "../../lib/queryKeys"; +import { importBusy, watchTakeout } from "./takeoutWatch"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +const st = (s: Partial): TakeoutStatus => ({ + queued: false, + max_bytes: 1, + run: null, + ...s, +}); +const running = { + status: "running", + started_at: "S", + finished_at: null, + result: null, + failure: null, +}; + +describe("the Takeout import's watch", () => { + it("a queued import is busy as much as a running one", () => { + expect(importBusy(st({ queued: true }))).toBe(true); + expect(importBusy(st({ run: { ...running, status: "running" } as TakeoutStatus["run"] }))).toBe( + true, + ); + expect(importBusy(st({ run: { ...running, status: "ok" } as TakeoutStatus["run"] }))).toBe( + false, + ); + expect(importBusy(undefined)).toBe(false); + }); + + it("asks until the import is done, then reads again everything it changed", async () => { + vi.useFakeTimers(); + const qc = new QueryClient(); + qc.setQueryData(qk.takeout(), st({ queued: true })); + const answers = [st({ run: running as TakeoutStatus["run"] }), st({})]; + const read = vi.spyOn(api, "takeoutStatus").mockImplementation(async () => answers.shift()!); + const stale = vi.spyOn(qc, "invalidateQueries"); + + const done = watchTakeout(qc); + void watchTakeout(qc); // a second call while it watches is the same watch + await vi.advanceTimersByTimeAsync(1500); + expect(read).toHaveBeenCalledTimes(1); + expect(stale).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1500); + await done; + + expect(read).toHaveBeenCalledTimes(2); + expect(stale.mock.calls.map(([f]) => f?.queryKey)).toEqual([ + qk.feed(), + qk.feedSummary(), + qk.feedYears(), + qk.playlist(), + qk.playlists(), + ]); + }); +}); diff --git a/frontend/src/modules/settings/takeoutWatch.ts b/frontend/src/modules/settings/takeoutWatch.ts new file mode 100644 index 00000000..13855b2f --- /dev/null +++ b/frontend/src/modules/settings/takeoutWatch.ts @@ -0,0 +1,57 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { api, type TakeoutStatus } from "../../lib/api"; +import { qk } from "../../lib/queryKeys"; + +// A Takeout import, watched to its end: its status is asked again until no run is queued or going, +// then the lists it changed are read afresh. It lives outside the section on purpose: the user may +// leave Settings while the import runs, and the feed they go to must not keep showing as new what +// the import has just marked watched. + +const POLL_MS = 1500; + +export const importBusy = (s: TakeoutStatus | undefined): boolean => + !!s && (s.queued || s.run?.status === "running"); + +let watching = false; + +const pause = () => new Promise((done) => setTimeout(done, POLL_MS)); + +async function read(qc: QueryClient): Promise { + try { + return await qc.fetchQuery({ + queryKey: qk.takeout(), + queryFn: api.takeoutStatus, + staleTime: 0, + }); + } catch { + // A read that failed is the section's to say (its query holds the error); the watch asks again. + return qc.getQueryData(qk.takeout()); + } +} + +/** Mark stale everything an import may have changed: the feed and its counts, Watch later, and the + * playlists that show its size. */ +export function invalidateImported(qc: QueryClient): void { + for (const queryKey of [ + qk.feed(), + qk.feedSummary(), + qk.feedYears(), + qk.playlist(), + qk.playlists(), + ]) + void qc.invalidateQueries({ queryKey }); +} + +/** Watch the user's import until it is done (once: a second call while it watches is that watch), + * then mark stale what it changed. */ +export async function watchTakeout(qc: QueryClient): Promise { + if (watching) return; + watching = true; + let status = qc.getQueryData(qk.takeout()); + while (importBusy(status)) { + await pause(); + status = await read(qc); + } + watching = false; + invalidateImported(qc); +} diff --git a/scripts/e2e_seed.py b/scripts/e2e_seed.py index c4a1be6c..8fa5236c 100644 --- a/scripts/e2e_seed.py +++ b/scripts/e2e_seed.py @@ -181,6 +181,28 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: saved = svid.id db.commit() + # 5. Videos the Takeout import marks watched: from the subscribed channels, and already cleared by + # the Shorts probe — one not yet probed would wait for it, and the import spec would count + # nothing. One per browser project: they share this account, and a video imported once is not + # marked again. + known = ( + list( + db.scalars( + select(Video.id) + .where( + Video.channel_id.in_(channel_ids), + Video.shorts_probed.is_(True), + Video.is_short.is_(False), + Video.id.not_in(used | {saved} if saved else used), + ) + .order_by(Video.published_at.desc()) + .limit(5) + ) + ) + if channel_ids + else [] + ) + return { "user_id": user.id, "email": email, @@ -188,6 +210,7 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: "in_progress": in_progress, "watched": watched, "saved": saved, + "takeout_known": known, } From 9c5f81fd944193c6249e1438a03259cd0a5f45bf Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 22:07:11 +0200 Subject: [PATCH 7/8] fix(reader): the Takeout import keeps a later upload, an un-mark and its memory bound - An ad-hoc request made while its target's run is under way is the run after it: the scheduler dropped a second fire of a running id, so a second Takeout file was never imported and its key stayed queued until a restart. - Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view no longer marks it watched again; un-marks take the user's Takeout lock. - One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the others are settled, and a failed lookup spends none of the daily share. - The upload reads the user's id before giving the read connection back. - A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a mark never pins it against the cleanup jobs. - The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and two uploads at once across all users (takeout_server_busy). - A Watch later list goes in as one presence read and one insert. --- .../alembic/versions/0114_video_unmarks.py | 44 +++ backend/app/bodylimit.py | 3 +- backend/app/ratelimit.py | 8 + backend/app/reader/models.py | 15 + backend/app/reader/queries.py | 42 +++ backend/app/reader/service.py | 53 +++- backend/app/reader/takeout.py | 156 ++++++++--- backend/app/routes/feed.py | 29 +- backend/app/scheduler.py | 73 ++++- backend/tests/conftest.py | 12 + backend/tests/test_scheduler.py | 35 +++ backend/tests/test_takeout.py | 258 +++++++++++++++++- backend/tests/test_unit_fences.py | 1 + frontend/src/i18n/locales/en/errors.json | 1 + frontend/src/i18n/locales/en/scheduler.json | 3 +- frontend/src/i18n/locales/hu/errors.json | 1 + frontend/src/i18n/locales/hu/scheduler.json | 3 +- 17 files changed, 667 insertions(+), 70 deletions(-) create mode 100644 backend/alembic/versions/0114_video_unmarks.py diff --git a/backend/alembic/versions/0114_video_unmarks.py b/backend/alembic/versions/0114_video_unmarks.py new file mode 100644 index 00000000..64e15403 --- /dev/null +++ b/backend/alembic/versions/0114_video_unmarks.py @@ -0,0 +1,44 @@ +"""When a user last took a video back to "new" + +Revision ID: 0114_video_unmarks +Revises: 0113_takeout_items +Create Date: 2026-10-23 + +Un-marking or resetting a video leaves no `video_states` row, so this is what tells an import of the +user's YouTube history a video they un-marked here from one never touched. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0114_video_unmarks" +down_revision: str | None = "0113_takeout_items" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "video_unmarks", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "video_id", + sa.String(16), + sa.ForeignKey("videos.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("user_id", "video_id", name="uq_video_unmark"), + ) + op.create_index("ix_video_unmarks_video_id", "video_unmarks", ["video_id"]) + + +def downgrade() -> None: + op.drop_index("ix_video_unmarks_video_id", table_name="video_unmarks") + op.drop_table("video_unmarks") diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index 432c6e1c..43e88f03 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -75,7 +75,8 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390 # bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an # account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The -# file is read an entry at a time (`takeout._json_list`), so it costs its own size, not a tree of it. +# file's text is decoded a window at a time and read an entry at a time (`takeout._json_list`), so +# it costs its bytes and a window, not a tree of it; the route takes two at once across every user. TAKEOUT_MAX_BYTES = 32 * 1024 * 1024 ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { diff --git a/backend/app/ratelimit.py b/backend/app/ratelimit.py index 6dcef515..313409e5 100644 --- a/backend/app/ratelimit.py +++ b/backend/app/ratelimit.py @@ -42,6 +42,14 @@ class RateLimiter: with self._lock: return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events + def room(self, key: str) -> float: + """How much cost ``key`` may still spend in the current window, WITHOUT counting anything — + for work counted only once it has succeeded: ask it first, ``allow`` what was done.""" + cutoff = time.monotonic() - self.window + with self._lock: + used = sum(c for t, c in self._hits.get(key, []) if t > cutoff) + return max(0.0, self.max_events - used) + def allow(self, key: str, cost: float = 1) -> bool: """Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit. diff --git a/backend/app/reader/models.py b/backend/app/reader/models.py index e5e3f031..7ff5d7b1 100644 --- a/backend/app/reader/models.py +++ b/backend/app/reader/models.py @@ -85,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) +class VideoUnmark(Base): + """When the user last took a video back to "new" here — un-marked it, or reset it. That leaves + no `VideoState` (its absence is "new"), so without this row a later import of what they watched + on YouTube could not tell a video they un-marked from one never touched, and marked it watched + again. Kept by user and video; the video's own deletion takes it.""" + + __tablename__ = "video_unmarks" + __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True) + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + class Playlist(Base, TimestampMixin, UpdatedAtMixin): """A per-user named, ordered collection of videos from the shared catalog. diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 9ac01add..b06eb402 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -26,6 +26,7 @@ from app.reader.models import ( TakeoutItem, TakeoutKind, VideoState, + VideoUnmark, ) from app.userscope import owned_by, owned_row @@ -376,6 +377,16 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan # --- Google Takeout -------------------------------------------------------------------------- +def _settled_here() -> ColumnElement[bool]: + """A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a + discovery video only a search or an explore brought in. A mark is a `VideoState`, which the + cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the + video, and its explored channel, for good. Such a row waits until the video is the catalog's own + (its channel subscribed to) — or is gone.""" + subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists() + return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed)) + + def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]: """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts: (row id, video id, Takeout time, is Short).""" @@ -388,6 +399,7 @@ def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datet TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.outcome.is_(None), Video.shorts_probed, + _settled_here(), ) ).tuples() ) @@ -421,6 +433,35 @@ def known_video_ids(db: Session, ids: list[str]) -> set[str]: return known +def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]: + """When the user last took each of `ids` back to "new" here, for those they ever did.""" + out: dict[str, datetime] = {} + for start in range(0, len(ids), _IN_CHUNK): + rows = db.execute( + select(VideoUnmark.video_id, VideoUnmark.at).where( + owned_by(VideoUnmark.user_id, user), + VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]), + ) + ).tuples() + out.update((vid, at) for vid, at in rows) + return out + + +def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]: + """Which of `ids` the playlist already holds.""" + held: set[str] = set() + for start in range(0, len(ids), _IN_CHUNK): + chunk = ids[start : start + _IN_CHUNK] + held.update( + db.execute( + select(PlaylistItem.video_id).where( + PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk) + ) + ).scalars() + ) + return held + + def takeout_rows_besides( db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str] ) -> int: @@ -476,6 +517,7 @@ def takeout_users_ready(db: Session) -> list[int]: TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.outcome.is_(None), Video.shorts_probed, + _settled_here(), ) .distinct() ).scalars() diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index 4d396aee..08bbabc2 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -25,6 +25,7 @@ from app.reader.models import ( TakeoutKind, TakeoutOutcome, VideoState, + VideoUnmark, ) from app.sync.playlists import lock_playlists, recompute_dirty from app.userscope import owned_by @@ -206,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None: the in-progress state, not where the user left off); otherwise the row goes. The player fires this beside a checkpoint that may delete the same row: the caller's commit then raises `StaleDataError`, which is that lost race and nothing worse.""" + lock_takeout(db, user) # an import settling meanwhile reads the un-mark below, or comes first row = queries.video_state(db, user, video_id) if row is None: return @@ -214,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None: row.watched_at = None else: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() +def _remember_unmark(db: Session, user: User, video_id: str) -> None: + """Note that the user took the video back to "new" now: the state they set leaves no row, and + an import of their YouTube history must not mark it watched again (`apply_takeout_watched`).""" + now = datetime.now(timezone.utc) + db.execute( + pg_insert(VideoUnmark) + .values(user_id=user.id, video_id=video_id, at=now) + .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now}) + ) + + def mark(db: Session, user: User, video_id: str, status: str) -> None: """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then @@ -241,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None: def clear_state(db: Session, user: User, video_id: str) -> None: """The video back to pristine for this user: status, resume point and `watched_at` all go. Playlist membership lives elsewhere and stays.""" + lock_takeout(db, user) # as `unmark` row = queries.video_state(db, user, video_id) if row is not None: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() @@ -517,7 +533,8 @@ _TAKEOUT_LOCK_NS = 0x544F def lock_takeout(db: Session, user: User) -> None: """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an import's settling each lock many of the same rows, one statement after another and in no common - order, so without one lock over both the two can deadlock.""" + order, so without one lock over both the two can deadlock. An un-mark takes it too: the import + reads the un-marks before it marks, and one landing between the two would be overwritten.""" db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id))) @@ -558,15 +575,21 @@ def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None: def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one - past is not watching it), and a state the user set here wins: watched or hidden, or anything - changed after the Takeout time. The upsert's own condition is that rule, so a state set while - the import runs wins too.""" + past is not watching it), and a state the user set here wins: watched or hidden, anything + changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own + condition is that rule, so a state set while the import runs wins too.""" lock_takeout(db, user) short: list[int] = [] todo: dict[str, tuple[int, datetime]] = {} - for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user): + ready = queries.takeout_watched_ready(db, user) + unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready]) + kept: list[int] = [] + for row_id, video_id, at, is_short in ready: if is_short: short.append(row_id) + elif video_id in unmarked and unmarked[video_id] >= at: + # Un-marked here since they watched it there: theirs, as a state they set is. + kept.append(row_id) else: todo[video_id] = (row_id, at) marked: set[str] = set() @@ -593,7 +616,7 @@ def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: ).returning(VideoState.video_id) ).scalars() ) - kept = [row_id for vid, (row_id, _) in items if vid not in marked] + kept += [row_id for vid, (row_id, _) in items if vid not in marked] _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED) _settle(db, kept, TakeoutOutcome.KEPT) _settle(db, short, TakeoutOutcome.SHORT) @@ -610,17 +633,27 @@ def apply_takeout_watch_later( if not rows: return {} lock_takeout(db, user) - known = queries.known_video_ids(db, [vid for _, vid in rows]) + ids = [vid for _, vid in rows] + known = queries.known_video_ids(db, ids) pl = watch_later(db, user) + # What add_item does a video at a time, for the whole list at once: one presence read and one + # insert, not four statements a video (a full list is thousands). Locked before the read, as there. + next_position = next_item_position(db, pl.id) + held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known]) done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} for row_id, vid in rows: if vid not in known: if vid in asked: done[TakeoutOutcome.MISSING].append(row_id) - elif add_item(db, pl, vid, mark_dirty=False): - done[TakeoutOutcome.ADDED].append(row_id) - else: + elif vid in held: done[TakeoutOutcome.PRESENT].append(row_id) + else: + db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position)) + next_position += 1 + held.add(vid) + done[TakeoutOutcome.ADDED].append(row_id) + if done[TakeoutOutcome.ADDED]: + touch(pl) for outcome, ids in done.items(): _settle(db, ids, outcome) db.flush() diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py index 67231e33..35d5fc1d 100644 --- a/backend/app/reader/takeout.py +++ b/backend/app/reader/takeout.py @@ -15,6 +15,7 @@ Ads". The title is in the account's language, so nothing here reads it. A Short header row (in the account's language) and `,` rows. """ +import codecs import csv import io import json @@ -28,7 +29,7 @@ from itertools import chain from sqlalchemy.orm import Session -from app import quota +from app import logs, quota from app.bodylimit import TAKEOUT_MAX_BYTES from app.db import SessionLocal from app.models import User @@ -39,6 +40,8 @@ from app.sync.runner import get_service_user from app.sync.videos import ensure_videos from app.youtube.client import YouTubeClient +log = logs.get_logger(__name__) + # The largest file the upload takes (its edge cap and the route's own are one number). MAX_BYTES = TAKEOUT_MAX_BYTES # The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is @@ -150,44 +153,113 @@ def _skip(text: str, at: int) -> int: return at -def _json_list(text: str) -> Iterator[object]: - """The entries of a JSON list, one at a time: the whole tree is never built, so a file of a - million tiny entries costs its text and one entry, not a million dicts at once.""" +# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a +# character (one emoji in a title is enough), several times the file on top of its bytes. +_WINDOW = 1024 * 1024 + + +class _Window: + """The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on. + The bytes are the caller's, so only the window is this reader's memory.""" + + def __init__(self, body: bytes, start: int): + self._body, self._next = body, start + self._decoder = codecs.getincrementaldecoder("utf-8")() # strict: bad UTF-8 raises + self.text, self.at = "", 0 + + @property + def done(self) -> bool: + return self._next >= len(self._body) + + def more(self, at_least: int = 0) -> None: + """Decode the next window (at least `at_least` bytes), dropping what is read already.""" + self.text, self.at = self.text[self.at :], 0 + size = max(_WINDOW, at_least) + chunk = self._body[self._next : self._next + size] + self._next += len(chunk) + self.text += self._decoder.decode(chunk, final=self.done) + + def skip(self) -> None: + """Past the whitespace, reading on as far as it goes.""" + while True: + self.at = _skip(self.text, self.at) + if self.at < len(self.text) or self.done: + return + self.more() + + def startswith(self, char: str) -> bool: + self.skip() + return self.text.startswith(char, self.at) + + +def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None: + """The JSON value at `at` and where it ends, or None where none reads (yet).""" + try: + return decoder.raw_decode(text, at) + # silent-ok: not a failure here — the caller reads on, or refuses the file once it has all. + except json.JSONDecodeError: + return None + + +def _json_list(body: bytes, start: int) -> Iterator[object]: + """The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never + built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not + a million dicts at once.""" decoder = json.JSONDecoder() - at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw - if text.startswith("]", at): - at += 1 + window = _Window(body, start + 1) # past the "[" the caller saw + if window.startswith("]"): + window.at += 1 else: while True: - entry, at = decoder.raw_decode(text, at) + window.skip() + read = _decode(decoder, window.text, window.at) + # An entry cut by the window's edge fails, or (a number) reads short: read on and try + # again, with a window twice what is left, so one entry larger than any window is read + # in a few tries, not one a window. + if read is None or (read[1] == len(window.text) and not window.done): + if window.done: + raise ValueError("not a JSON list") + window.more(2 * (len(window.text) - window.at)) + continue + entry, window.at = read yield entry - at = _skip(text, at) - if text.startswith(",", at): - at = _skip(text, at + 1) - elif text.startswith("]", at): - at += 1 + if window.startswith(","): + window.at += 1 + elif window.startswith("]"): + window.at += 1 break else: raise ValueError("not a JSON list") - if _skip(text, at) != len(text): + window.skip() + if window.at != len(window.text): raise ValueError("text after the JSON list") +# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten +# times that is not one — and, unlike the history, it is decoded whole. +CSV_MAX_BYTES = 2 * 1024 * 1024 +_BOM = codecs.BOM_UTF8 + + def parse(body: bytes) -> Parsed: """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file with nothing to import in it.""" - try: - text = body.decode("utf-8-sig") - except UnicodeDecodeError: - raise TakeoutError("not_takeout") from None - stripped = text.lstrip() - if stripped.startswith(("{", "<")): + start = len(_BOM) if body.startswith(_BOM) else 0 + while start < len(body) and body[start] in b" \t\n\r\f\v": + start += 1 + first = body[start : start + 1] + if first in (b"{", b"<"): # Another JSON file of the export, or the history's HTML variant. raise TakeoutError("not_takeout") + if first != b"[" and len(body) - start > CSV_MAX_BYTES: + raise TakeoutError("too_many") try: + # A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file. parsed = ( - _history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped) + _history(_json_list(body, start)) + if first == b"[" + else _watch_later(body[start:].decode("utf-8")) ) # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit. @@ -200,10 +272,10 @@ def parse(body: bytes) -> Parsed: return parsed -# One import per user at a time: a second file uploaded while the first one's run is under way -# starts a run of its own (the first one's id has left the scheduler), and the two would settle the -# same rows. The second waits, then takes what the first left. One API process runs the scheduler, -# so a lock in this module holds it (as the channel backfill's does). +# One import per user at a time: a second file uploaded while the first one's run is under way runs +# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a +# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a +# lock in this module holds it (as the channel backfill's does). _user_locks: dict[int, threading.Lock] = {} _user_locks_guard = threading.Lock() @@ -266,11 +338,14 @@ def _put_in_watch_later(db: Session, user: User) -> dict: if ( fetcher is not None and not quota.backfill_reserve_reached() - and _lookups.allow(str(user.id), cost=len(ask)) + and _lookups.room(str(user.id)) >= len(ask) ): with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP): with YouTubeClient(db, fetcher) as yt: ensure_videos(db, yt, ask) + # Counted once done: a lookup that failed spent none of the share (one run per user at a + # time, so the room asked above is still there). + _lookups.allow(str(user.id), cost=len(ask)) asked = set(ask) # Only the rows read above: one a second upload staged meanwhile was never asked about. return service.apply_takeout_watch_later(db, user, waiting, asked) @@ -301,23 +376,36 @@ def apply_waiting(db: Session) -> dict: search's), and the Watch later videos a run could not put in yet. Run after each run of the `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when nothing was waiting that could move.""" - marked = added = 0 + marked = added = failed = 0 ready = set(queries.takeout_users_ready(db)) later = set(queries.takeout_users_watch_later_waiting(db)) for user_id in sorted(ready | later): user = db.get(User, user_id) if user is None: continue - with _one_import(user_id): - if user_id in ready: - marked += service.apply_takeout_watched(db, user)["watched"] - db.commit() - if user_id in later: - added += _put_in_watch_later(db, user).get("watch_later_added", 0) - db.commit() + # One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that + # runs this nor keeps the users after them waiting. What it did not settle waits. + try: + with _one_import(user_id): + if user_id in ready: + marked += service.apply_takeout_watched(db, user)["watched"] + db.commit() + if user_id in later: + added += _put_in_watch_later(db, user).get("watch_later_added", 0) + db.commit() + except Exception: + db.rollback() + failed += 1 + log.exception( + "Takeout settling failed for user %s", + user_id, + extra=logs.event("takeout.settle.failed", user_id=user_id), + ) out = {} if ready: out["takeout_watched"] = marked if added: out["takeout_watch_later_added"] = added + if failed: + out["takeout_failed"] = failed return out diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index db16714e..491a104b 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -309,38 +309,47 @@ def get_video_detail( # --- Google Takeout ------------------------------------------------------------------------------ -# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these -# bound what one account can make the process hold. The section uploads its files one after another. +# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are +# held whole while it is read (its text a window at a time), so these bound what the accounts can +# make the process hold. The section uploads its files one after another. _takeout_limiter = RateLimiter(max_events=10, window_seconds=3600) _takeout_uploading: set[int] = set() _takeout_uploading_guard = threading.Lock() +_TAKEOUT_AT_ONCE = 2 async def _takeout_body( request: Request, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> AsyncIterator[bytes]: # The sign-in check has run, and it was a SELECT: hand the connection back before a file of - # megabytes arrives, so a slow upload does not hold one `idle in transaction`. + # megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is + # taken first: the release expires the user, and reading it after would reload it, taking a + # connection straight back out. + uid = user.id release_read_connection(db) with _takeout_uploading_guard: - busy = user.id in _takeout_uploading - if not busy: - _takeout_uploading.add(user.id) - if busy or not _takeout_limiter.allow(str(user.id)): - if not busy: + busy = uid in _takeout_uploading + full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE + taken = not busy and not full + if taken: + _takeout_uploading.add(uid) + if not taken or not _takeout_limiter.allow(str(uid)): + if taken: with _takeout_uploading_guard: - _takeout_uploading.discard(user.id) + _takeout_uploading.discard(uid) # Read to the end before answering: refused mid-upload, the browser reports a network # failure instead of the refusal (capped_stream_to says why). await drain_body(request) if busy: raise refused(429, "takeout_busy") + if full: + raise refused(429, "takeout_server_busy") raise refused(429, "takeout_cooldown") try: yield await capped_body(request, takeout.MAX_BYTES) finally: with _takeout_uploading_guard: - _takeout_uploading.discard(user.id) + _takeout_uploading.discard(uid) @router.post("/takeout") diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 39dd3797..e23eff0b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -637,6 +637,15 @@ def ad_hoc_target_title(db, job: str, target: str) -> str | None: # run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too. _queued: set[str] = set() _queued_guard = threading.Lock() +# What the scheduler is told to run is only the key: the request itself waits here, the newest one +# per key, until a fire of that key takes it. A request made while its key's run is under way is so +# kept for the run after it, which that run asks for as it ends (`_running`): the scheduler would +# drop a second fire of an id whose run has not returned (one instance per job id). +_pending: dict[str, tuple[Callable[[], dict], int, bool]] = {} +_running: set[str] = set() +# Two at once, for the moment between a run letting go of its key and its thread returning: a fire +# then is what takes the next request, and is not dropped. +_AD_HOC_INSTANCES = 2 def ad_hoc_queued(job: str, target: str) -> bool: @@ -650,37 +659,77 @@ def run_ad_hoc( ) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so - a second request before the first has even started is that same run. `notify` posts its outcome - to the user's inbox, for work whose page does not watch it run.""" + a second request before the first has even started is that same run, and one made while it runs + is the run after it. `notify` posts its outcome to the user's inbox, for work whose page does not + watch it run.""" if job not in AD_HOC_JOBS: raise KeyError(job) # an unnamed job would show on the page as its bare id - sched = _scheduler - if sched is None: + if _scheduler is None: # Only between shutdown and exit: start_scheduler runs at every API startup. raise RuntimeError("the scheduler is not running") key = f"{job}@{target}" with _queued_guard: + _pending[key] = (fn, actor_id, notify) _queued.add(key) + running = key in _running + if running: + log.info("ad-hoc %s for %s runs again once the run under way ends", job, target) + return + _fire(job, target) + + +def _fire(job: str, target: str) -> None: + """Add the one-shot that takes `job@target`'s pending request. Raises (the request dropped) only + when the scheduler cannot take it.""" + key = f"{job}@{target}" + sched = _scheduler try: + if sched is None: + raise RuntimeError("the scheduler is not running") sched.add_job( - _ad_hoc, + _ad_hoc_fired, "date", - args=(job, fn, actor_id, target, notify), + args=(job, target), id=key, misfire_grace_time=None, + max_instances=_AD_HOC_INSTANCES, ) - # silent-ok: not a failure — the run for this target is already queued, and it does this work. + # silent-ok: not a failure — a fire of this key is already queued, and it takes this request. except ConflictingIdError: log.info("ad-hoc %s for %s already queued", job, target) except Exception: with _queued_guard: + _pending.pop(key, None) _queued.discard(key) raise +def _ad_hoc_fired(job: str, target: str) -> None: + key = f"{job}@{target}" + with _queued_guard: + request = _pending.pop(key, None) + if request is None: + return # an earlier fire took it + _running.add(key) + fn, actor_id, notify = request + try: + _ad_hoc(job, fn, actor_id, target, notify) + finally: + with _queued_guard: + _running.discard(key) + again = key in _pending + if again: + try: + _fire(job, target) + except Exception: + # The request made meanwhile is lost with the scheduler: shut down, nothing runs. + log.exception("ad-hoc %s for %s not run again", job, target) + + def _ad_hoc( job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False ) -> None: + key = f"{job}@{target}" try: run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) except Exception: @@ -688,9 +737,11 @@ def _ad_hoc( log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return finally: - # Recorded (or never will be): the run row now speaks for it. + # Recorded (or never will be): the run row now speaks for it — unless another request came + # meanwhile, which is still queued. with _queued_guard: - _queued.discard(f"{job}@{target}") + if key not in _pending: + _queued.discard(key) _opened(run_id, job) try: _ad_hoc_work(job, fn, run_id, actor_id if notify else None) @@ -765,4 +816,6 @@ def shutdown_scheduler() -> None: _scheduler.shutdown(wait=False) _scheduler = None with _queued_guard: - _queued.clear() # what it held will not run + # What it held will not run. + _queued.clear() + _pending.clear() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 01b9c1ce..66cb9a8b 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -327,3 +327,15 @@ def _reset_log_erasures(): from app import logs logs._erased.clear() + + +@pytest.fixture(autouse=True) +def _reset_ad_hoc_queue(): + """The scheduler's ad-hoc queue is module-level too: a request a test left pending would read as + a later test's user's import still queued (ids restart from 1).""" + from app import scheduler + + with scheduler._queued_guard: + scheduler._queued.clear() + scheduler._pending.clear() + scheduler._running.clear() diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index d3fa418e..cab4b7bf 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -6,6 +6,7 @@ through (at the top of _job): holding _activity_lock across the check-and-set me caller can hold a job's "running" claim at a time, and it's claimable again once released.""" import logging +import threading import time from datetime import datetime, timedelta, timezone @@ -426,6 +427,40 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next +def test_a_request_made_while_its_run_is_under_way_is_the_run_after_it( + db, make_user, live_scheduler +): + """The scheduler drops a second fire of an id whose run has not returned: the second Takeout file + of a pair, uploaded while the first one's import ran, was never imported, and its key stayed + queued until a restart (the page spun on "Processing…" for good).""" + user = make_user("again@example.com") + first_runs, release = threading.Event(), threading.Event() + calls = [] + + def first(): + first_runs.set() + assert release.wait(10) + calls.append("first") + return {"n": 1} + + def second(): + calls.append("second") + return {"n": 2} + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, first, actor_id=user.id, target="UC_again") + assert first_runs.wait(10) + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, second, actor_id=user.id, target="UC_again") + assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + release.set() + + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2: + time.sleep(0.05) + assert calls == ["first", "second"] + assert sorted(r.result["n"] for r in _finished(db, "channel_backfill")) == [1, 2] + assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + + def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): """The scheduler's pool threads are reused: an actor a manual run left set made the next scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py index 823b5d3e..d63f3475 100644 --- a/backend/tests/test_takeout.py +++ b/backend/tests/test_takeout.py @@ -7,12 +7,15 @@ import tracemalloc from datetime import datetime, timedelta, timezone import pytest +from fastapi import Depends from sqlalchemy import select, text from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session from app import jobruns, quota, scheduler -from app.db import SessionLocal -from app.catalog.models import Channel, Video +from app.auth import current_user +from app.db import SessionLocal, get_db +from app.catalog.models import Channel, Subscription, Video from app.models import JobRun, Notification, User from app.runstatus import RunStatus, RunTrigger from app.reader import queries, service, takeout @@ -145,6 +148,42 @@ def test_the_history_is_read_an_entry_at_a_time_not_as_one_tree(): assert peak < 5 * len(body), f"parsing held {peak / len(body):.1f} times the file" +def test_a_history_with_an_emoji_costs_less_than_its_own_size_to_read(monkeypatch): + """Decoded whole, one emoji makes the text four bytes a character: a 32 MiB history held some + 128 MiB of it on top of its bytes. Read a window at a time, it holds a window (a few times its + bytes as text) whatever the file's size — a small one here, so a 9 MB file can show it.""" + monkeypatch.setattr(takeout, "_WINDOW", 64 * 1024) + view = _view("aaaaaaaaaaa", title="Watched " + "😀" * 40) + body = b"[" + b",".join([json.dumps(view, ensure_ascii=False).encode()] * 20_000) + b"]" + tracemalloc.start() + try: + parsed = takeout.parse(body) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert parsed.entries == 20_000 + assert peak < len(body) / 2, f"parsing held {peak / len(body):.1f} times the file" + + +@pytest.mark.parametrize("split", [1, 7, 100, 4096]) +def test_the_history_reads_the_same_whatever_its_window(monkeypatch, split): + """An entry, a character or a number cut by the window's edge is read whole.""" + monkeypatch.setattr(takeout, "_WINDOW", split) + views = [_view(f"{c * 11}", title="ő😀" * 3) for c in "abcdef"] + body = " [\n" + ",\n".join(json.dumps(v, ensure_ascii=False) for v in views) + "\n]\n" + parsed = takeout.parse(body.encode()) + assert (parsed.entries, sorted(parsed.videos)) == (6, [c * 11 for c in "abcdef"]) + numbers = takeout._json_list(b"[123456789, 2]", 0) + assert list(numbers) == [123456789, 2] + + +def test_a_watch_later_file_far_larger_than_any_list_is_refused_unread(): + body = b"Video ID,Time\n" + b"x" * (takeout.CSV_MAX_BYTES) + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == "too_many" + + def test_the_id_lists_are_asked_in_chunks(db, make_user, monkeypatch): user = make_user("chunks@example.com") ids = [f"chunk{n:06d}" for n in range(5)] @@ -253,6 +292,33 @@ def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_o assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40) +@pytest.mark.parametrize("undo", ["unmark", "clear_state"]) +def test_a_video_the_user_un_marked_here_is_not_marked_again_by_an_older_view(db, make_user, undo): + """Un-marking (or resetting) a video leaves no state row, and the import inserted a watched one + over the user's own choice. A view on YouTube after the un-mark is a new view, and counts.""" + user = make_user(f"{undo}@example.com") + for vid in ("undone00000", "again000000"): + _video(db, vid) + db.commit() + for vid in ("undone00000", "again000000"): + service.mark(db, user, vid, "watched") + getattr(service, undo)(db, user, vid) + db.commit() + assert _states(db, user) == {} + later = datetime.now(timezone.utc) + timedelta(minutes=5) + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"undone00000": (T0, None), "again000000": (later, None)}, + ) + db.commit() + + assert takeout.run_import(user.id)["watched"] == 1 + assert set(_states(db, user)) == {"again000000"} + assert _rows(db, user)["undone00000"].outcome == "kept" + + def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user): user = make_user("again@example.com") _video(db, "plain000000") @@ -305,6 +371,40 @@ def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_hol assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl] +@pytest.mark.parametrize("n", [3, 40]) +def test_a_watch_later_list_goes_in_in_statements_that_do_not_grow_with_it(db, make_user, n): + """A video at a time, a full list of 5000 was some 20,000 statements under the user's lock.""" + from sqlalchemy import event + + user = make_user(f"many{n}@example.com") + ids = [f"wl{i:09d}" for i in range(n)] + for vid in ids: + _video(db, vid) + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + {v: (T0 + timedelta(minutes=i), None) for i, v in enumerate(ids)}, + ) + db.commit() + rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + service.watch_later(db, user) # the list made beforehand: its own insert is not this count's + db.flush() + statements = [] + engine = db.get_bind() + listener = lambda *a: statements.append(a[2]) # noqa: E731 — the event's own signature + event.listen(engine, "before_cursor_execute", listener) + try: + assert service.apply_takeout_watch_later(db, user, rows, set())["watch_later_added"] == n + finally: + event.remove(engine, "before_cursor_execute", listener) + db.commit() + assert len(statements) <= 9, statements + held = [i.video_id for i in queries.items(db, service.watch_later(db, user).id)] + assert held == ids + + def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( db, make_user, monkeypatch ): @@ -407,6 +507,38 @@ def test_the_shorts_job_puts_in_the_watch_later_videos_a_run_could_not(db, make_ assert takeout.apply_waiting(db) == {} +def test_one_users_failure_in_the_shorts_job_leaves_the_others_and_the_probe_alone( + db, make_user, monkeypatch +): + """A Watch later lookup that raised (a revoked token, a refused quota) escaped the Shorts job: + its run was recorded as failed with its own counts lost, and every user after this one waited.""" + first, second = make_user("first@example.com"), make_user("second@example.com") + _video(db, "ready000000") + db.commit() + service.stage_takeout(db, first, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)}) + service.stage_takeout(db, second, TakeoutKind.WATCHED, {"ready000000": (T0, None)}) + db.commit() + + def refused(session, yt, ids): + raise RuntimeError("invalid_grant") + + monkeypatch.setattr(takeout, "ensure_videos", refused) + monkeypatch.setattr(takeout, "get_service_user", lambda session: first) + monkeypatch.setattr(scheduler, "run_shorts", lambda session: {"probed": 4}) + + scheduler._shorts_job() + + db.expire_all() + run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one() + assert run.status == "ok" + assert run.result == {"probed": 4, "takeout_watched": 1, "takeout_failed": 1} + assert _states(db, second)["ready000000"].status == "watched" + assert _rows(db, first)["WL:unknown0000"].outcome is None + # A lookup that failed spent none of the user's daily share. + assert not takeout._lookups.exhausted(str(first.id)) + assert takeout._lookups.room(str(first.id)) == 2 * takeout.WATCH_LATER_MAX + + def test_a_watch_later_row_staged_while_youtube_is_asked_is_left_for_the_next_run( db, make_user, monkeypatch ): @@ -469,6 +601,9 @@ def test_staging_and_settling_take_the_users_one_lock(db, make_user): "watch later": lambda s, u: service.apply_takeout_watch_later( s, u, [(1, "aaaaaaaaaaa")], set() ), + # Between the import's read of the un-marks and its marking, one would be overwritten. + "unmark": lambda s, u: service.unmark(s, u, "aaaaaaaaaaa"), + "reset": lambda s, u: service.clear_state(s, u, "aaaaaaaaaaa"), } with SessionLocal() as holder: service.lock_takeout(holder, holder.get(User, user.id)) @@ -570,6 +705,39 @@ def test_a_signed_in_upload_has_its_body_read(db, make_user, as_user, queued): assert len(pulled) == 4 +def test_the_upload_arrives_with_no_connection_held( + db, make_user, as_user, override_dep, queued, monkeypatch +): + """The read connection is given back before the body: reading the user's id after that reloaded + the user and took one straight back out, `idle in transaction` for the whole upload.""" + user = make_user("idle@example.com") + client = as_user(user) + + def signed_in(session: Session = Depends(get_db)) -> User: + # As the real sign-in check: a SELECT on the request's session, which the release ends — + # and expires what it loaded. (`as_user`'s own resolver reads nothing.) + session.expire_all() + return session.get(User, user.id) + + override_dep(current_user, signed_in) + sessions, held = [], [] + release, capped = feed_routes.release_read_connection, feed_routes.capped_body + + def releasing(session): + sessions.append(session) + release(session) + + async def reading(request, limit): + held.extend(s.in_transaction() for s in sessions) + return await capped(request, limit) + + monkeypatch.setattr(feed_routes, "release_read_connection", releasing) + monkeypatch.setattr(feed_routes, "capped_body", reading) + resp = client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 200, resp.text + assert held == [False] + + def test_uploads_past_the_hourly_limit_are_refused_after_their_body_arrived( db, make_user, as_user, queued ): @@ -600,6 +768,20 @@ def test_a_second_upload_while_one_of_the_users_is_still_arriving_is_refused( assert feed_routes._takeout_uploading == {user.id} +def test_a_third_upload_across_users_while_two_arrive_is_refused_after_its_body( + db, make_user, as_user, queued, monkeypatch +): + """Each upload's bytes are held whole while it arrives: with no bound across users, a few at + once could take the process past its memory.""" + monkeypatch.setattr(feed_routes, "_takeout_uploading", {9001, 9002}) + body, pulled = _counting_body() + resp = as_user(make_user("third@example.com")).post("/api/takeout", content=body) + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "takeout_server_busy" + assert len(pulled) == 4 + assert queued == [] and feed_routes._takeout_uploading == {9001, 9002} + + def test_an_upload_that_would_keep_too_many_videos_is_refused( db, make_user, as_user, queued, monkeypatch ): @@ -679,12 +861,49 @@ def test_a_queued_import_is_told_apart_from_the_last_one_until_its_run_is_record assert scheduler.ad_hoc_queued("takeout_import", target) assert as_user(user).get("/api/takeout").json()["queued"] is True - scheduler._ad_hoc("takeout_import", dict, user.id, target) + scheduler._ad_hoc_fired("takeout_import", target) assert not scheduler.ad_hoc_queued("takeout_import", target) assert as_user(user).get("/api/takeout").json()["queued"] is False assert added == [f"takeout_import@{target}"] +def test_an_upload_while_its_import_starts_stays_queued_for_the_run_after( + db, make_user, monkeypatch +): + """A request landing while the run before it writes its row is the run after that one: still + queued, so the page does not take the run starting now for it.""" + user = make_user("starting@example.com") + added, ran = [], [] + + class Held: + def add_job(self, *args, **kwargs): + added.append(kwargs["id"]) + + monkeypatch.setattr(scheduler, "_scheduler", Held()) + target = takeout.run_target(user.id) + start = jobruns.start + + def starting(*args, **kwargs): + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(2) or {}, actor_id=user.id, target=target + ) + return start(*args, **kwargs) + + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(1) or {}, actor_id=user.id, target=target + ) + monkeypatch.setattr(scheduler.jobruns, "start", starting) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1] + assert scheduler.ad_hoc_queued("takeout_import", target) + assert added == [f"takeout_import@{target}"] * 2 # the run after it, asked for as it ended + + monkeypatch.setattr(scheduler.jobruns, "start", start) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1, 2] + assert not scheduler.ad_hoc_queued("takeout_import", target) + + # --- the videos that wait ---------------------------------------------------------------------- @@ -711,6 +930,39 @@ def test_waiting_videos_are_marked_once_the_shorts_probe_has_cleared_them(db, ma assert takeout.apply_waiting(db) == {} +def test_a_discovery_video_is_not_marked_until_the_catalog_keeps_it(db, make_user): + """A mark is a VideoState, which the cleanup jobs read as a reader keeping the video: on a video + only someone's search or explore brought in, it kept that video (and its channel) for good.""" + user = make_user("discover@example.com") + for vid, flag in (("search00000", "via_search"), ("explore0000", "via_explore")): + _video(db, vid) + db.flush() + setattr(db.get(Video, vid), flag, True) + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {v: (T0, None) for v in ("search00000", "explore0000", "plain000000")}, + ) + db.commit() + + assert takeout.apply_waiting(db) == {"takeout_watched": 1} + assert set(_states(db, user)) == {"plain000000"} + assert {v for v, r in _rows(db, user).items() if r.outcome is None} == { + "search00000", + "explore0000", + } + + # Subscribed to: the catalog keeps the channel's videos of its own, the search one included. + db.add(Subscription(user_id=user.id, channel_id=CHANNEL)) + db.get(Video, "explore0000").via_explore = False # as subscribing does (catalog.service) + db.commit() + assert takeout.apply_waiting(db) == {"takeout_watched": 2} + assert set(_states(db, user)) == {"plain000000", "search00000", "explore0000"} + + def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_user): user = make_user("job-shorts@example.com") _video(db, "ready000000") # probed already: the job's probe has nothing left to do diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index 14f11f33..c189977e 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -45,6 +45,7 @@ UNIT_MODELS = { "SavedView", "ComfortPin", "TakeoutItem", + "VideoUnmark", "Tag", "ChannelTag", }, diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index c9066502..c5d7d4dd 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -254,6 +254,7 @@ "takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.", "takeout_no_videos": "This file has no videos to import", "takeout_busy": "Another of your files is still being uploaded — wait for it to finish", + "takeout_server_busy": "Other imports are being received right now — try again in a minute", "takeout_cooldown": "You have uploaded many files in the last hour — try again later", "takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds", "takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can" diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index 1eafde92..f5b559c9 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -188,7 +188,8 @@ "watch_later_missing": "no longer on YouTube", "watch_later_waiting": "Watch later waiting", "takeout_watched": "marked watched from Takeout", - "takeout_watch_later_added": "put into Watch later from Takeout" + "takeout_watch_later_added": "put into Watch later from Takeout", + "takeout_failed": "Takeout users left for the next run after an error" }, "skip": { "disabled": "disabled", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index 6063a8e4..39b735b4 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -254,6 +254,7 @@ "takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.", "takeout_no_videos": "Ebben a fájlban nincs importálható videó", "takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik", + "takeout_server_busy": "Épp más importok érkeznek — próbáld újra egy perc múlva", "takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később", "takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van", "takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet" diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index 2506ca42..7fa014aa 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -188,7 +188,8 @@ "watch_later_missing": "már nincs a YouTube-on", "watch_later_waiting": "a „Megnézem később” listára vár", "takeout_watched": "Takeoutból megnézettre jelölve", - "takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve" + "takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve", + "takeout_failed": "hiba miatt a következő futásra maradt Takeout-felhasználók" }, "skip": { "disabled": "kikapcsolva", From 420baff463968ba708ebbf1ca283c389c2fe43ef Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 22:39:20 +0200 Subject: [PATCH 8/8] chore(release): 0.110.0 --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- frontend/src/content/release-notes/en.json | 11 +++++++++++ frontend/src/content/release-notes/hu.json | 11 +++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b592f7..a1cc11d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.110.0 — 2026-10-23 + +### New + +- YouTube history: upload your Google Takeout export in Settings › Account. What you watched on YouTube is marked watched here (never a Short, never over a state you set), and your YouTube Watch later list joins yours. +- A video the catalog does not know yet is remembered and marked once it arrives. The import's outcome reaches your inbox, and admins see each run on the Scheduler page. + +### Fixed + +- Scheduler: a channel's full-history backfill asked for again while one is running now runs after it, instead of being silently dropped. + ## 0.109.2 — 2026-10-23 ### Fixed diff --git a/VERSION b/VERSION index 1b691737..068bced3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.109.2 +0.110.0 diff --git a/frontend/src/content/release-notes/en.json b/frontend/src/content/release-notes/en.json index 73ceeb50..2a1304d6 100644 --- a/frontend/src/content/release-notes/en.json +++ b/frontend/src/content/release-notes/en.json @@ -1,4 +1,15 @@ [ + { + "version": "0.110.0", + "date": "2026-10-23", + "new": [ + "YouTube history: upload your Google Takeout export in Settings › Account. What you watched on YouTube is marked watched here (never a Short, never over a state you set), and your YouTube Watch later list joins yours.", + "A video the catalog does not know yet is remembered and marked once it arrives. The import's outcome reaches your inbox, and admins see each run on the Scheduler page." + ], + "fixed": [ + "Scheduler: a channel's full-history backfill asked for again while one is running now runs after it, instead of being silently dropped." + ] + }, { "version": "0.109.2", "date": "2026-10-23", diff --git a/frontend/src/content/release-notes/hu.json b/frontend/src/content/release-notes/hu.json index 3dde8916..8166f125 100644 --- a/frontend/src/content/release-notes/hu.json +++ b/frontend/src/content/release-notes/hu.json @@ -1,4 +1,15 @@ [ + { + "version": "0.110.0", + "date": "2026-10-23", + "new": [ + "YouTube-előzmények: a Google Takeout exportod a Beállítások › Fiók oldalon tölthető fel. Amit a YouTube-on megnéztél, itt megnézettnek jelölődik (Short soha, és amit itt beállítottál, azt nem írja felül), a „Megnézem később” listád pedig az ittenihez adódik.", + "Egy videót, amit a katalógus még nem ismer, megjegyez, és megjelöli, amint bekerül. Az import eredménye a postafiókodba érkezik, az adminok pedig minden futását látják az Ütemező oldalon." + ], + "fixed": [ + "Ütemező: ha egy csatorna teljes előzményének letöltését futás közben újra kérik, az most utána lefut, nem vész el csendben." + ] + }, { "version": "0.109.2", "date": "2026-10-23",