diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b592f7..a1cc11d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.110.0 — 2026-10-23 + +### New + +- YouTube history: upload your Google Takeout export in Settings › Account. What you watched on YouTube is marked watched here (never a Short, never over a state you set), and your YouTube Watch later list joins yours. +- A video the catalog does not know yet is remembered and marked once it arrives. The import's outcome reaches your inbox, and admins see each run on the Scheduler page. + +### Fixed + +- Scheduler: a channel's full-history backfill asked for again while one is running now runs after it, instead of being silently dropped. + ## 0.109.2 — 2026-10-23 ### Fixed diff --git a/VERSION b/VERSION index 1b691737..068bced3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.109.2 +0.110.0 diff --git a/backend/alembic/versions/0113_takeout_items.py b/backend/alembic/versions/0113_takeout_items.py new file mode 100644 index 00000000..8a901789 --- /dev/null +++ b/backend/alembic/versions/0113_takeout_items.py @@ -0,0 +1,51 @@ +"""A user's Google Takeout export: what they watched on YouTube, and their Watch later list + +Revision ID: 0113_takeout_items +Revises: 0112_one_watch_later +Create Date: 2026-10-23 + +One row per video and kind. A video the catalog does not know yet waits with no outcome, so the id +is not a foreign key; the partial index is what a newly known video looks its waiting rows up by. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0113_takeout_items" +down_revision: str | None = "0112_one_watch_later" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "takeout_items", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("kind", sa.String(12), nullable=False), + sa.Column("video_id", sa.String(16), nullable=False), + sa.Column("channel_id", sa.String(32), nullable=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.Column("outcome", sa.String(12), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + sa.CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + ) + op.create_index( + "ix_takeout_items_waiting", + "takeout_items", + ["video_id"], + postgresql_where=sa.text("outcome IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_takeout_items_waiting", table_name="takeout_items") + op.drop_table("takeout_items") diff --git a/backend/alembic/versions/0114_video_unmarks.py b/backend/alembic/versions/0114_video_unmarks.py new file mode 100644 index 00000000..64e15403 --- /dev/null +++ b/backend/alembic/versions/0114_video_unmarks.py @@ -0,0 +1,44 @@ +"""When a user last took a video back to "new" + +Revision ID: 0114_video_unmarks +Revises: 0113_takeout_items +Create Date: 2026-10-23 + +Un-marking or resetting a video leaves no `video_states` row, so this is what tells an import of the +user's YouTube history a video they un-marked here from one never touched. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0114_video_unmarks" +down_revision: str | None = "0113_takeout_items" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "video_unmarks", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "video_id", + sa.String(16), + sa.ForeignKey("videos.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint("user_id", "video_id", name="uq_video_unmark"), + ) + op.create_index("ix_video_unmarks_video_id", "video_unmarks", ["video_id"]) + + +def downgrade() -> None: + op.drop_index("ix_video_unmarks_video_id", table_name="video_unmarks") + op.drop_table("video_unmarks") diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index c0ecab4c..43e88f03 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -72,6 +72,13 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why # `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the # default, so listing it could only widen what it accepts. +# A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390 +# bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an +# account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The +# file's text is decoded a window at a time and read an entry at a time (`takeout._json_list`), so +# it costs its bytes and a window, not a tree of it; the route takes two at once across every user. +TAKEOUT_MAX_BYTES = 32 * 1024 * 1024 + ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # Ingesting a local file. The body IS the file now — no multipart envelope to allow for — # so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s @@ -95,6 +102,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports. ("POST", "/api/files/upload"): settings.download_upload_max_bytes, ("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES, + ("POST", "/api/takeout"): TAKEOUT_MAX_BYTES, } # The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders diff --git a/backend/app/jobruns.py b/backend/app/jobruns.py index 7beb7572..5596e651 100644 --- a/backend/app/jobruns.py +++ b/backend/app/jobruns.py @@ -119,6 +119,16 @@ def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[Jo return out +def last_run(db: Session, job: str, target: str) -> JobRun | None: + """The newest run of an ad-hoc `job` on `target` (a user's own import), or None.""" + return db.execute( + select(JobRun) + .where(JobRun.job == job, JobRun.target == target) + .order_by(JobRun.started_at.desc(), JobRun.id.desc()) + .limit(1) + ).scalar_one_or_none() + + def gc(db: Session, retention_days: int) -> int: """Delete finished runs older than `retention_days` (0 = keep forever). A run still going is never reaped. The caller commits.""" diff --git a/backend/app/ratelimit.py b/backend/app/ratelimit.py index 6dcef515..313409e5 100644 --- a/backend/app/ratelimit.py +++ b/backend/app/ratelimit.py @@ -42,6 +42,14 @@ class RateLimiter: with self._lock: return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events + def room(self, key: str) -> float: + """How much cost ``key`` may still spend in the current window, WITHOUT counting anything — + for work counted only once it has succeeded: ask it first, ``allow`` what was done.""" + cutoff = time.monotonic() - self.window + with self._lock: + used = sum(c for t, c in self._hits.get(key, []) if t > cutoff) + return max(0.0, self.max_events - used) + def allow(self, key: str, cost: float = 1) -> bool: """Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit. diff --git a/backend/app/reader/models.py b/backend/app/reader/models.py index f55bd23a..7ff5d7b1 100644 --- a/backend/app/reader/models.py +++ b/backend/app/reader/models.py @@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th edge, or asks this unit's service.""" from datetime import datetime +from enum import StrEnum from sqlalchemy import ( Boolean, @@ -84,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin): progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) +class VideoUnmark(Base): + """When the user last took a video back to "new" here — un-marked it, or reset it. That leaves + no `VideoState` (its absence is "new"), so without this row a later import of what they watched + on YouTube could not tell a video they un-marked from one never touched, and marked it watched + again. Kept by user and video; the video's own deletion takes it.""" + + __tablename__ = "video_unmarks" + __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True) + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + class Playlist(Base, TimestampMixin, UpdatedAtMixin): """A per-user named, ordered collection of videos from the shared catalog. @@ -213,3 +229,51 @@ class ComfortPin(Base, TimestampMixin): # 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder # endpoint rewrites wholesale beats a fractional index nobody would ever exercise here. position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False) + + +class TakeoutKind(StrEnum): + """What a Google Takeout row says the user did with a video.""" + + WATCHED = "watched" # history/watch-history.json + WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv + + +class TakeoutOutcome(StrEnum): + """What the import did with a row; NULL = still waiting for its video to be known (and, for a + watched one, probed for Shorts).""" + + WATCHED = "watched" # the video is marked watched at the Takeout time + KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays + SHORT = "short" # a Short: swiping one past is not watching it + ADDED = "added" # added to Watch later + PRESENT = "present" # already in Watch later + MISSING = "missing" # YouTube no longer has the video + + +class TakeoutItem(Base, TimestampMixin): + """One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch + later list (with when it was added). The rows are both the import job's input and its memory: a + video the catalog does not know yet waits here, with no outcome, until it enters the catalog. + The id is not a foreign key for that reason. The channel is kept for a later "channels you + watch but do not follow" list.""" + + __tablename__ = "takeout_items" + __table_args__ = ( + UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + # The waiting rows, by video: what a newly known video looks up. + Index( + "ix_takeout_items_waiting", + "video_id", + postgresql_where=text("outcome IS NULL"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + kind: Mapped[str] = mapped_column(String(12)) + video_id: Mapped[str] = mapped_column(String(16)) + channel_id: Mapped[str | None] = mapped_column(String(32)) + # Watched: the last time in the export; Watch later: when it was added. + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + outcome: Mapped[str | None] = mapped_column(String(12)) diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 453b7e88..b06eb402 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -1,5 +1,5 @@ -"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and -for the other units that ask about them. +"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP +layer and for the other units that ask about them. Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and @@ -7,6 +7,9 @@ length, whether a video exists); another unit asks the questions at the end of t than joining Reader's tables itself. """ +from collections.abc import Iterable +from datetime import datetime + from sqlalchemy import ColumnElement, and_, func, or_, select from sqlalchemy.orm import InstrumentedAttribute, Session, aliased @@ -14,7 +17,17 @@ from app.collation import collated from app.feedquery import feed_columns, serialize from app.models import User from app.catalog.models import Channel, Subscription, Video -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + VideoState, + VideoUnmark, +) from app.userscope import owned_by, owned_row # --- catalog, read along the edge ------------------------------------------------------------ @@ -359,3 +372,164 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan owned_by(ChannelTag.user_id, user), ) ).scalar_one_or_none() + + +# --- Google Takeout -------------------------------------------------------------------------- + + +def _settled_here() -> ColumnElement[bool]: + """A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a + discovery video only a search or an explore brought in. A mark is a `VideoState`, which the + cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the + video, and its explored channel, for good. Such a row waits until the video is the catalog's own + (its channel subscribed to) — or is gone.""" + subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists() + return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed)) + + +def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]: + """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts: + (row id, video id, Takeout time, is Short).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short) + .join(Video, Video.id == TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.outcome.is_(None), + Video.shorts_probed, + _settled_here(), + ) + ).tuples() + ) + + +def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]: + """The user's rows of `kind` the import has not settled, oldest first: (row id, video id).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == kind, + TakeoutItem.outcome.is_(None), + ) + .order_by(TakeoutItem.at, TakeoutItem.id) + ).tuples() + ) + + +# Ids per IN list: a list is bound one parameter an id, and a statement takes at most 65,535. +_IN_CHUNK = 5000 + + +def known_video_ids(db: Session, ids: list[str]) -> set[str]: + """Which of `ids` the catalog holds.""" + known: set[str] = set() + for start in range(0, len(ids), _IN_CHUNK): + chunk = ids[start : start + _IN_CHUNK] + known.update(db.execute(select(Video.id).where(Video.id.in_(chunk))).scalars()) + return known + + +def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]: + """When the user last took each of `ids` back to "new" here, for those they ever did.""" + out: dict[str, datetime] = {} + for start in range(0, len(ids), _IN_CHUNK): + rows = db.execute( + select(VideoUnmark.video_id, VideoUnmark.at).where( + owned_by(VideoUnmark.user_id, user), + VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]), + ) + ).tuples() + out.update((vid, at) for vid, at in rows) + return out + + +def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]: + """Which of `ids` the playlist already holds.""" + held: set[str] = set() + for start in range(0, len(ids), _IN_CHUNK): + chunk = ids[start : start + _IN_CHUNK] + held.update( + db.execute( + select(PlaylistItem.video_id).where( + PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk) + ) + ).scalars() + ) + return held + + +def takeout_rows_besides( + db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str] +) -> int: + """How many Takeout rows the user keeps other than `video_ids` of `kind`: what an upload of + those videos would be added to.""" + total = ( + db.scalar(select(func.count(TakeoutItem.id)).where(owned_by(TakeoutItem.user_id, user))) + or 0 + ) + ids = list(video_ids) + for start in range(0, len(ids), _IN_CHUNK): + total -= ( + db.scalar( + select(func.count(TakeoutItem.id)).where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == kind, + TakeoutItem.video_id.in_(ids[start : start + _IN_CHUNK]), + ) + ) + or 0 + ) + return total + + +def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]: + """How many of the user's rows each kind and outcome holds (None = waiting).""" + rows = db.execute( + select(TakeoutItem.kind, TakeoutItem.outcome, func.count()) + .where(owned_by(TakeoutItem.user_id, user)) + .group_by(TakeoutItem.kind, TakeoutItem.outcome) + ).all() + return {(kind, outcome): n for kind, outcome, n in rows} + + +def takeout_summary(db: Session, user: User) -> dict[str, dict[str, int]]: + """The user's kept Takeout videos by kind and outcome, for their settings page: `{"watched": + {"watched": n, "short": n, "waiting": n, …}, "watch_later": {…}}` — the import's outcome words, + "waiting" for a video not known yet.""" + out: dict[str, dict[str, int]] = {kind.value: {} for kind in TakeoutKind} + for (kind, outcome), n in takeout_outcomes(db, user).items(): + out[kind][outcome or "waiting"] = n + return out + + +def takeout_users_ready(db: Session) -> list[int]: + """Every user with a waiting "watched" row whose video is now known and probed for Shorts: the + rows a later run of the Shorts probe made ready.""" + return list( + db.execute( + select(TakeoutItem.user_id) + .join(Video, Video.id == TakeoutItem.video_id) + .where( + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.outcome.is_(None), + Video.shorts_probed, + _settled_here(), + ) + .distinct() + ).scalars() + ) + + +def takeout_users_watch_later_waiting(db: Session) -> list[int]: + """Every user with a Watch later row a run could not put in yet.""" + return list( + db.execute( + select(TakeoutItem.user_id) + .where(TakeoutItem.kind == TakeoutKind.WATCH_LATER, TakeoutItem.outcome.is_(None)) + .distinct() + ).scalars() + ) diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index ca16ab49..08bbabc2 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -15,7 +15,18 @@ from sqlalchemy.orm import Session from app.models import User from app.reader import queries -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + TakeoutOutcome, + VideoState, + VideoUnmark, +) from app.sync.playlists import lock_playlists, recompute_dirty from app.userscope import owned_by @@ -196,6 +207,7 @@ def unmark(db: Session, user: User, video_id: str) -> None: the in-progress state, not where the user left off); otherwise the row goes. The player fires this beside a checkpoint that may delete the same row: the caller's commit then raises `StaleDataError`, which is that lost race and nothing worse.""" + lock_takeout(db, user) # an import settling meanwhile reads the un-mark below, or comes first row = queries.video_state(db, user, video_id) if row is None: return @@ -204,9 +216,21 @@ def unmark(db: Session, user: User, video_id: str) -> None: row.watched_at = None else: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() +def _remember_unmark(db: Session, user: User, video_id: str) -> None: + """Note that the user took the video back to "new" now: the state they set leaves no row, and + an import of their YouTube history must not mark it watched again (`apply_takeout_watched`).""" + now = datetime.now(timezone.utc) + db.execute( + pg_insert(VideoUnmark) + .values(user_id=user.id, video_id=video_id, at=now) + .on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now}) + ) + + def mark(db: Session, user: User, video_id: str, status: str) -> None: """ "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then @@ -231,9 +255,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None: def clear_state(db: Session, user: User, video_id: str) -> None: """The video back to pristine for this user: status, resume point and `watched_at` all go. Playlist membership lives elsewhere and stays.""" + lock_takeout(db, user) # as `unmark` row = queries.video_state(db, user, video_id) if row is not None: db.delete(row) + _remember_unmark(db, user, video_id) db.flush() @@ -495,3 +521,144 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No if row is not None: db.delete(row) db.flush() + + +# --- Google Takeout -------------------------------------------------------------------------- + +_TAKEOUT_BATCH = 1000 +# The advisory-lock namespace of one user's Takeout rows ("TO"), so a key cannot meet another lock's. +_TAKEOUT_LOCK_NS = 0x544F + + +def lock_takeout(db: Session, user: User) -> None: + """Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an + import's settling each lock many of the same rows, one statement after another and in no common + order, so without one lock over both the two can deadlock. An un-mark takes it too: the import + reads the un-marks before it marks, and one landing between the two would be overwritten.""" + db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id))) + + +def stage_takeout( + db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]] +) -> None: + """Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job. + A second upload of the same export, or a newer one, adds what is new: a video already kept keeps + the later time, and a row the import has settled keeps its outcome.""" + lock_takeout(db, user) + values = [ + {"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel} + for vid, (at, channel) in rows.items() + ] + for start in range(0, len(values), _TAKEOUT_BATCH): + stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH]) + db.execute( + stmt.on_conflict_do_update( + constraint="uq_takeout_item", + set_={ + "at": func.greatest(TakeoutItem.at, stmt.excluded.at), + "channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id), + }, + ) + ) + db.flush() + + +def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None: + for start in range(0, len(ids), _TAKEOUT_BATCH): + db.execute( + update(TakeoutItem) + .where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH])) + .values(outcome=outcome) + ) + + +def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: + """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed + for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one + past is not watching it), and a state the user set here wins: watched or hidden, anything + changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own + condition is that rule, so a state set while the import runs wins too.""" + lock_takeout(db, user) + short: list[int] = [] + todo: dict[str, tuple[int, datetime]] = {} + ready = queries.takeout_watched_ready(db, user) + unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready]) + kept: list[int] = [] + for row_id, video_id, at, is_short in ready: + if is_short: + short.append(row_id) + elif video_id in unmarked and unmarked[video_id] >= at: + # Un-marked here since they watched it there: theirs, as a state they set is. + kept.append(row_id) + else: + todo[video_id] = (row_id, at) + marked: set[str] = set() + items = list(todo.items()) + now = datetime.now(timezone.utc) + for start in range(0, len(items), _TAKEOUT_BATCH): + stmt = pg_insert(VideoState).values( + [ + {"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at} + for vid, (_, at) in items[start : start + _TAKEOUT_BATCH] + ] + ) + marked.update( + db.execute( + stmt.on_conflict_do_update( + constraint="uq_user_video", + set_={ + "status": "watched", + "watched_at": stmt.excluded.watched_at, + "updated_at": now, + }, + where=(VideoState.status == "new") + & (VideoState.updated_at < stmt.excluded.watched_at), + ).returning(VideoState.video_id) + ).scalars() + ) + kept += [row_id for vid, (row_id, _) in items if vid not in marked] + _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED) + _settle(db, kept, TakeoutOutcome.KEPT) + _settle(db, short, TakeoutOutcome.SHORT) + db.flush() + return {"watched": len(marked), "kept": len(kept), "shorts": len(short)} + + +def apply_takeout_watch_later( + db: Session, user: User, rows: list[tuple[int, str]], asked: set[str] +) -> dict[str, int]: + """Put the videos of the user's waiting Watch later `rows`, (row id, video id) oldest first, into + their list here, after what it already holds. `asked` is the videos YouTube was just asked for: one + of those the catalog still lacks is gone from YouTube; any other unknown one keeps waiting.""" + if not rows: + return {} + lock_takeout(db, user) + ids = [vid for _, vid in rows] + known = queries.known_video_ids(db, ids) + pl = watch_later(db, user) + # What add_item does a video at a time, for the whole list at once: one presence read and one + # insert, not four statements a video (a full list is thousands). Locked before the read, as there. + next_position = next_item_position(db, pl.id) + held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known]) + done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} + for row_id, vid in rows: + if vid not in known: + if vid in asked: + done[TakeoutOutcome.MISSING].append(row_id) + elif vid in held: + done[TakeoutOutcome.PRESENT].append(row_id) + else: + db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position)) + next_position += 1 + held.add(vid) + done[TakeoutOutcome.ADDED].append(row_id) + if done[TakeoutOutcome.ADDED]: + touch(pl) + for outcome, ids in done.items(): + _settle(db, ids, outcome) + db.flush() + return { + "watch_later_added": len(done[TakeoutOutcome.ADDED]), + "watch_later_present": len(done[TakeoutOutcome.PRESENT]), + "watch_later_missing": len(done[TakeoutOutcome.MISSING]), + } diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py new file mode 100644 index 00000000..35d5fc1d --- /dev/null +++ b/backend/app/reader/takeout.py @@ -0,0 +1,411 @@ +"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their +YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the +only way in. + +Two files of the export are read, each uploaded on its own: `history/watch-history.json` and +`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies +what it can; a video the catalog does not know yet waits for it. + +The format, measured on a real export (2026-10): the history is a JSON list, one entry per view, +newest first. A view of a video has a `titleUrl` with `watch?v=` (on www.youtube.com or +music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles` +entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google +Ads". The title is in the account's language, so nothing here reads it. A Short is a plain +`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a +header row (in the account's language) and `,` rows. +""" + +import codecs +import csv +import io +import json +import re +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime +from itertools import chain + +from sqlalchemy.orm import Session + +from app import logs, quota +from app.bodylimit import TAKEOUT_MAX_BYTES +from app.db import SessionLocal +from app.models import User +from app.ratelimit import RateLimiter +from app.reader import queries, service +from app.reader.models import TakeoutKind +from app.sync.runner import get_service_user +from app.sync.videos import ensure_videos +from app.youtube.client import YouTubeClient + +log = logs.get_logger(__name__) + +# The largest file the upload takes (its edge cap and the route's own are one number). +MAX_BYTES = TAKEOUT_MAX_BYTES +# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is +# looked up on the shared API budget, so a longer list is not one YouTube made. +WATCH_LATER_MAX = 5000 +# The most Takeout videos one user keeps: twenty times the real export measured here (~9,000 videos +# across both files), so it only stops uploads of made-up ones piling up for good. +ROWS_MAX = 200_000 + +_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") +_WATCH_URL = re.compile( + r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})" +) +_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})") + + +class TakeoutError(ValueError): + """The file is not one the import reads; `reason` is the client's word for why.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +@dataclass +class Parsed: + """One file's videos, `{video id: (time, channel id)}`, and what it left out, counted.""" + + kind: TakeoutKind + videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict) + entries: int = 0 + ads: int = 0 + other: int = 0 # not a view of a video (a post, a story) or a video with no link left + malformed: int = 0 + + +def _time(raw: object) -> datetime | None: + if not isinstance(raw, str): + return None + try: + at = datetime.fromisoformat(raw.replace("Z", "+00:00")) + # silent-ok: the row is counted as malformed, and the count is in the upload's answer. + except ValueError: + return None + return at if at.tzinfo is not None else None + + +def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None: + old = parsed.videos.get(vid) + if old is None or at > old[0]: + parsed.videos[vid] = (at, channel or (old[1] if old else None)) + + +def _history(data: Iterator[object]) -> Parsed: + parsed = Parsed(TakeoutKind.WATCHED) + for entry in data: + parsed.entries += 1 + if not isinstance(entry, dict): + parsed.malformed += 1 + continue + details = entry.get("details") + if isinstance(details, list) and any( + isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details + ): + parsed.ads += 1 + continue + url = entry.get("titleUrl") + match = _WATCH_URL.match(url) if isinstance(url, str) else None + if match is None: + parsed.other += 1 + continue + at = _time(entry.get("time")) + if at is None: + parsed.malformed += 1 + continue + channel = None + subtitles = entry.get("subtitles") + if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict): + found = _CHANNEL_URL.search(str(subtitles[0].get("url", ""))) + channel = found.group(1) if found else None + _keep(parsed, match.group(1), at, channel) + return parsed + + +def _watch_later(text: str) -> Parsed: + parsed = Parsed(TakeoutKind.WATCH_LATER) + rows = (row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)) + first = next(rows, None) + if first is None or len(first) < 2: + raise TakeoutError("not_takeout") + # The first row is the header, in the account's language, unless it is already a video. + for row in chain([first], rows) if _VIDEO_ID.match(first[0].strip()) else rows: + parsed.entries += 1 + vid = row[0].strip() + at = _time(row[1].strip()) if len(row) > 1 else None + if not _VIDEO_ID.match(vid) or at is None: + parsed.malformed += 1 + continue + _keep(parsed, vid, at, None) + if len(parsed.videos) > WATCH_LATER_MAX: + raise TakeoutError("too_many") + return parsed + + +def _skip(text: str, at: int) -> int: + """The first index at or after `at` that is not JSON whitespace.""" + while at < len(text) and text[at] in " \t\n\r": + at += 1 + return at + + +# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a +# character (one emoji in a title is enough), several times the file on top of its bytes. +_WINDOW = 1024 * 1024 + + +class _Window: + """The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on. + The bytes are the caller's, so only the window is this reader's memory.""" + + def __init__(self, body: bytes, start: int): + self._body, self._next = body, start + self._decoder = codecs.getincrementaldecoder("utf-8")() # strict: bad UTF-8 raises + self.text, self.at = "", 0 + + @property + def done(self) -> bool: + return self._next >= len(self._body) + + def more(self, at_least: int = 0) -> None: + """Decode the next window (at least `at_least` bytes), dropping what is read already.""" + self.text, self.at = self.text[self.at :], 0 + size = max(_WINDOW, at_least) + chunk = self._body[self._next : self._next + size] + self._next += len(chunk) + self.text += self._decoder.decode(chunk, final=self.done) + + def skip(self) -> None: + """Past the whitespace, reading on as far as it goes.""" + while True: + self.at = _skip(self.text, self.at) + if self.at < len(self.text) or self.done: + return + self.more() + + def startswith(self, char: str) -> bool: + self.skip() + return self.text.startswith(char, self.at) + + +def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None: + """The JSON value at `at` and where it ends, or None where none reads (yet).""" + try: + return decoder.raw_decode(text, at) + # silent-ok: not a failure here — the caller reads on, or refuses the file once it has all. + except json.JSONDecodeError: + return None + + +def _json_list(body: bytes, start: int) -> Iterator[object]: + """The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never + built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not + a million dicts at once.""" + decoder = json.JSONDecoder() + window = _Window(body, start + 1) # past the "[" the caller saw + if window.startswith("]"): + window.at += 1 + else: + while True: + window.skip() + read = _decode(decoder, window.text, window.at) + # An entry cut by the window's edge fails, or (a number) reads short: read on and try + # again, with a window twice what is left, so one entry larger than any window is read + # in a few tries, not one a window. + if read is None or (read[1] == len(window.text) and not window.done): + if window.done: + raise ValueError("not a JSON list") + window.more(2 * (len(window.text) - window.at)) + continue + entry, window.at = read + yield entry + if window.startswith(","): + window.at += 1 + elif window.startswith("]"): + window.at += 1 + break + else: + raise ValueError("not a JSON list") + window.skip() + if window.at != len(window.text): + raise ValueError("text after the JSON list") + + +# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten +# times that is not one — and, unlike the history, it is decoded whole. +CSV_MAX_BYTES = 2 * 1024 * 1024 +_BOM = codecs.BOM_UTF8 + + +def parse(body: bytes) -> Parsed: + """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told + apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file + with nothing to import in it.""" + start = len(_BOM) if body.startswith(_BOM) else 0 + while start < len(body) and body[start] in b" \t\n\r\f\v": + start += 1 + first = body[start : start + 1] + if first in (b"{", b"<"): + # Another JSON file of the export, or the history's HTML variant. + raise TakeoutError("not_takeout") + if first != b"[" and len(body) - start > CSV_MAX_BYTES: + raise TakeoutError("too_many") + try: + # A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file. + parsed = ( + _history(_json_list(body, start)) + if first == b"[" + else _watch_later(body[start:].decode("utf-8")) + ) + # A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too + # long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit. + except (ValueError, RecursionError, csv.Error) as exc: + if isinstance(exc, TakeoutError): + raise + raise TakeoutError("not_takeout") from None + if not parsed.videos: + raise TakeoutError("no_videos") + return parsed + + +# One import per user at a time: a second file uploaded while the first one's run is under way runs +# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a +# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a +# lock in this module holds it (as the channel backfill's does). +_user_locks: dict[int, threading.Lock] = {} +_user_locks_guard = threading.Lock() + + +@contextmanager +def _one_import(user_id: int) -> Iterator[None]: + with _user_locks_guard: + lock = _user_locks.setdefault(user_id, threading.Lock()) + with lock: + yield + + +def run_target(user_id: int) -> str: + """The import run's target, `job_runs.target`: one user's import.""" + return f"user:{user_id}" + + +def target_title(db: Session, target: str) -> str | None: + """Who an import run's target is, for the admin's run lists: the user's name, else their email.""" + _, _, raw = target.partition("user:") + user = db.get(User, int(raw)) if raw.isdigit() else None + return (user.display_name or user.email) if user is not None else None + + +class TooMany(Exception): + """The upload would take the user past `ROWS_MAX` kept videos.""" + + +def stage(db: Session, user: User, parsed: Parsed) -> None: + """Keep an uploaded file's videos for the import, unless that would take the user past + `ROWS_MAX` (`TooMany`). The caller commits.""" + service.lock_takeout(db, user) # the count and the insert are one user's, not two uploads' each + if ( + queries.takeout_rows_besides(db, user, parsed.kind, parsed.videos) + len(parsed.videos) + > ROWS_MAX + ): + raise TooMany + service.stage_takeout(db, user, parsed.kind, parsed.videos) + + +# How many unknown Watch later videos one user may have looked up on the shared API budget in a day: +# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs. +_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600) + + +def _put_in_watch_later(db: Session, user: User) -> dict: + """Put the user's waiting Watch later videos into theirs here: the ones the catalog knows at once, + the others once YouTube was asked for them — when a YouTube-connected account can ask, today's + budget is above the backfill reserve, and the user's own daily share of lookups allows it. A video + not asked about yet waits for a later run. The lookup commits what it fetched; the caller commits + the rest.""" + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if not waiting: + return {} + ids = [vid for _, vid in waiting] + known = queries.known_video_ids(db, ids) + ask = [vid for vid in ids if vid not in known][:WATCH_LATER_MAX] + asked: set[str] = set() + fetcher = get_service_user(db) if ask else None + if ( + fetcher is not None + and not quota.backfill_reserve_reached() + and _lookups.room(str(user.id)) >= len(ask) + ): + with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP): + with YouTubeClient(db, fetcher) as yt: + ensure_videos(db, yt, ask) + # Counted once done: a lookup that failed spent none of the share (one run per user at a + # time, so the room asked above is still there). + _lookups.allow(str(user.id), cost=len(ask)) + asked = set(ask) + # Only the rows read above: one a second upload staged meanwhile was never asked about. + return service.apply_takeout_watch_later(db, user, waiting, asked) + + +def run_import(user_id: int) -> dict: + """The import job: apply what the user's kept Takeout rows can say now, as one recorded run. + The watched part commits before the Watch later part asks YouTube for the videos the catalog + lacks, so a YouTube failure there loses nothing already marked. Returns the counts.""" + with _one_import(user_id), SessionLocal() as db: + user = db.get(User, user_id) + if user is None: + return {"skipped": "no user"} + result = service.apply_takeout_watched(db, user) + db.commit() + result |= _put_in_watch_later(db, user) + db.commit() + outcomes = queries.takeout_outcomes(db, user) + result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0) + if outcomes.get((TakeoutKind.WATCH_LATER, None)): + result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)] + return result + + +def apply_waiting(db: Session) -> dict: + """Settle what every user's import left waiting and can be settled now, one commit each: a + watched video the catalog has come to know and the Shorts probe has cleared (by its job or a + search's), and the Watch later videos a run could not put in yet. Run after each run of the + `shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when + nothing was waiting that could move.""" + marked = added = failed = 0 + ready = set(queries.takeout_users_ready(db)) + later = set(queries.takeout_users_watch_later_waiting(db)) + for user_id in sorted(ready | later): + user = db.get(User, user_id) + if user is None: + continue + # One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that + # runs this nor keeps the users after them waiting. What it did not settle waits. + try: + with _one_import(user_id): + if user_id in ready: + marked += service.apply_takeout_watched(db, user)["watched"] + db.commit() + if user_id in later: + added += _put_in_watch_later(db, user).get("watch_later_added", 0) + db.commit() + except Exception: + db.rollback() + failed += 1 + log.exception( + "Takeout settling failed for user %s", + user_id, + extra=logs.event("takeout.settle.failed", user_id=user_id), + ) + out = {} + if ready: + out["takeout_watched"] = marked + if added: + out["takeout_watch_later_added"] = added + if failed: + out["takeout_failed"] = failed + return out diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py index 32e75e53..6f904dad 100644 --- a/backend/app/routes/_common.py +++ b/backend/app/routes/_common.py @@ -124,6 +124,14 @@ async def capped_body(request: Request, limit: int) -> bytes: return bytes(body) +async def drain_body(request: Request) -> None: + """Read the rest of the body to nowhere, before a refusal is raised: answered mid-upload, the + client reports a network failure, not the refusal (`capped_stream_to` says why). `bodylimit` + bounds how much there can be.""" + async for _ in request.stream(): + pass + + class ByteSink(Protocol): """Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one.""" diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 001679a0..491a104b 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,18 +1,29 @@ +import threading +from collections.abc import AsyncIterator from datetime import date +from functools import partial -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Request from pydantic import BaseModel from sqlalchemy.orm import Session from sqlalchemy.orm.exc import StaleDataError -from app import quota +from app import jobruns, quota, scheduler from app.auth import current_user -from app.db import get_db +from app.db import get_db, release_read_connection from app.mailtext import lang_for_user +from app.ratelimit import RateLimiter from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years from app.models import User -from app.reader import queries, service -from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal +from app.reader import queries, service, takeout +from app.routes._common import ( + ProgressIn, + capped_body, + drain_body, + kept_position, + not_found, + youtube_refusal, +) from app.sync.videos import parse_iso8601_duration from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError @@ -293,3 +304,116 @@ def get_video_detail( items[0].get("contentDetails", {}).get("duration") ), } + + +# --- Google Takeout ------------------------------------------------------------------------------ + + +# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are +# held whole while it is read (its text a window at a time), so these bound what the accounts can +# make the process hold. The section uploads its files one after another. +_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600) +_takeout_uploading: set[int] = set() +_takeout_uploading_guard = threading.Lock() +_TAKEOUT_AT_ONCE = 2 + + +async def _takeout_body( + request: Request, user: User = Depends(current_user), db: Session = Depends(get_db) +) -> AsyncIterator[bytes]: + # The sign-in check has run, and it was a SELECT: hand the connection back before a file of + # megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is + # taken first: the release expires the user, and reading it after would reload it, taking a + # connection straight back out. + uid = user.id + release_read_connection(db) + with _takeout_uploading_guard: + busy = uid in _takeout_uploading + full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE + taken = not busy and not full + if taken: + _takeout_uploading.add(uid) + if not taken or not _takeout_limiter.allow(str(uid)): + if taken: + with _takeout_uploading_guard: + _takeout_uploading.discard(uid) + # Read to the end before answering: refused mid-upload, the browser reports a network + # failure instead of the refusal (capped_stream_to says why). + await drain_body(request) + if busy: + raise refused(429, "takeout_busy") + if full: + raise refused(429, "takeout_server_busy") + raise refused(429, "takeout_cooldown") + try: + yield await capped_body(request, takeout.MAX_BYTES) + finally: + with _takeout_uploading_guard: + _takeout_uploading.discard(uid) + + +@router.post("/takeout") +def upload_takeout( + # The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's + # file is refused before a byte of it is read. + user: User = Depends(current_user), + body: bytes = Depends(_takeout_body), + db: Session = Depends(get_db), +): + """One file of the user's Google Takeout export — the watch history or the Watch later list. + Its videos are kept and the import runs in the background; its outcome reaches the user's inbox.""" + try: + parsed = takeout.parse(body) + except takeout.TakeoutError as exc: + if exc.reason == "no_videos": + raise refused(422, "takeout_no_videos") from None + if exc.reason == "too_many": + raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None + raise refused(422, "takeout_unreadable") from None + try: + takeout.stage(db, user, parsed) + except takeout.TooMany: + db.rollback() + raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None + db.commit() + scheduler.run_ad_hoc( + scheduler.TAKEOUT_IMPORT, + partial(takeout.run_import, user.id), + actor_id=user.id, + target=takeout.run_target(user.id), + notify=True, + ) + return { + "ok": True, + "kind": parsed.kind, + "videos": len(parsed.videos), + "entries": parsed.entries, + "ads": parsed.ads, + "other": parsed.other, + "malformed": parsed.malformed, + } + + +@router.get("/takeout") +def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)): + """What the user's Takeout import has done: its last run (none before the first upload), and + every video kept from the export, by kind and outcome — a video the catalog did not know yet + counts as "waiting" until it does, and is marked then.""" + target = takeout.run_target(user.id) + run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target) + return { + # Queued and not started: its run row is not written yet, and the last run is still the + # previous one, so the page waits for it rather than taking that one for this upload's. + "queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target), + "max_bytes": takeout.MAX_BYTES, + "run": None + if run is None + else { + "status": run.status, + "started_at": iso(run.started_at), + "finished_at": iso(run.finished_at), + "result": run.result, + "failure": run.failure, + }, + **queries.takeout_summary(db, user), + } diff --git a/backend/app/routes/runs.py b/backend/app/routes/runs.py index 2672d802..7f882b27 100644 --- a/backend/app/routes/runs.py +++ b/backend/app/routes/runs.py @@ -3,7 +3,7 @@ Admin-only and read-only — rows leave the table through the retention job (`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and the audit log beside it on the activity page. A row names what a reader needs without a second -request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only +request: what a user's ad-hoc run worked on (the channel, the importing user), and how many log lines the run left (only WARNING and up are stored, so most runs have none — the count says so before the click). """ @@ -116,8 +116,12 @@ def list_runs( limit, lambda r: r.started_at, ) - channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target} + channels = {r.target for r in runs if r.job == scheduler.CHANNEL_BACKFILL and r.target} titles = catalog_queries.channel_titles(db, channels) + for target in {r.target for r in runs if r.job == scheduler.TAKEOUT_IMPORT and r.target}: + title = scheduler.ad_hoc_target_title(db, scheduler.TAKEOUT_IMPORT, target) + if title is not None: + titles[target] = title lines: dict[int, int] = {} if runs: counted = db.execute( diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index c2a3ae09..e23eff0b 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -22,6 +22,7 @@ from app.plex.tmdb import run_people_enrich from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync from app.notifications import create_notification from app.runstatus import RunStatus, RunTrigger +from app.reader import takeout from app.state import is_sync_paused from app.sync.autotag import run_autotag_all from app.sync.explore import purge_ephemeral @@ -118,8 +119,9 @@ def _notify_done( summary: str | None, result: dict | None = None, ) -> None: - """Post a completion notice to the triggering admin's inbox (manual runs only). Best - effort — a notification failure must never mask the job's own outcome.""" + """Post a completion notice to the inbox of who started the run: an admin's manual run, or a + user's ad-hoc run that asks for one. Best effort — a notification failure must never mask the + job's own outcome.""" try: create_notification( db, @@ -272,7 +274,6 @@ def scheduler_snapshot(db) -> dict: open_runs = _open_run_ids() recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS) open_runs |= _open_run_ids() - going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None) jobs = [] for job_id, default_interval in JOB_INTERVALS.items(): a = acts.get(job_id, {}) @@ -288,30 +289,11 @@ def scheduler_snapshot(db) -> dict: ) # Work a user starts on one target: no interval, no "run now" (it needs its target), but its # runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the - # target's own (sync/runner), so "running" is the runs this process has open: two channels can - # backfill at once, so a run still going counts even when a newer one already ended. That run then - # leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the - # newer run's outcome under the older one's channel. - backfills = recent.get(CHANNEL_BACKFILL, []) - going = db.get(JobRun, going_id) if going_id is not None else None - if going is not None and going.status != RunStatus.RUNNING: - going = None # its outcome is written; only the bookkeeping has not caught up - if going is not None: - backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS] - shown = backfills[0] if backfills else None - ad_hoc = [ - { - "id": CHANNEL_BACKFILL, - "running": going is not None, - "target": shown.target if shown else None, - "target_title": ( - catalog_queries.channel_title(db, shown.target) - if shown is not None and shown.target - else None - ), - **_last_runs(backfills, open_runs), - } - ] + # target's own (sync/runner, the import's per-user lock), so "running" is the runs this process + # has open: two targets can run at once, so a run still going counts even when a newer one already + # ended. That run then leads the row, so its target, its "no outcome yet" and its dot are one + # run's, never the newer run's outcome under the older one's target. + ad_hoc = [_ad_hoc_row(db, job, recent.get(job, []), open_runs) for job in AD_HOC_JOBS] return { "running_here": running_here, "enabled": settings.scheduler_enabled, @@ -333,6 +315,27 @@ def _seen(run, open_runs: dict[int, str]) -> RunStatus: return RunStatus(run.status) +def _ad_hoc_row(db, job: str, runs: list[JobRun], open_runs: dict[int, str]) -> dict: + going_id = max((r for r, j in open_runs.items() if j == job), default=None) + going = db.get(JobRun, going_id) if going_id is not None else None + if going is not None and going.status != RunStatus.RUNNING: + going = None # its outcome is written; only the bookkeeping has not caught up + if going is not None: + runs = [going, *(r for r in runs if r.id != going.id)][:RECENT_RUNS] + shown = runs[0] if runs else None + return { + "id": job, + "running": going is not None, + "target": shown.target if shown else None, + "target_title": ( + ad_hoc_target_title(db, job, shown.target) + if shown is not None and shown.target + else None + ), + **_last_runs(runs, open_runs), + } + + def _last_runs(runs: list, open_runs: dict[int, str]) -> dict: """A row's run fields from its newest runs (newest first): how the last one went, and the few before it for the history dots.""" @@ -402,7 +405,12 @@ def _autotag_job() -> None: def _shorts_job() -> None: - _job("shorts", run_shorts) + def work(db): + # A video waiting in someone's Takeout import becomes markable the moment the probe has + # said it is not a Short, so the import's waiting rows are settled right after it. + return run_shorts(db) | takeout.apply_waiting(db) + + _job("shorts", work) def _subscriptions_job() -> None: @@ -608,48 +616,140 @@ def trigger_all(actor_id: int | None = None) -> list[str]: # one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and # spent is in `job_runs` like any job's, not a background task only the log ever heard of. CHANNEL_BACKFILL = "channel_backfill" +# A user's Google Takeout import (`reader.takeout`): its target is the user. +TAKEOUT_IMPORT = "takeout_import" # Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). -AD_HOC_JOBS = (CHANNEL_BACKFILL,) +AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT) +# What each one's target is called on the admin's pages: a channel's title, an importing user's name. +_TARGET_TITLES: dict[str, Callable[..., str | None]] = { + CHANNEL_BACKFILL: catalog_queries.channel_title, + TAKEOUT_IMPORT: takeout.target_title, +} -def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: +def ad_hoc_target_title(db, job: str, target: str) -> str | None: + """The name of an ad-hoc run's target (None for a job with none, or a target since gone).""" + title = _TARGET_TITLES.get(job) + return title(db, target) if title is not None else None + + +# The ad-hoc runs added and not yet recorded, `job@target`: between the two the target's newest +# run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too. +_queued: set[str] = set() +_queued_guard = threading.Lock() +# What the scheduler is told to run is only the key: the request itself waits here, the newest one +# per key, until a fire of that key takes it. A request made while its key's run is under way is so +# kept for the run after it, which that run asks for as it ends (`_running`): the scheduler would +# drop a second fire of an id whose run has not returned (one instance per job id). +_pending: dict[str, tuple[Callable[[], dict], int, bool]] = {} +_running: set[str] = set() +# Two at once, for the moment between a run letting go of its key and its thread returning: a fire +# then is what takes the next request, and is not dropped. +_AD_HOC_INSTANCES = 2 + + +def ad_hoc_queued(job: str, target: str) -> bool: + """Whether a run of `job` on `target` is queued and its run row not written yet.""" + with _queued_guard: + return f"{job}@{target}" in _queued + + +def run_ad_hoc( + job: str, fn: Callable[[], dict], *, actor_id: int, target: str, notify: bool = False +) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so - a second request before the first has even started is that same run.""" + a second request before the first has even started is that same run, and one made while it runs + is the run after it. `notify` posts its outcome to the user's inbox, for work whose page does not + watch it run.""" if job not in AD_HOC_JOBS: raise KeyError(job) # an unnamed job would show on the page as its bare id - sched = _scheduler - if sched is None: + if _scheduler is None: # Only between shutdown and exit: start_scheduler runs at every API startup. raise RuntimeError("the scheduler is not running") + key = f"{job}@{target}" + with _queued_guard: + _pending[key] = (fn, actor_id, notify) + _queued.add(key) + running = key in _running + if running: + log.info("ad-hoc %s for %s runs again once the run under way ends", job, target) + return + _fire(job, target) + + +def _fire(job: str, target: str) -> None: + """Add the one-shot that takes `job@target`'s pending request. Raises (the request dropped) only + when the scheduler cannot take it.""" + key = f"{job}@{target}" + sched = _scheduler try: + if sched is None: + raise RuntimeError("the scheduler is not running") sched.add_job( - _ad_hoc, + _ad_hoc_fired, "date", - args=(job, fn, actor_id, target), - id=f"{job}@{target}", + args=(job, target), + id=key, misfire_grace_time=None, + max_instances=_AD_HOC_INSTANCES, ) - # silent-ok: not a failure — the run for this target is already queued, and it does this work. + # silent-ok: not a failure — a fire of this key is already queued, and it takes this request. except ConflictingIdError: log.info("ad-hoc %s for %s already queued", job, target) + except Exception: + with _queued_guard: + _pending.pop(key, None) + _queued.discard(key) + raise -def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None: +def _ad_hoc_fired(job: str, target: str) -> None: + key = f"{job}@{target}" + with _queued_guard: + request = _pending.pop(key, None) + if request is None: + return # an earlier fire took it + _running.add(key) + fn, actor_id, notify = request + try: + _ad_hoc(job, fn, actor_id, target, notify) + finally: + with _queued_guard: + _running.discard(key) + again = key in _pending + if again: + try: + _fire(job, target) + except Exception: + # The request made meanwhile is lost with the scheduler: shut down, nothing runs. + log.exception("ad-hoc %s for %s not run again", job, target) + + +def _ad_hoc( + job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False +) -> None: + key = f"{job}@{target}" try: run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) except Exception: # No row to tell it in: the database is what failed. The log is the record. log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target) return + finally: + # Recorded (or never will be): the run row now speaks for it — unless another request came + # meanwhile, which is still queued. + with _queued_guard: + if key not in _pending: + _queued.discard(key) _opened(run_id, job) try: - _ad_hoc_work(job, fn, run_id) + _ad_hoc_work(job, fn, run_id, actor_id if notify else None) finally: _closed(run_id) -def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: +def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | None) -> None: with logs.bind(run_id=run_id): # As in _run: only the work is inside the try. try: @@ -658,8 +758,23 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) err, failure = _failure(exc) _close(run_id, RunStatus.ERROR, error=err, failure=failure) + # The user's notice names the failure, never the upstream's own text: that is an English + # diagnostic, for the admin's run row and the log, not a user's inbox. + _notify_ad_hoc( + notify_id, job, RunStatus.ERROR, None, failure or {"failure": "unexpected"} + ) return - _finish_ok(job, run_id, result) + summary, data = _finish_ok(job, run_id, result) + _notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data) + + +def _notify_ad_hoc( + user_id: int | None, job: str, status: RunStatus, summary: str | None, result: dict | None +) -> None: + if user_id is None: + return + with SessionLocal() as db: + _notify_done(db, user_id, job, status, summary, result) # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of @@ -700,3 +815,7 @@ def shutdown_scheduler() -> None: if _scheduler is not None: _scheduler.shutdown(wait=False) _scheduler = None + with _queued_guard: + # What it held will not run. + _queued.clear() + _pending.clear() diff --git a/backend/requirements.lock b/backend/requirements.lock index 88f15315..487b8557 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -6,7 +6,7 @@ # bumped in place instead of a full regeneration, which would move every other package too — then # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403); -# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2). +# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322). APScheduler==3.11.3 Authlib==1.7.2 Mako==1.4.1 @@ -21,7 +21,7 @@ anyio==4.14.2 argon2-cffi-bindings==25.1.0 argon2-cffi==25.1.0 bgutil-ytdlp-pot-provider==1.3.1 -brotli==1.2.0 +brotli==1.2.1 certifi==2026.10.19 cffi==2.1.1 charset-normalizer==3.5.1 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 01b9c1ce..66cb9a8b 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -327,3 +327,15 @@ def _reset_log_erasures(): from app import logs logs._erased.clear() + + +@pytest.fixture(autouse=True) +def _reset_ad_hoc_queue(): + """The scheduler's ad-hoc queue is module-level too: a request a test left pending would read as + a later test's user's import still queued (ids restart from 1).""" + from app import scheduler + + with scheduler._queued_guard: + scheduler._queued.clear() + scheduler._pending.clear() + scheduler._running.clear() diff --git a/backend/tests/test_activity_reads.py b/backend/tests/test_activity_reads.py index c4df1b02..59efed36 100644 --- a/backend/tests/test_activity_reads.py +++ b/backend/tests/test_activity_reads.py @@ -159,6 +159,15 @@ def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db, assert facets["total"] == 4 +def test_the_run_list_names_the_user_a_takeout_import_ran_for(db, make_user, as_user): + admin = make_user(role="admin") + importer = make_user("importer@example.com") + imported = _run(db, "takeout_import", "ok", importer.id, T0, f"user:{importer.id}", "ad_hoc") + db.commit() + (row,) = as_user(admin).get("/api/admin/runs").json()["items"] + assert (row["id"], row["target_title"]) == (imported, "importer@example.com") + + def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user): admin = make_user(role="admin") now = datetime.now(timezone.utc) diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py index dc308d01..f2239d2a 100644 --- a/backend/tests/test_body_limit.py +++ b/backend/tests/test_body_limit.py @@ -23,6 +23,7 @@ from app import bodylimit, debugsession, sysconfig from app.config import settings from app.models import SystemConfig from app.bodylimit import DEFAULT_MAX_BODY_BYTES, BodyLimitMiddleware +from app.reader import takeout from app.routes import diagnostics as diagnostics_routes SMALL = b"x" * 16 @@ -221,6 +222,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes(): CAPPED_BODY_SITES = { ("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"), ("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"), + ("feed.py", "_takeout_body", "capped_body", "takeout.MAX_BYTES"), ("files.py", "spool_upload", "capped_stream_to", "cap"), } @@ -368,6 +370,9 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling(): # The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather # than being a constant to compare against — that pairing is its own test below. assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES + # A Takeout file is larger than the default: without its ceiling every real history is a 413. + assert takeout.MAX_BYTES > DEFAULT_MAX_BODY_BYTES + assert bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/takeout")] == takeout.MAX_BYTES def test_release_read_connection_ends_the_transaction_it_was_called_for(db): diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 8f795b46..cab4b7bf 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -6,6 +6,7 @@ through (at the top of _job): holding _activity_lock across the check-and-set me caller can hold a job's "running" claim at a time, and it's claimable again once released.""" import logging +import threading import time from datetime import datetime, timedelta, timezone @@ -342,6 +343,29 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure assert (failed.status, failed.error) == ("error", "quota exceeded") +def _ad_hoc_row(db, job: str) -> dict: + rows = scheduler.scheduler_snapshot(db)["ad_hoc"] + assert [r["id"] for r in rows] == list(scheduler.AD_HOC_JOBS) # every one has its row + return next(r for r in rows if r["id"] == job) + + +def test_a_takeout_import_has_its_row_named_by_the_importing_user(db, make_user): + user = make_user("importer@example.com") + user.display_name = "Ida Importer" + db.commit() + assert _ad_hoc_row(db, "takeout_import")["recent"] == [] + scheduler._ad_hoc("takeout_import", lambda: {"watched": 2}, user.id, f"user:{user.id}") + + row = _ad_hoc_row(db, "takeout_import") + assert (row["running"], row["target"], row["target_title"], row["status"]) == ( + False, + f"user:{user.id}", + "Ida Importer", + "ok", + ) + assert _ad_hoc_row(db, "channel_backfill")["recent"] == [] + + def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user): user = make_user("shown@example.com") db.add(Channel(id="UC_shown", title="BBC Earth")) @@ -349,7 +373,7 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none") scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown") - (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + row = _ad_hoc_row(db, "channel_backfill") assert (row["id"], row["running"], row["target"], row["target_title"]) == ( "channel_backfill", False, @@ -367,12 +391,12 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m seen = [] scheduler._ad_hoc( "channel_backfill", - lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {}, + lambda: seen.append(_ad_hoc_row(db, "channel_backfill")["running"]) or {}, user.id, "UC_shown", ) assert seen == [True] - assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False + assert _ad_hoc_row(db, "channel_backfill")["running"] is False def _finished(db, job) -> list[JobRun]: @@ -397,9 +421,46 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run( "channel_backfill@UC_x", "channel_backfill@UC_y", ] + # Queued, so its target's newest run row is not this run's yet: a page watching it can tell. + assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_x") + assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_z") live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next +def test_a_request_made_while_its_run_is_under_way_is_the_run_after_it( + db, make_user, live_scheduler +): + """The scheduler drops a second fire of an id whose run has not returned: the second Takeout file + of a pair, uploaded while the first one's import ran, was never imported, and its key stayed + queued until a restart (the page spun on "Processing…" for good).""" + user = make_user("again@example.com") + first_runs, release = threading.Event(), threading.Event() + calls = [] + + def first(): + first_runs.set() + assert release.wait(10) + calls.append("first") + return {"n": 1} + + def second(): + calls.append("second") + return {"n": 2} + + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, first, actor_id=user.id, target="UC_again") + assert first_runs.wait(10) + scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, second, actor_id=user.id, target="UC_again") + assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + release.set() + + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2: + time.sleep(0.05) + assert calls == ["first", "second"] + assert sorted(r.result["n"] for r in _finished(db, "channel_backfill")) == [1, 2] + assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again") + + def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user): """The scheduler's pool threads are reused: an actor a manual run left set made the next scheduled run on that thread an admin's "manual" run, with a notice in their inbox.""" @@ -475,7 +536,7 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip") assert calls == ["ok"] # Its row is still `running`, but nothing here is: the page does not call it going. - (row,) = scheduler.scheduler_snapshot(db)["ad_hoc"] + row = _ad_hoc_row(db, "channel_backfill") assert (row["running"], row["status"], row["recent"][0]["status"]) == ( False, "interrupted", @@ -529,7 +590,7 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use return {} scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long") - (row,) = rows + (row,) = [r for r in rows if r["id"] == "channel_backfill"] assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one") # The outcome beside it is the long one's (none yet), never the short one's skip. assert (row["status"], row["result"], row["last_error"]) == (None, None, None) diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py new file mode 100644 index 00000000..d63f3475 --- /dev/null +++ b/backend/tests/test_takeout.py @@ -0,0 +1,1019 @@ +"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the +import job makes of them — watched here (never a Short, never over a state the user set), waiting for +a video the catalog does not know yet, and the Watch later list joined to the user's own.""" + +import json +import tracemalloc +from datetime import datetime, timedelta, timezone + +import pytest +from fastapi import Depends +from sqlalchemy import select, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import Session + +from app import jobruns, quota, scheduler +from app.auth import current_user +from app.db import SessionLocal, get_db +from app.catalog.models import Channel, Subscription, Video +from app.models import JobRun, Notification, User +from app.runstatus import RunStatus, RunTrigger +from app.reader import queries, service, takeout +from app.routes import feed as feed_routes +from app.reader.models import ( + Playlist, + PlaylistItem, + TakeoutItem, + TakeoutKind, + VideoState, +) + +T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc) +CHANNEL = "UC" + "a" * 22 + + +def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra): + entry = { + "header": "YouTube", + "title": "Megnézted: valami", # the account's language: never read + "titleUrl": f"https://{host}.youtube.com/watch?v={vid}", + "time": at.isoformat().replace("+00:00", "Z"), + "products": ["YouTube"], + "activityControls": ["YouTube watch history"], + } + if channel: + entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}] + return entry | extra + + +def _history(*entries) -> bytes: + return json.dumps(list(entries)).encode() + + +# --- reading the files ------------------------------------------------------------------------- + + +def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel(): + later = T0 + timedelta(days=3) + parsed = takeout.parse( + _history( + _view("aaaaaaaaaaa", later), + _view("aaaaaaaaaaa", T0), + _view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel + _view("ccccccccccc", T0, host="music"), + # Out of order: the later view still wins. + _view("ddddddddddd", T0, channel=None), + _view("ddddddddddd", later), + ) + ) + assert parsed.kind == TakeoutKind.WATCHED + assert parsed.entries == 6 + assert parsed.videos == { + "aaaaaaaaaaa": (later, CHANNEL), + "bbbbbbbbbbb": (T0, None), + "ccccccccccc": (T0, CHANNEL), + "ddddddddddd": (later, CHANNEL), + } + + +def test_the_history_counts_what_it_leaves_out(): + ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}]) + post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"} + no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"} + bad_time = _view("ggggggggggg") | {"time": "yesterday"} + no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"} + parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk")) + assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2) + assert parsed.videos == { + "hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL) + } + + +def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none(): + body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n" + parsed = takeout.parse(("" + body).encode()) + assert parsed.kind == TakeoutKind.WATCH_LATER + assert parsed.videos == { + "Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None) + } + assert parsed.malformed == 1 + headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n") + assert list(headless.videos) == ["Ngb04xUK8uU"] + + +@pytest.mark.parametrize( + ("body", "reason"), + [ + (b'{"some": "other", "export": "file"}', "not_takeout"), + (b"watch history, as a page", "not_takeout"), + (b"[not json", "not_takeout"), + (b"\xff\xfe\x00", "not_takeout"), + (b"one column only\n", "not_takeout"), + (b"[]", "no_videos"), + ( + json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(), + "no_videos", + ), + # Read an entry at a time, the list must still be one: nothing after it, nothing missing. + (_history(_view("aaaaaaaaaaa")) + b" []", "not_takeout"), + (_history(_view("aaaaaaaaaaa"))[:-1], "not_takeout"), + (b"[" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" x]", "not_takeout"), + # However a hostile file fails, it is one the import cannot read, not a server error. + (b"[" * 100_000, "not_takeout"), + (b"[" + b"1" * 5000 + b"]", "not_takeout"), + (b"Ngb04xUK8uU," + b"x" * 200_000 + b"\n", "not_takeout"), + ( + "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode(), + "too_many", + ), + ], +) +def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason): + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == reason + + +def test_the_history_is_read_an_entry_at_a_time_not_as_one_tree(): + """A file of a million tiny entries must cost about its own size, not a million dicts at once: + the whole tree built first measured ~25 times the file.""" + body = b"[" + b",".join([b"{}"] * 1_000_000) + b"]" # 3 MB + tracemalloc.start() + try: + with pytest.raises(takeout.TakeoutError): + takeout.parse(body) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < 5 * len(body), f"parsing held {peak / len(body):.1f} times the file" + + +def test_a_history_with_an_emoji_costs_less_than_its_own_size_to_read(monkeypatch): + """Decoded whole, one emoji makes the text four bytes a character: a 32 MiB history held some + 128 MiB of it on top of its bytes. Read a window at a time, it holds a window (a few times its + bytes as text) whatever the file's size — a small one here, so a 9 MB file can show it.""" + monkeypatch.setattr(takeout, "_WINDOW", 64 * 1024) + view = _view("aaaaaaaaaaa", title="Watched " + "😀" * 40) + body = b"[" + b",".join([json.dumps(view, ensure_ascii=False).encode()] * 20_000) + b"]" + tracemalloc.start() + try: + parsed = takeout.parse(body) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert parsed.entries == 20_000 + assert peak < len(body) / 2, f"parsing held {peak / len(body):.1f} times the file" + + +@pytest.mark.parametrize("split", [1, 7, 100, 4096]) +def test_the_history_reads_the_same_whatever_its_window(monkeypatch, split): + """An entry, a character or a number cut by the window's edge is read whole.""" + monkeypatch.setattr(takeout, "_WINDOW", split) + views = [_view(f"{c * 11}", title="ő😀" * 3) for c in "abcdef"] + body = " [\n" + ",\n".join(json.dumps(v, ensure_ascii=False) for v in views) + "\n]\n" + parsed = takeout.parse(body.encode()) + assert (parsed.entries, sorted(parsed.videos)) == (6, [c * 11 for c in "abcdef"]) + numbers = takeout._json_list(b"[123456789, 2]", 0) + assert list(numbers) == [123456789, 2] + + +def test_a_watch_later_file_far_larger_than_any_list_is_refused_unread(): + body = b"Video ID,Time\n" + b"x" * (takeout.CSV_MAX_BYTES) + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == "too_many" + + +def test_the_id_lists_are_asked_in_chunks(db, make_user, monkeypatch): + user = make_user("chunks@example.com") + ids = [f"chunk{n:06d}" for n in range(5)] + for vid in ids[:3]: + _video(db, vid) + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {vid: (T0, None) for vid in ids[:4]}) + db.commit() + monkeypatch.setattr(queries, "_IN_CHUNK", 2) + assert queries.known_video_ids(db, ids) == set(ids[:3]) + assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids) == 0 + assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids[:1]) == 3 + + +def test_the_history_is_read_whatever_its_whitespace(): + body = b" \n[ \n" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" ,\n\t" + body += json.dumps(_view("bbbbbbbbbbb")).encode() + b"\n] \n" + assert sorted(takeout.parse(body).videos) == ["aaaaaaaaaaa", "bbbbbbbbbbb"] + full = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5000)) + assert len(takeout.parse(full.encode()).videos) == 5000 # YouTube's own cap still fits + + +# --- keeping and applying ---------------------------------------------------------------------- + + +def _video(db, vid: str, *, probed=True, short=False): + if db.get(Channel, CHANNEL) is None: + db.add(Channel(id=CHANNEL, title="Ch")) + db.flush() + db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short)) + + +def _state(db, user, vid, status, *, changed, position=0, watched_at=None): + db.add( + VideoState( + user_id=user.id, + video_id=vid, + status=status, + position_seconds=position, + watched_at=watched_at, + updated_at=changed, + ) + ) + + +def _rows(db, user) -> dict[str, TakeoutItem]: + db.expire_all() + rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars() + return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows} + + +def _states(db, user) -> dict[str, VideoState]: + db.expire_all() + rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars() + return {r.video_id: r for r in rows} + + +def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept( + db, make_user +): + user = make_user("takeout@example.com") + for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"): + _video(db, vid, probed=vid != "unprobed000", short=vid == "short000000") + for vid in ("newer000000", "older000000"): + _video(db, vid) + earlier = T0 - timedelta(days=30) + _state(db, user, "hidden00000", "hidden", changed=earlier) + _state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier) + # In progress here, touched after the YouTube view: what the user did here is newer. + _state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40) + # In progress here, but last touched before the YouTube view: they finished it there. + _state(db, user, "older000000", "new", changed=earlier, position=40) + db.commit() + ids = [ + "plain000000", + "short000000", + "unprobed000", + "unknown0000", + "hidden00000", + "watched0000", + "newer000000", + "older000000", + ] + service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids}) + + counts = service.apply_takeout_watched(db, user) + db.commit() + + assert counts == {"watched": 2, "kept": 3, "shorts": 1} + rows, states = _rows(db, user), _states(db, user) + assert {v: rows[v].outcome for v in ids} == { + "plain000000": "watched", + "short000000": "short", + "unprobed000": None, + "unknown0000": None, + "hidden00000": "kept", + "watched0000": "kept", + "newer000000": "kept", + "older000000": "watched", + } + assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0) + assert "short000000" not in states + assert states["hidden00000"].status == "hidden" + assert states["watched0000"].watched_at == earlier + assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40) + assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40) + + +@pytest.mark.parametrize("undo", ["unmark", "clear_state"]) +def test_a_video_the_user_un_marked_here_is_not_marked_again_by_an_older_view(db, make_user, undo): + """Un-marking (or resetting) a video leaves no state row, and the import inserted a watched one + over the user's own choice. A view on YouTube after the un-mark is a new view, and counts.""" + user = make_user(f"{undo}@example.com") + for vid in ("undone00000", "again000000"): + _video(db, vid) + db.commit() + for vid in ("undone00000", "again000000"): + service.mark(db, user, vid, "watched") + getattr(service, undo)(db, user, vid) + db.commit() + assert _states(db, user) == {} + later = datetime.now(timezone.utc) + timedelta(minutes=5) + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"undone00000": (T0, None), "again000000": (later, None)}, + ) + db.commit() + + assert takeout.run_import(user.id)["watched"] == 1 + assert set(_states(db, user)) == {"again000000"} + assert _rows(db, user)["undone00000"].outcome == "kept" + + +def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user): + user = make_user("again@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + service.apply_takeout_watched(db, user) + later = T0 + timedelta(days=1) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)}) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + db.commit() + row = _rows(db, user)["plain000000"] + assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched") + + +def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user): + user = make_user("later@example.com") + for vid in ("held0000000", "old00000000", "new00000000"): + _video(db, vid) + db.commit() + pl = service.watch_later(db, user) + service.add_item(db, pl, "held0000000", mark_dirty=False) + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + { + "new00000000": (T0, None), + "held0000000": (T0, None), + "old00000000": (T0 - timedelta(days=9), None), + "gone0000000": (T0, None), # YouTube no longer has it + }, + ) + + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unasked0000": (T0, None)}) + rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + + counts = service.apply_takeout_watch_later(db, user, rows, asked={"gone0000000"}) + db.commit() + + assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1} + # Unknown and not asked about: it waits, it is not called gone. + assert _rows(db, user)["WL:unasked0000"].outcome is None + items = db.execute( + select(PlaylistItem.video_id) + .where(PlaylistItem.playlist_id == pl.id) + .order_by(PlaylistItem.position) + ).scalars() + assert list(items) == ["held0000000", "old00000000", "new00000000"] + assert _rows(db, user)["WL:gone0000000"].outcome == "missing" + assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl] + + +@pytest.mark.parametrize("n", [3, 40]) +def test_a_watch_later_list_goes_in_in_statements_that_do_not_grow_with_it(db, make_user, n): + """A video at a time, a full list of 5000 was some 20,000 statements under the user's lock.""" + from sqlalchemy import event + + user = make_user(f"many{n}@example.com") + ids = [f"wl{i:09d}" for i in range(n)] + for vid in ids: + _video(db, vid) + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + {v: (T0 + timedelta(minutes=i), None) for i, v in enumerate(ids)}, + ) + db.commit() + rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + service.watch_later(db, user) # the list made beforehand: its own insert is not this count's + db.flush() + statements = [] + engine = db.get_bind() + listener = lambda *a: statements.append(a[2]) # noqa: E731 — the event's own signature + event.listen(engine, "before_cursor_execute", listener) + try: + assert service.apply_takeout_watch_later(db, user, rows, set())["watch_later_added"] == n + finally: + event.remove(engine, "before_cursor_execute", listener) + db.commit() + assert len(statements) <= 9, statements + held = [i.video_id for i in queries.items(db, service.watch_later(db, user).id)] + assert held == ids + + +def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( + db, make_user, monkeypatch +): + user = make_user("job@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)}, + ) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)}) + db.commit() + asked = [] + + def fake_ensure(session, yt, ids): + # The lookup is the importing user's spend, on the import's own action. + asked.append((ids, quota._actor_id.get(), quota._action.get())) + _video(session, "fetched0000") + session.flush() + + monkeypatch.setattr(takeout, "ensure_videos", fake_ensure) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + + result = takeout.run_import(user.id) + + assert asked == [(["fetched0000"], user.id, quota.QuotaAction.VIDEOS_LOOKUP)] + assert result == { + "watched": 1, + "kept": 0, + "shorts": 0, + "watch_later_added": 1, + "watch_later_present": 0, + "watch_later_missing": 0, + "waiting": 1, + } + + +def _wl_job(db, make_user, monkeypatch, email, *, fetcher=True, reserve=False): + user = make_user(email) + _video(db, "known000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + {"known000000": (T0, None), "unknown0000": (T0 + timedelta(minutes=1), None)}, + ) + db.commit() + asked = [] + monkeypatch.setattr(takeout, "ensure_videos", lambda session, yt, ids: asked.append(ids)) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user if fetcher else None) + monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: reserve) + return user, asked + + +@pytest.mark.parametrize( + "why", ["no account to ask with", "the budget's reserve", "the user's share"] +) +def test_a_watch_later_video_not_asked_about_waits_and_the_known_ones_go_in( + db, make_user, monkeypatch, why +): + user, asked = _wl_job( + db, + make_user, + monkeypatch, + "wait@example.com", + fetcher=why != "no account to ask with", + reserve=why == "the budget's reserve", + ) + if why == "the user's share": + monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(0, 3600)) + + result = takeout.run_import(user.id) + + assert asked == [] + assert (result["watch_later_added"], result["watch_later_missing"]) == (1, 0) + assert result["watch_later_waiting"] == 1 + assert _rows(db, user)["WL:unknown0000"].outcome is None + + +def test_the_shorts_job_puts_in_the_watch_later_videos_a_run_could_not(db, make_user, monkeypatch): + user, asked = _wl_job(db, make_user, monkeypatch, "retry@example.com", fetcher=False) + takeout.run_import(user.id) + assert asked == [] + + # A YouTube-connected account exists now; the next probe run asks for what waits. + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + + def fetched(session, yt, ids): + asked.append(ids) + _video(session, "unknown0000") + session.flush() + + monkeypatch.setattr(takeout, "ensure_videos", fetched) + assert takeout.apply_waiting(db) == {"takeout_watch_later_added": 1} + assert asked == [["unknown0000"]] + assert _rows(db, user)["WL:unknown0000"].outcome == "added" + assert takeout.apply_waiting(db) == {} + + +def test_one_users_failure_in_the_shorts_job_leaves_the_others_and_the_probe_alone( + db, make_user, monkeypatch +): + """A Watch later lookup that raised (a revoked token, a refused quota) escaped the Shorts job: + its run was recorded as failed with its own counts lost, and every user after this one waited.""" + first, second = make_user("first@example.com"), make_user("second@example.com") + _video(db, "ready000000") + db.commit() + service.stage_takeout(db, first, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)}) + service.stage_takeout(db, second, TakeoutKind.WATCHED, {"ready000000": (T0, None)}) + db.commit() + + def refused(session, yt, ids): + raise RuntimeError("invalid_grant") + + monkeypatch.setattr(takeout, "ensure_videos", refused) + monkeypatch.setattr(takeout, "get_service_user", lambda session: first) + monkeypatch.setattr(scheduler, "run_shorts", lambda session: {"probed": 4}) + + scheduler._shorts_job() + + db.expire_all() + run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one() + assert run.status == "ok" + assert run.result == {"probed": 4, "takeout_watched": 1, "takeout_failed": 1} + assert _states(db, second)["ready000000"].status == "watched" + assert _rows(db, first)["WL:unknown0000"].outcome is None + # A lookup that failed spent none of the user's daily share. + assert not takeout._lookups.exhausted(str(first.id)) + assert takeout._lookups.room(str(first.id)) == 2 * takeout.WATCH_LATER_MAX + + +def test_a_watch_later_row_staged_while_youtube_is_asked_is_left_for_the_next_run( + db, make_user, monkeypatch +): + user, _ = _wl_job(db, make_user, monkeypatch, "race@example.com") + + def meanwhile(session, yt, ids): + # A second upload lands while the lookup runs: its video was never asked about. + with SessionLocal() as other: + service.stage_takeout( + other, + other.get(User, user.id), + TakeoutKind.WATCH_LATER, + {"second00000": (T0, None)}, + ) + other.commit() + + monkeypatch.setattr(takeout, "ensure_videos", meanwhile) + takeout.run_import(user.id) + + rows = _rows(db, user) + assert rows["WL:unknown0000"].outcome == "missing" # asked, and YouTube has it no longer + assert rows["WL:second00000"].outcome is None + + +def test_a_youtube_failure_keeps_what_the_watched_part_already_marked(db, make_user, monkeypatch): + user = make_user("fails@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, CHANNEL)}) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)}) + db.commit() + + def refused(session, yt, ids): + raise RuntimeError("quota exceeded") + + monkeypatch.setattr(takeout, "ensure_videos", refused) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + with pytest.raises(RuntimeError): + takeout.run_import(user.id) + + with SessionLocal() as fresh: + state = fresh.execute(select(VideoState).where(VideoState.user_id == user.id)).scalar_one() + assert (state.video_id, state.status) == ("plain000000", "watched") + outcome = fresh.execute( + select(TakeoutItem.outcome).where(TakeoutItem.video_id == "plain000000") + ).scalar_one() + assert outcome == "watched" + + +def test_staging_and_settling_take_the_users_one_lock(db, make_user): + """An upload's staging and an import's settling lock the same rows in no common order: one lock + over both is what keeps the two from deadlocking.""" + user = make_user("lock@example.com") + db.commit() + steps = { + "stage": lambda s, u: service.stage_takeout( + s, u, TakeoutKind.WATCHED, {"aaaaaaaaaaa": (T0, None)} + ), + "watched": lambda s, u: service.apply_takeout_watched(s, u), + "watch later": lambda s, u: service.apply_takeout_watch_later( + s, u, [(1, "aaaaaaaaaaa")], set() + ), + # Between the import's read of the un-marks and its marking, one would be overwritten. + "unmark": lambda s, u: service.unmark(s, u, "aaaaaaaaaaa"), + "reset": lambda s, u: service.clear_state(s, u, "aaaaaaaaaaa"), + } + with SessionLocal() as holder: + service.lock_takeout(holder, holder.get(User, user.id)) + for name, step in steps.items(): + with SessionLocal() as other: + other.execute(text("SET lock_timeout = '200ms'")) + with pytest.raises(OperationalError, match="lock timeout"): + step(other, other.get(User, user.id)) + pytest.fail(f"{name} did not wait for the user's lock") + holder.rollback() + + +# --- the upload -------------------------------------------------------------------------------- + + +@pytest.fixture +def queued(monkeypatch): + runs = [] + monkeypatch.setattr( + scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, fn.func, fn.args, kw)) + ) + return runs + + +def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice( + db, make_user, as_user, queued +): + user = make_user("upload@example.com") + body = _history( + _view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}]) + ) + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 200, resp.text + assert resp.json() == { + "ok": True, + "kind": "watched", + "videos": 1, + "entries": 2, + "ads": 1, + "other": 0, + "malformed": 0, + } + assert list(_rows(db, user)) == ["aaaaaaaaaaa"] + assert queued == [ + ( + "takeout_import", + takeout.run_import, + (user.id,), + {"actor_id": user.id, "target": f"user:{user.id}", "notify": True}, + ) + ] + + +@pytest.mark.parametrize( + ("body", "reason"), + [(b"", "takeout_unreadable"), (b"[]", "takeout_no_videos")], +) +def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language( + db, make_user, as_user, queued, body, reason +): + user = make_user("refused@example.com") + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "refused", "reason": reason} + assert queued == [] and _rows(db, user) == {} + + +def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch): + user = make_user("big@example.com") + monkeypatch.setattr(takeout, "MAX_BYTES", 64) + resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 413 + assert _rows(db, user) == {} + + +def _counting_body(chunks: int = 4, size: int = 1024): + """A body that records how much of itself the server pulled (test_upload.py's probe).""" + pulled: list[int] = [] + + def body(): + for _ in range(chunks): + pulled.append(size) + yield b"x" * size + + return body(), pulled + + +def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client): + body, pulled = _counting_body() + assert client.post("/api/takeout", content=body).status_code == 401 + assert pulled == [], f"the server read {sum(pulled)} bytes from an anonymous caller" + + +def test_a_signed_in_upload_has_its_body_read(db, make_user, as_user, queued): + """The mutation check for the test above: a route reading nobody's body would pass it too.""" + body, pulled = _counting_body() + c = as_user(make_user("read@example.com")) + assert c.post("/api/takeout", content=body).status_code == 422 + assert len(pulled) == 4 + + +def test_the_upload_arrives_with_no_connection_held( + db, make_user, as_user, override_dep, queued, monkeypatch +): + """The read connection is given back before the body: reading the user's id after that reloaded + the user and took one straight back out, `idle in transaction` for the whole upload.""" + user = make_user("idle@example.com") + client = as_user(user) + + def signed_in(session: Session = Depends(get_db)) -> User: + # As the real sign-in check: a SELECT on the request's session, which the release ends — + # and expires what it loaded. (`as_user`'s own resolver reads nothing.) + session.expire_all() + return session.get(User, user.id) + + override_dep(current_user, signed_in) + sessions, held = [], [] + release, capped = feed_routes.release_read_connection, feed_routes.capped_body + + def releasing(session): + sessions.append(session) + release(session) + + async def reading(request, limit): + held.extend(s.in_transaction() for s in sessions) + return await capped(request, limit) + + monkeypatch.setattr(feed_routes, "release_read_connection", releasing) + monkeypatch.setattr(feed_routes, "capped_body", reading) + resp = client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 200, resp.text + assert held == [False] + + +def test_uploads_past_the_hourly_limit_are_refused_after_their_body_arrived( + db, make_user, as_user, queued +): + c = as_user(make_user("many@example.com")) + for _ in range(10): + assert c.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200 + body, pulled = _counting_body() + resp = c.post("/api/takeout", content=body) + assert resp.status_code == 429 + assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_cooldown"} + # Read to the end first: refused mid-upload, the browser would say "network error" instead. + assert len(pulled) == 4 + assert len(queued) == 10 + + +def test_a_second_upload_while_one_of_the_users_is_still_arriving_is_refused( + db, make_user, as_user, queued, monkeypatch +): + user = make_user("busy@example.com") + monkeypatch.setattr(feed_routes, "_takeout_uploading", {user.id}) + resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "takeout_busy" + assert queued == [] + # Another user's upload is not held up by it, and lets go of its own place when it is done. + other = as_user(make_user("free@example.com")) + assert other.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200 + assert feed_routes._takeout_uploading == {user.id} + + +def test_a_third_upload_across_users_while_two_arrive_is_refused_after_its_body( + db, make_user, as_user, queued, monkeypatch +): + """Each upload's bytes are held whole while it arrives: with no bound across users, a few at + once could take the process past its memory.""" + monkeypatch.setattr(feed_routes, "_takeout_uploading", {9001, 9002}) + body, pulled = _counting_body() + resp = as_user(make_user("third@example.com")).post("/api/takeout", content=body) + assert resp.status_code == 429 + assert resp.json()["detail"]["reason"] == "takeout_server_busy" + assert len(pulled) == 4 + assert queued == [] and feed_routes._takeout_uploading == {9001, 9002} + + +def test_an_upload_that_would_keep_too_many_videos_is_refused( + db, make_user, as_user, queued, monkeypatch +): + user = make_user("cap@example.com") + c = as_user(user) + monkeypatch.setattr(takeout, "ROWS_MAX", 3) + two = _history(_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb")) + assert c.post("/api/takeout", content=two).status_code == 200 + assert c.post("/api/takeout", content=two).status_code == 200 # the same videos again: no more + resp = c.post("/api/takeout", content=_history(_view("ccccccccccc"), _view("ddddddddddd"))) + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_too_many", "limit": 3} + assert sorted(_rows(db, user)) == ["aaaaaaaaaaa", "bbbbbbbbbbb"] + assert len(queued) == 2 + + +def test_a_watch_later_file_longer_than_youtube_allows_is_refused(db, make_user, as_user, queued): + body = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode() + resp = as_user(make_user("long@example.com")).post("/api/takeout", content=body) + assert resp.status_code == 422 + assert resp.json()["detail"] == { + "code": "refused", + "reason": "takeout_list_too_long", + "limit": 5000, + } + + +# --- the notice -------------------------------------------------------------------------------- + + +def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user): + user = make_user("notice@example.com") + scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True) + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x") + db.expire_all() + notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars() + assert [(n.data["job_id"], n.data["result"]) for n in notices] == [ + ("takeout_import", {"watched": 3}) + ] + + +def test_a_failed_run_tells_its_user_the_failure_never_the_upstreams_text(db, make_user): + user = make_user("failnotice@example.com") + + def fails(): + raise RuntimeError("(psycopg.errors.Something) [SQL: SELECT secret]") + + scheduler._ad_hoc("takeout_import", fails, user.id, "user:1", True) + scheduler._ad_hoc("channel_backfill", fails, user.id, "UC_x") + db.expire_all() + (notice,) = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars() + assert notice.body is None + assert notice.data == { + "job_id": "takeout_import", + "status": "error", + "summary": None, + "result": {"failure": "unexpected"}, + } + # The admin's run row keeps the diagnostic. + run = db.execute(select(JobRun).where(JobRun.job == "takeout_import")).scalar_one() + assert "SELECT secret" in run.error + + +def test_a_queued_import_is_told_apart_from_the_last_one_until_its_run_is_recorded( + db, make_user, as_user, monkeypatch +): + user = make_user("queued@example.com") + added = [] + + class Held: + def add_job(self, *args, **kwargs): + added.append(kwargs["id"]) + + monkeypatch.setattr(scheduler, "_scheduler", Held()) + target = takeout.run_target(user.id) + scheduler.run_ad_hoc("takeout_import", dict, actor_id=user.id, target=target) + assert scheduler.ad_hoc_queued("takeout_import", target) + assert as_user(user).get("/api/takeout").json()["queued"] is True + + scheduler._ad_hoc_fired("takeout_import", target) + assert not scheduler.ad_hoc_queued("takeout_import", target) + assert as_user(user).get("/api/takeout").json()["queued"] is False + assert added == [f"takeout_import@{target}"] + + +def test_an_upload_while_its_import_starts_stays_queued_for_the_run_after( + db, make_user, monkeypatch +): + """A request landing while the run before it writes its row is the run after that one: still + queued, so the page does not take the run starting now for it.""" + user = make_user("starting@example.com") + added, ran = [], [] + + class Held: + def add_job(self, *args, **kwargs): + added.append(kwargs["id"]) + + monkeypatch.setattr(scheduler, "_scheduler", Held()) + target = takeout.run_target(user.id) + start = jobruns.start + + def starting(*args, **kwargs): + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(2) or {}, actor_id=user.id, target=target + ) + return start(*args, **kwargs) + + scheduler.run_ad_hoc( + "takeout_import", lambda: ran.append(1) or {}, actor_id=user.id, target=target + ) + monkeypatch.setattr(scheduler.jobruns, "start", starting) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1] + assert scheduler.ad_hoc_queued("takeout_import", target) + assert added == [f"takeout_import@{target}"] * 2 # the run after it, asked for as it ended + + monkeypatch.setattr(scheduler.jobruns, "start", start) + scheduler._ad_hoc_fired("takeout_import", target) + assert ran == [1, 2] + assert not scheduler.ad_hoc_queued("takeout_import", target) + + +# --- the videos that wait ---------------------------------------------------------------------- + + +def test_waiting_videos_are_marked_once_the_shorts_probe_has_cleared_them(db, make_user): + a, b = make_user("a@example.com"), make_user("b@example.com") + _video(db, "later000000", probed=False) + _video(db, "notyet00000", probed=False) + db.commit() + service.stage_takeout( + db, a, TakeoutKind.WATCHED, {"later000000": (T0, None), "notyet00000": (T0, None)} + ) + service.stage_takeout(db, b, TakeoutKind.WATCHED, {"later000000": (T0, None)}) + db.commit() + assert takeout.apply_waiting(db) == {} + + # The probe clears one of them; the other is still unprobed. + db.get(Video, "later000000").shorts_probed = True + db.commit() + assert takeout.apply_waiting(db) == {"takeout_watched": 2} + + assert _states(db, a)["later000000"].watched_at == T0 + assert _states(db, b)["later000000"].status == "watched" + assert _rows(db, a)["notyet00000"].outcome is None + assert takeout.apply_waiting(db) == {} + + +def test_a_discovery_video_is_not_marked_until_the_catalog_keeps_it(db, make_user): + """A mark is a VideoState, which the cleanup jobs read as a reader keeping the video: on a video + only someone's search or explore brought in, it kept that video (and its channel) for good.""" + user = make_user("discover@example.com") + for vid, flag in (("search00000", "via_search"), ("explore0000", "via_explore")): + _video(db, vid) + db.flush() + setattr(db.get(Video, vid), flag, True) + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {v: (T0, None) for v in ("search00000", "explore0000", "plain000000")}, + ) + db.commit() + + assert takeout.apply_waiting(db) == {"takeout_watched": 1} + assert set(_states(db, user)) == {"plain000000"} + assert {v for v, r in _rows(db, user).items() if r.outcome is None} == { + "search00000", + "explore0000", + } + + # Subscribed to: the catalog keeps the channel's videos of its own, the search one included. + db.add(Subscription(user_id=user.id, channel_id=CHANNEL)) + db.get(Video, "explore0000").via_explore = False # as subscribing does (catalog.service) + db.commit() + assert takeout.apply_waiting(db) == {"takeout_watched": 2} + assert set(_states(db, user)) == {"plain000000", "search00000", "explore0000"} + + +def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_user): + user = make_user("job-shorts@example.com") + _video(db, "ready000000") # probed already: the job's probe has nothing left to do + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"ready000000": (T0, None)}) + db.commit() + + scheduler._shorts_job() + + db.expire_all() + run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one() + assert run.result == {"probed": 0, "shorts": 0, "takeout_watched": 1} + assert _states(db, user)["ready000000"].status == "watched" + + +# --- what the settings page reads -------------------------------------------------------------- + + +def test_the_status_is_the_users_own_last_run_and_counts(db, make_user, as_user): + user, other = make_user("status@example.com"), make_user("other@example.com") + assert as_user(user).get("/api/takeout").json() == { + "queued": False, + "max_bytes": takeout.MAX_BYTES, + "run": None, + "watched": {}, + "watch_later": {}, + } + _video(db, "plain000000") + _video(db, "short000000", short=True) + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {v: (T0, None) for v in ("plain000000", "short000000", "unknown0000")}, + ) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"plain000000": (T0, None)}) + service.stage_takeout(db, other, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + service.apply_takeout_watched(db, user) + db.commit() + older = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}") + jobruns.finish(older, RunStatus.OK, summary=None, result={"watched": 0}) + newest = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}") + jobruns.finish(newest, RunStatus.OK, summary=None, result={"watched": 1}) + theirs = jobruns.start("takeout_import", RunTrigger.AD_HOC, other.id, f"user:{other.id}") + jobruns.finish(theirs, RunStatus.OK, summary=None, result={"watched": 99}) + + body = as_user(user).get("/api/takeout").json() + + assert body["run"]["status"] == "ok" + assert body["run"]["result"] == {"watched": 1} + assert body["run"]["finished_at"] is not None + assert body["watched"] == {"watched": 1, "short": 1, "waiting": 1} + assert body["watch_later"] == {"waiting": 1} diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index 9bce2f17..c189977e 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -44,6 +44,8 @@ UNIT_MODELS = { "PlaylistItem", "SavedView", "ComfortPin", + "TakeoutItem", + "VideoUnmark", "Tag", "ChannelTag", }, @@ -360,6 +362,11 @@ UNIT_COMMITS = { "sync/playlists.py:pull_user_playlists", "sync/playlists.py:merge_playlist", "sync/playlists.py:repull_playlist", + # The Takeout import is a recorded run with its own session: the watched part lands before + # the Watch later part asks YouTube (which commits the videos it fetched). + "reader/takeout.py:run_import", + # Run inside the Shorts job, on its session: each user's waiting rows land on their own. + "reader/takeout.py:apply_waiting", }, # The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page # fetched and stored is not lost when the next call fails, and an enrich that fails rolls back diff --git a/frontend/e2e/global.setup.ts b/frontend/e2e/global.setup.ts index a7942a82..2c68e31a 100644 --- a/frontend/e2e/global.setup.ts +++ b/frontend/e2e/global.setup.ts @@ -1,9 +1,9 @@ import { test as setup, expect, type Page } from "@playwright/test"; import { execFileSync } from "node:child_process"; -import { mkdirSync, readFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { MEMBER_STORAGE } from "./helpers"; +import { MEMBER_STORAGE, SEED_FILE } from "./helpers"; import { retryConnect } from "./retryConnect"; /** @@ -99,6 +99,8 @@ setup("seed + authenticate", async ({ page, browser }) => { { input: readFileSync(SEED_SCRIPT) }, ); console.log(`[e2e seed] ${out.toString().trim()}`); + // What the seed chose, for the specs that need one of its videos (helpers.seeded). + writeFileSync(SEED_FILE, out); } catch (err: unknown) { const e = err as { stderr?: Buffer; message?: string }; throw new Error(`E2E seed failed: ${e.stderr?.toString() || e.message}`); diff --git a/frontend/e2e/helpers.ts b/frontend/e2e/helpers.ts index 8abc629c..e4ddef32 100644 --- a/frontend/e2e/helpers.ts +++ b/frontend/e2e/helpers.ts @@ -1,8 +1,18 @@ +import { readFileSync } from "node:fs"; import { expect, type Page } from "@playwright/test"; /** The plain member's saved session (global.setup.ts signs it in); `member.spec.ts` and * `devices.spec.ts` run from it. */ export const MEMBER_STORAGE = "e2e/.auth/member.json"; +// The seed's own answer (scripts/e2e_seed.py), written by global.setup: the videos it chose. +export const SEED_FILE = "e2e/.auth/seed.json"; + +/** The videos the seed chose for the main account under `key` in its answer. */ +export function seeded(key: string): string[] { + const ids = (JSON.parse(readFileSync(SEED_FILE, "utf-8")) as Record)[key]; + if (!Array.isArray(ids) || !ids.length) throw new Error(`the seed chose no ${key} videos`); + return ids as string[]; +} /** Open the app and wait for the authenticated shell (the nav rail) to be ready. */ export async function openApp(page: Page, path = "/"): Promise { diff --git a/frontend/e2e/takeout.spec.ts b/frontend/e2e/takeout.spec.ts new file mode 100644 index 00000000..a42c0d25 --- /dev/null +++ b/frontend/e2e/takeout.spec.ts @@ -0,0 +1,59 @@ +import { test, expect } from "@playwright/test"; +import { openApp, gotoPage, seeded } from "./helpers"; + +// Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real +// picker, is read, imported in the background, and the section says what it did — a video the +// catalog knows is marked watched, one it does not know is remembered for later. The seed clears +// the account's kept Takeout videos each run, but the projects share one account: a later one's +// import of the same video adds nothing new, so what each adds is counted against what was there +// before it. +test("a Takeout watch history uploads, imports, and the section says what it did", async ({ + page, +}) => { + await openApp(page); + await gotoPage(page, "settings"); + await page.getByRole("tab", { name: "Account" }).click(); + await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible(); + // The first project's upload is an import already made: its "New import" opens the drop zone. + const zone = page.getByRole("button", { name: "Drop the files here, or choose them" }); + const again = page.getByRole("button", { name: "New import" }); + await expect(zone.or(again)).toBeVisible(); + if (await again.isVisible()) await again.click(); + await expect(zone).toBeVisible(); + const before = (await (await page.request.get("/api/takeout")).json()) as { + watched?: Record; + }; + const watchedBefore = before.watched?.watched ?? 0; + // A video the catalog knows and the Shorts probe has cleared, chosen by the seed (one not probed + // yet would wait for the probe): the next one no earlier project of this run has imported. + const known = seeded("takeout_known")[watchedBefore]; + expect(known, "the seed chose too few videos for every project").toBeTruthy(); + + // Watched now: later than any state the seed set, which the import would otherwise leave alone. + const now = new Date().toISOString(); + const history = [ + { titleUrl: `https://www.youtube.com/watch?v=${known}`, time: now }, + { titleUrl: "https://www.youtube.com/watch?v=e2eUnknown0", time: now }, + { + titleUrl: "https://www.youtube.com/watch?v=e2eAdvert00", + time: now, + details: [{ name: "From Google Ads" }], + }, + ]; + const uploaded = page.waitForResponse( + (r) => r.url().endsWith("/api/takeout") && r.request().method() === "POST", + ); + await page.locator('input[type="file"][accept=".json,.csv"]').setInputFiles({ + name: "watch-history.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(history)), + }); + expect((await uploaded).ok()).toBeTruthy(); + + await expect(page.getByText("watch-history.json: 2 videos read")).toBeVisible(); + const line = page.getByTestId("takeout-last"); + await expect(line).toContainText("Last import:"); + await expect(line).toContainText(`${watchedBefore + 1} watched`); + await expect(line).toContainText("1 remembered"); + await expect(page.getByRole("button", { name: "New import" })).toBeVisible(); +}); diff --git a/frontend/src/components/ui/DropZone.dom.test.tsx b/frontend/src/components/ui/DropZone.dom.test.tsx new file mode 100644 index 00000000..affa0212 --- /dev/null +++ b/frontend/src/components/ui/DropZone.dom.test.tsx @@ -0,0 +1,64 @@ +// @vitest-environment jsdom +// +// ui/DropZone: a drop or a pick hands over the files (only the first unless `multiple`), the drag +// highlight survives the cursor crossing the zone's own children, and a disabled zone takes nothing. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { DropZone } from "./DropZone"; + +afterEach(cleanup); + +const files = (...names: string[]) => names.map((n) => new File(["x"], n)); +function mount(props: Partial[0]> = {}) { + const onFiles = vi.fn(); + render( + + {(over) => {over ? "over" : "idle"}} + , + ); + return { onFiles, zone: screen.getByRole("button"), inner: screen.getByTestId("inner") }; +} + +describe("DropZone", () => { + it("a drop hands over the first file, or all of them when multiple", () => { + const one = mount(); + fireEvent.drop(one.zone, { dataTransfer: { files: files("a", "b") } }); + expect(one.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a"]); + cleanup(); + const many = mount({ multiple: true }); + fireEvent.drop(many.zone, { dataTransfer: { files: files("a", "b") } }); + expect(many.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a", "b"]); + }); + + it("a pick hands over the picked files; an empty pick hands over nothing", () => { + const { onFiles } = mount({ multiple: true }); + const input = document.querySelector('input[type="file"]')!; + expect(input.multiple).toBe(true); + fireEvent.change(input, { target: { files: [] } }); + expect(onFiles).not.toHaveBeenCalled(); + fireEvent.change(input, { target: { files: files("c") } }); + expect(onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["c"]); + }); + + it("stays highlighted while the cursor crosses its own children, and clears on the last leave", () => { + const { zone, inner } = mount(); + fireEvent.dragEnter(zone); + fireEvent.dragEnter(inner); // into a child: the zone gets its leave after this enter + fireEvent.dragLeave(zone); + expect(inner.textContent).toBe("over"); + fireEvent.dragLeave(inner); + expect(inner.textContent).toBe("idle"); + }); + + it("a disabled zone takes no drop", () => { + const { zone, onFiles } = mount({ disabled: true }); + fireEvent.drop(zone, { dataTransfer: { files: files("a") } }); + expect(onFiles).not.toHaveBeenCalled(); + }); + + it("an invalid zone reads as invalid and points at the reason", () => { + const { zone } = mount({ invalid: true, describedBy: "why" }); + expect(zone.getAttribute("aria-invalid")).toBe("true"); + expect(zone.getAttribute("aria-describedby")).toBe("why"); + }); +}); diff --git a/frontend/src/components/ui/DropZone.tsx b/frontend/src/components/ui/DropZone.tsx new file mode 100644 index 00000000..65a6130c --- /dev/null +++ b/frontend/src/components/ui/DropZone.tsx @@ -0,0 +1,101 @@ +import { useRef, useState, type ReactNode, type Ref } from "react"; +import clsx from "clsx"; +import { Pressable } from "./Pressable"; + +// A place to drop files onto, which is also the button that opens the native file dialog: the +// dashed surface an upload starts from. It owns what each such surface needed and the first one +// got right only after a flicker: dragenter/dragleave fire as the cursor crosses the zone's OWN +// children (an icon, a line of text), so a plain boolean flickers — enter and leave are counted +// and the highlight clears at depth 0. The caller draws the inside, told whether a drag is over it. +export function DropZone({ + ref, + onFiles, + multiple = false, + accept, + disabled = false, + invalid = false, + describedBy, + className, + children, +}: { + /** The zone's button, for a caller that moves the focus to it. */ + ref?: Ref; + /** The files dropped or picked, never empty. One unless `multiple`. */ + onFiles: (files: [File, ...File[]]) => void; + multiple?: boolean; + /** The picker's filter (``); a drop is not filtered by it. */ + accept?: string; + disabled?: boolean; + /** What the zone holds is refused: the border turns danger and `describedBy` says why. */ + invalid?: boolean; + describedBy?: string; + /** Lays the zone out (its padding); the border and the drag paint are the zone's. */ + className?: string; + children: (dragOver: boolean) => ReactNode; +}) { + const [dragOver, setDragOver] = useState(false); + const depth = useRef(0); + const inputRef = useRef(null); + const take = (list: FileList | null | undefined) => { + const [first, ...rest] = [...(list ?? [])]; + if (first) onFiles(multiple ? [first, ...rest] : [first]); + }; + return ( + <> + inputRef.current?.click()} + onDragEnter={(e) => { + e.preventDefault(); + depth.current += 1; + setDragOver(true); + }} + onDragOver={(e) => e.preventDefault()} + onDragLeave={() => { + depth.current = Math.max(0, depth.current - 1); + if (depth.current === 0) setDragOver(false); + }} + onDrop={(e) => { + e.preventDefault(); + depth.current = 0; + setDragOver(false); + if (!disabled) take(e.dataTransfer.files); + }} + disabled={disabled} + // The refusal is tied to the control it invalidates, not left to a one-shot announcement: + // the zone reads as invalid and points at the sentence, so a reader who tabs back to it + // hears WHY. (Measured, because `aria-invalid` is not in `button`'s ARIA-supported list: + // Chromium's accessibility node for a native button does carry `invalid` from it.) The + // signal that is NOT colour is the caller's sentence — the border only reinforces it. + aria-invalid={invalid} + aria-describedby={describedBy} + // `dragOver` outranks `invalid`: a drag is transient and the refusal repaints the instant + // the drop lands, whereas a zone that stays red under the pointer tells the user their drop + // target is dead — in the one state where they are trying to replace the file. + className={clsx( + "w-full flex flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed transition disabled:opacity-60", + dragOver + ? "border-accent bg-accent/5" + : invalid + ? "border-danger" + : "border-border hover:border-fg/40", + className, + )} + > + {children(dragOver)} + + { + take(e.target.files); + e.target.value = ""; // let re-picking the same file fire onChange again + }} + /> + + ); +} diff --git a/frontend/src/components/ui/README.md b/frontend/src/components/ui/README.md index 2253e6a0..8dd13ac4 100644 --- a/frontend/src/components/ui/README.md +++ b/frontend/src/components/ui/README.md @@ -35,13 +35,13 @@ Counter-examples, deliberately NOT here: `PosterTile` (knows about Plex montage ## What lives here -| area | components | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| overlays & anchoring | `AnchoredPopover` + `anchoredPopoverHooks` (the popover engine: arrows, typeahead, Escape, anchor-flip), `ActionMenu`, `MembershipMenu` (the body of a "which sets is it in" menu: checkbox rows + a field naming a new set), `ContextMenu` + `useContextMenu` (the SAME menu, opened by right-click / long-press / Shift+F10 / a row's `MoreButton` "…" via `buttonProps`; `anchor` for a panel a row opens; `PageMenu` for a page's background menu), `Modal` (+ `ModalFooter`; `initialFocus` by purpose, C4), `Overlay`, `SidePanel`, `Tooltip`, `Toaster` (+ `toastLevels`, `overlayOrder`; `lib/toastDock` docks one control under the stack, in every host), `CommandPalette`, `KeyboardHelp`, `ConfirmProvider` + `confirmContext` (`useConfirm` / `useChoice`: the app's promise-based confirm, mounted once at the root; a decision with more than two outcomes is one `useChoice`, never two chained confirms), `dialogFocus` (the focus half of the dialog contract: `Modal`'s, and the wizard's own shell), `menuGroups` (one menu from its groups, a rule only between non-empty ones), `pageMenuSlot` + `pageMenuItems` (a page's background menu, mirroring its own toolbar) | -| actions & inputs | `Button`, `IconButton`, `Pressable` (`selectable` when its text is a title or a name), `Chip` + `chipStyles`, `DragHandle` + `OrderButtons` / `useOrderItems` (the non-drag reorder, WCAG 2.5.7) (one per button family, docs/DESIGN.md; a raw ` )} {isTokenExpiring && ( diff --git a/frontend/src/modules/settings/SettingsPanel.tsx b/frontend/src/modules/settings/SettingsPanel.tsx index a4ef2711..5a825d49 100644 --- a/frontend/src/modules/settings/SettingsPanel.tsx +++ b/frontend/src/modules/settings/SettingsPanel.tsx @@ -19,6 +19,7 @@ import { import { DownloadFormats } from "./ProfileEditor"; import DiagnosticsSettings from "./DiagnosticsSettings"; import ApiTokens from "./ApiTokens"; +import TakeoutImport from "./TakeoutImport"; import { type SettingsTabId } from "../../lib/settingsTab"; import { SCHEMES, type Scheme } from "../../lib/theme"; import { api, HttpError, type Me } from "../../lib/api"; @@ -870,6 +871,8 @@ function Account({ me, onOpenWizard }: { me: Me; onOpenWizard: () => void }) { ) : null} + +

{t("settings.account.deleteHint")} diff --git a/frontend/src/modules/settings/TakeoutImport.dom.test.tsx b/frontend/src/modules/settings/TakeoutImport.dom.test.tsx new file mode 100644 index 00000000..88701b94 --- /dev/null +++ b/frontend/src/modules/settings/TakeoutImport.dom.test.tsx @@ -0,0 +1,323 @@ +// @vitest-environment jsdom +// +// Settings › Account › YouTube history: the Google Takeout files are dropped (or picked) and sent one +// request each, every file's outcome said on its own; the section then says what the import did — +// only the counts it produced, history first — or that it is running, failed or was cut off. Before +// the first import the drop zone is open; after one, "New import" opens it. i18n is key-passthrough +// (a key's options ride along in brackets). +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { api, HttpError, type TakeoutStatus, type TakeoutUpload } from "../../lib/api"; + +const mocks = vi.hoisted(() => ({ notify: vi.fn(), notifyActionError: vi.fn() })); +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal()), + useTranslation: () => ({ + i18n: { language: "en" }, + t: (key: string, o?: Record) => + o && Object.keys(o).length ? `${key}(${Object.values(o).join(",")})` : key, + }), +})); +vi.mock("../../lib/notifications", async (importOriginal) => ({ + ...(await importOriginal()), + notify: mocks.notify, +})); +vi.mock("../../lib/actionErrors", () => ({ notifyActionError: mocks.notifyActionError })); +vi.mock("../../lib/format", async (importOriginal) => ({ + ...(await importOriginal()), + formatDateTime: (iso: string) => `at ${iso}`, +})); + +import TakeoutImport from "./TakeoutImport"; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + mocks.notify.mockClear(); + mocks.notifyActionError.mockClear(); +}); + +const run = (status: string, extra: Partial> = {}) => ({ + status: status as NonNullable["status"], + started_at: "S", + finished_at: status === "running" || status === "interrupted" ? null : "F", + result: null, + failure: null, + ...extra, +}); +const uploaded = (videos: number): TakeoutUpload => ({ + ok: true, + kind: "watched", + videos, + entries: videos, + ads: 0, + other: 0, + malformed: 0, +}); + +// A whole status: nothing queued and room for any file, unless the test says otherwise. +const st = (s: Partial): TakeoutStatus => ({ + queued: false, + max_bytes: 1_000_000, + run: null, + ...s, +}); + +function mount(status: TakeoutStatus | Error | Promise) { + const spy = vi.spyOn(api, "takeoutStatus"); + if (status instanceof Error) spy.mockRejectedValue(status); + else if (status instanceof Promise) spy.mockReturnValue(status); + else spy.mockResolvedValue(status); + render( + + + , + ); + return spy; +} + +const zone = () => screen.queryByRole("button", { name: /settings\.takeout\.drop/ }); +const line = () => screen.getByTestId("takeout-last").textContent; +const drop = (files: File[]) => + act(async () => { + fireEvent.drop(zone()!, { dataTransfer: { files } }); + }); +const file = (name: string, size = 2) => new File(["x".repeat(size)], name); +const unavailable = (el: HTMLElement | null) => + !!el && ((el as HTMLButtonElement).disabled || el.getAttribute("aria-disabled") === "true"); + +describe("YouTube history import · what it says", () => { + it("before the first import: the drop zone, and no line about an import", async () => { + mount(st({ run: null, watched: {}, watch_later: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + expect(screen.queryByTestId("takeout-last")).toBeNull(); + }); + + it("not while the last import is still being asked for", () => { + mount(new Promise(() => {})); + expect(zone()).toBeNull(); + }); + + it("after one: when, then only the counts it produced, the history's first", async () => { + mount( + st({ + run: run("ok"), + watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 }, + watch_later: { added: 41, waiting: 2 }, + }), + ); + await waitFor(() => expect(screen.getByTestId("takeout-last")).toBeTruthy()); + expect(line()).toBe( + [ + "settings.takeout.last(at F)", + "settings.takeout.count.watched(1020,1,020)", + "settings.takeout.count.waiting(6785,6,785)", + "settings.takeout.count.kept(81,81)", + "settings.takeout.count.added(41,41)", + "settings.takeout.count.laterWaiting(2,2)", + ].join(" · "), + ); + expect(zone()).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + expect(zone()).toBeTruthy(); + }); + + it("a failed run and one a restart cut off say so, not counts", async () => { + mount(st({ run: run("error"), watched: { watched: 3 } })); + await waitFor(() => expect(line()).toBe("settings.takeout.failed(at F)")); + cleanup(); + mount(st({ run: run("interrupted"), watched: { watched: 3 } })); + await waitFor(() => expect(line()).toBe("settings.takeout.interrupted(at S)")); + }); + + it("a running import: the running line, the zone unavailable, asked again until it ends", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); + await waitFor(() => + expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"), + ); + expect(unavailable(zone())).toBe(true); + spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 5 } })); + await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(5,5)"), { + timeout: 3000, + }); + }); + + it("a status that could not be read says so, with a retry — not an empty section", async () => { + mount(new HttpError(500, "boom")); + await waitFor(() => + expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"), + ); + expect(zone()).toBeNull(); + expect(screen.queryByTestId("takeout-last")).toBeNull(); + }); + + it("a re-read that fails does not leave the previous answer standing under it", async () => { + const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => expect(line()).toContain("OLD")); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + status.mockRejectedValue(new HttpError(500, "boom")); + await drop([file("watch-history.json")]); + await waitFor(() => + expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"), + ); + expect(screen.queryByTestId("takeout-last")).toBeNull(); + }); +}); + +describe("YouTube history import · the upload", () => { + it("sends each dropped file on its own, says each outcome, and reads the new run", async () => { + const status = mount(st({ run: null, watched: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + const refused = new HttpError(422, "refused"); + const sent: string[] = []; + vi.spyOn(api, "takeoutUpload").mockImplementation(async (f: File) => { + sent.push(f.name); + if (f.name === "bad.html") throw refused; + return uploaded(f.name === "watch-history.json" ? 9034 : 56); + }); + status.mockResolvedValue(st({ run: run("ok"), watched: { watched: 7 } })); + await drop([file("watch-history.json"), file("bad.html"), file("Watch later-videos.csv")]); + + await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(7,7)")); + expect(sent).toEqual(["watch-history.json", "bad.html", "Watch later-videos.csv"]); + expect(mocks.notify.mock.calls.map(([n]) => (n as { message: string }).message)).toEqual([ + "settings.takeout.read(watch-history.json,9034,9,034)", + "settings.takeout.read(Watch later-videos.csv,56,56)", + ]); + expect(mocks.notifyActionError).toHaveBeenCalledWith( + refused, + "errors.writeFailed.takeoutUpload", + ); + }); + + it("names the file being sent, then never shows the previous import's line before the new one", async () => { + const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => expect(line()).toContain("OLD")); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + let finish!: (v: TakeoutUpload) => void; + vi.spyOn(api, "takeoutUpload").mockReturnValue(new Promise((r) => (finish = r))); + let answer!: (v: TakeoutStatus) => void; + status.mockReturnValue(new Promise((r) => (answer = r))); + await drop([file("watch-history.json")]); + expect(screen.getByTestId("takeout-running").textContent).toBe( + "settings.takeout.uploading(watch-history.json)", + ); + await act(async () => finish(uploaded(1))); + // The new run is still being read: the upload's line stays, the old import's does not return. + expect(screen.queryByTestId("takeout-last")).toBeNull(); + await act(async () => answer(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }))); + await waitFor(() => expect(line()).toContain("NEW")); + }); +}); + +describe("YouTube history import · while it runs", () => { + it("a queued import is this upload's, not the last one: the running line until its run reads", async () => { + const spy = mount(st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => + expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"), + ); + expect(screen.queryByTestId("takeout-last")).toBeNull(); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + }); + + it("a re-read that fails mid-run does not leave the spinner claiming progress under it", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); + await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy()); + spy.mockRejectedValue(new HttpError(500, "boom")); + await waitFor( + () => expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"), + { timeout: 3000 }, + ); + expect(screen.queryByTestId("takeout-running")).toBeNull(); + spy.mockResolvedValue(st({ run: run("ok"), watched: {} })); + await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 3000 }); + }); + + it("one live region says the run and then its outcome", async () => { + const spy = mount(st({ run: run("running"), watched: {} })); + await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy()); + const region = screen.getByRole("status"); + spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 2 } })); + await waitFor(() => expect(line()).toContain("watched(2,2)"), { timeout: 3000 }); + expect(screen.getByRole("status")).toBe(region); + expect(region.contains(screen.getByTestId("takeout-last"))).toBe(true); + }); + + it("a file dropped beside the zone is kept from the browser, which would open it", async () => { + mount(st({ run: null, watched: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + const missed = (type: string, types: string[]) => { + const e = new Event(type, { bubbles: true, cancelable: true }); + Object.defineProperty(e, "dataTransfer", { value: { types, dropEffect: "copy" } }); + document.body.dispatchEvent(e); + return e; + }; + expect(missed("drop", ["Files"]).defaultPrevented).toBe(true); + expect(missed("dragover", ["Files"]).defaultPrevented).toBe(true); + // A drag of something else (a link, text) is the page's own business. + expect(missed("drop", ["text/plain"]).defaultPrevented).toBe(false); + cleanup(); + expect(missed("drop", ["Files"]).defaultPrevented).toBe(false); + }); + + it("a file larger than the import takes is refused before it is sent, naming the limit", async () => { + mount(st({ max_bytes: 10, run: run("ok"), watched: {} })); + await waitFor(() => expect(line()).toBeTruthy()); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + const send = vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + await drop([file("watch-history.json", 11)]); + expect(send).not.toHaveBeenCalled(); + expect(mocks.notify).toHaveBeenCalledWith({ + level: "error", + message: "settings.takeout.tooLarge(watch-history.json,10 B)", + }); + // Nothing was taken: the zone stays open for the right file, and so after a refusal. + await waitFor(() => expect(zone()).toBeTruthy()); + send.mockRejectedValue(new HttpError(422, "refused")); + await drop([file("bad.html")]); + await waitFor(() => expect(mocks.notifyActionError).toHaveBeenCalled()); + expect(zone()).toBeTruthy(); + }); + + it("the focus follows: New import hands it to the zone, the zone back to New import", async () => { + const spy = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} })); + await waitFor(() => expect(line()).toContain("OLD")); + fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" })); + expect(document.activeElement).toBe(zone()); + + vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + spy.mockResolvedValue( + st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }), + ); + await drop([file("watch-history.json")]); + // While it runs the zone stays, unavailable, and keeps the focus. + expect(document.activeElement).toBe(zone()); + expect(unavailable(zone())).toBe(true); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + expect(zone()).toBeNull(); + expect(document.activeElement).toBe( + screen.getByRole("button", { name: "settings.takeout.newImport" }), + ); + }); + + it("a focus the user moved elsewhere meanwhile is not taken back", async () => { + const spy = mount(st({ run: null, watched: {} })); + await waitFor(() => expect(zone()).toBeTruthy()); + const elsewhere = document.body.appendChild(document.createElement("button")); + zone()!.focus(); + vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1)); + spy.mockResolvedValue(st({ queued: true, run: null, watched: {} })); + await drop([file("watch-history.json")]); + elsewhere.focus(); + spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })); + await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 }); + expect(document.activeElement).toBe(elsewhere); + elsewhere.remove(); + }); +}); diff --git a/frontend/src/modules/settings/TakeoutImport.tsx b/frontend/src/modules/settings/TakeoutImport.tsx new file mode 100644 index 00000000..9bab8821 --- /dev/null +++ b/frontend/src/modules/settings/TakeoutImport.tsx @@ -0,0 +1,252 @@ +import { useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { RefreshCw, UploadCloud } from "lucide-react"; +import { Section } from "../../components/ui/form"; +import { Button } from "../../components/ui/Button"; +import { DropZone } from "../../components/ui/DropZone"; +import { StateMessage } from "../../components/ui/QueryState"; +import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api"; +import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed"; +import { qk } from "../../lib/queryKeys"; +import { notify } from "../../lib/notifications"; +import { notifyActionError } from "../../lib/actionErrors"; +import { formatBytes, formatDateTime, formatNumber } from "../../lib/format"; +import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch"; + +// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and +// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one +// request each; the server tells them apart and runs the import in the background, and its outcome +// also reaches the inbox. The import has no steps to count (the whole history is applied in one +// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`). + +// The outcome counts, in the order they are said: the history's, then Watch later's. A count the +// import did not produce is not said at all. +const HISTORY = [ + ["watched", "watched"], + ["waiting", "waiting"], + ["short", "short"], + ["kept", "kept"], +] as const; +const WATCH_LATER = [ + ["added", "added"], + ["present", "present"], + ["missing", "missing"], + ["waiting", "laterWaiting"], +] as const; + +// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it). +type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number]; +type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0]; +const everyOutcomeSaid: [Exclude] extends [never] ? true : never = true; +void everyOutcomeSaid; + +function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] { + const parts: string[] = []; + for (const [outcome, key] of HISTORY) { + const n = s.watched?.[outcome] ?? 0; + if (n) parts.push(say(key, n)); + } + for (const [outcome, key] of WATCH_LATER) { + const n = s.watch_later?.[outcome] ?? 0; + if (n) parts.push(say(key, n)); + } + return parts; +} + +export default function TakeoutImport() { + const { t, i18n } = useTranslation(); + const lang = i18n.language; + const qc = useQueryClient(); + const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus }); + const [uploading, setUploading] = useState(null); + // "New import" reopens the drop zone over a finished one; before the first import it is open. + const [reopened, setReopened] = useState(false); + const run = status.data?.run ?? null; + // Queued counts too: until its run row is written, `run` is still the import before it. + const busy = importBusy(status.data); + const running = uploading !== null || busy; + const finishedAt = run?.finished_at ?? null; + + // A run under way is watched to its end, even if the user leaves Settings meanwhile. + useEffect(() => { + if (busy) void watchTakeout(qc); + }, [busy, qc]); + // One that ended while nothing here watched it (a reload, another tab) has changed what the feed + // and Watch later show too: those lists are read again when next opened, not from the cache. + useEffect(() => { + if (finishedAt) invalidateImported(qc); + }, [finishedAt, qc]); + + // A file dropped beside the zone, or while it is not there, is kept from the browser: it would + // open the file in the tab, unloading the app and the upload under way. The zone takes its own. + useEffect(() => { + const missed = (e: DragEvent) => { + if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return; + e.preventDefault(); + if (e.type === "dragover") e.dataTransfer.dropEffect = "none"; + }; + window.addEventListener("dragover", missed); + window.addEventListener("drop", missed); + return () => { + window.removeEventListener("dragover", missed); + window.removeEventListener("drop", missed); + }; + }, []); + + // Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and + // the zone, gone once the import it started is done, hands it to "New import". + const zone = useRef(null); + const focusNext = useRef<"zone" | "newImport" | null>(null); + const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => { + if (!el || focusNext.current !== which) return; + focusNext.current = null; + // Only focus that was lost with the control that held it: never taken from where the user went. + if (document.activeElement === null || document.activeElement === document.body) el.focus(); + }; + const zoneRef = (el: HTMLButtonElement | null) => { + zone.current = el; + focusIfNext("zone", el); + }; + const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el); + + const upload = async (files: File[]) => { + if (document.activeElement === zone.current) focusNext.current = "newImport"; + let taken = false; + for (const file of files) if (await uploadOne(file)) taken = true; + // The new run read first, then the upload's line let go: the other way round the previous + // import's line showed for a moment, as if this upload had done nothing. + await qc.invalidateQueries({ queryKey: qk.takeout() }); + setUploading(null); + // Nothing taken, nothing imported: the zone stays open for the right file. + if (taken) setReopened(false); + else focusNext.current = null; + }; + /** Send one file; whether the server took it. */ + const uploadOne = async (file: File): Promise => { + const max = status.data?.max_bytes ?? Infinity; + if (file.size > max) { + // Refused here, not after minutes of upload, and with the limit named. + notify({ + level: "error", + message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }), + }); + return false; + } + setUploading(file.name); + let res: TakeoutUpload | null = null; + try { + res = await api.takeoutUpload(file); + } catch (e) { + // A refusal names its reason (not a Takeout file, nothing in it); the next file still goes. + notifyActionError(e, t("errors.writeFailed.takeoutUpload")); + } + if (res === null) return false; + notify({ + level: "success", + message: t("settings.takeout.read", { + name: file.name, + count: res.videos, + n: formatNumber(res.videos, lang), + }), + }); + return true; + }; + + const say = (key: string, n: number) => + t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) }); + const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : ""); + // Not while the last import is still being asked for: a zone that flashes before the line that + // replaces it would offer a first import to someone who has done one. While an import runs the + // zone stays, unavailable: the control that started it keeps the focus. + const showZone = status.isSuccess && (running || run === null || reopened); + // A failed re-read keeps the previous answer: it is not shown under the line saying the read + // failed, as if it were current — a run's spinner no more than its outcome. + const current = !status.isError || uploading !== null; + + return ( +

+

{t("settings.takeout.intro")}

+ + {status.isError && ( + void status.refetch()}> + {t("settings.takeout.loadFailed")} + + )} + + {showZone && ( + <> + + {(dragOver) => ( + <> + + + {dragOver ? t("settings.takeout.dropActive") : t("settings.takeout.drop")} + + + )} + +
+ + {t("settings.takeout.howTo")} + +

+ {t("settings.takeout.howToBody")} +

+
+ + )} + +
+ {/* One live region for the run and its outcome, mounted throughout: what replaces + "Processing…" is announced, which a region mounted with its text is not. */} +
+ {running && current ? ( + + + {uploading !== null + ? t("settings.takeout.uploading", { name: uploading }) + : t("settings.takeout.running")} + + ) : ( + !running && + run && + status.data && + current && ( +

+ {run.status === "error" + ? t("settings.takeout.failed", { when: when(run.finished_at) }) + : run.status === "interrupted" + ? t("settings.takeout.interrupted", { when: when(run.started_at) }) + : [ + t("settings.takeout.last", { when: when(run.finished_at) }), + ...countParts(status.data, say), + ].join(" · ")} +

+ ) + )} +
+ {!running && run && status.data && current && !reopened && ( + + )} +
+
+ ); +} diff --git a/frontend/src/modules/settings/takeoutWatch.test.ts b/frontend/src/modules/settings/takeoutWatch.test.ts new file mode 100644 index 00000000..c2d435d2 --- /dev/null +++ b/frontend/src/modules/settings/takeoutWatch.test.ts @@ -0,0 +1,67 @@ +// @vitest-environment jsdom +// +// A Takeout import is watched to its end whether or not its section is still on screen, and what it +// changed is read afresh then: the feed and its counts, Watch later, and the playlists' sizes. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { QueryClient } from "@tanstack/react-query"; +import { api, type TakeoutStatus } from "../../lib/api"; +import { qk } from "../../lib/queryKeys"; +import { importBusy, watchTakeout } from "./takeoutWatch"; + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +const st = (s: Partial): TakeoutStatus => ({ + queued: false, + max_bytes: 1, + run: null, + ...s, +}); +const running = { + status: "running", + started_at: "S", + finished_at: null, + result: null, + failure: null, +}; + +describe("the Takeout import's watch", () => { + it("a queued import is busy as much as a running one", () => { + expect(importBusy(st({ queued: true }))).toBe(true); + expect(importBusy(st({ run: { ...running, status: "running" } as TakeoutStatus["run"] }))).toBe( + true, + ); + expect(importBusy(st({ run: { ...running, status: "ok" } as TakeoutStatus["run"] }))).toBe( + false, + ); + expect(importBusy(undefined)).toBe(false); + }); + + it("asks until the import is done, then reads again everything it changed", async () => { + vi.useFakeTimers(); + const qc = new QueryClient(); + qc.setQueryData(qk.takeout(), st({ queued: true })); + const answers = [st({ run: running as TakeoutStatus["run"] }), st({})]; + const read = vi.spyOn(api, "takeoutStatus").mockImplementation(async () => answers.shift()!); + const stale = vi.spyOn(qc, "invalidateQueries"); + + const done = watchTakeout(qc); + void watchTakeout(qc); // a second call while it watches is the same watch + await vi.advanceTimersByTimeAsync(1500); + expect(read).toHaveBeenCalledTimes(1); + expect(stale).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1500); + await done; + + expect(read).toHaveBeenCalledTimes(2); + expect(stale.mock.calls.map(([f]) => f?.queryKey)).toEqual([ + qk.feed(), + qk.feedSummary(), + qk.feedYears(), + qk.playlist(), + qk.playlists(), + ]); + }); +}); diff --git a/frontend/src/modules/settings/takeoutWatch.ts b/frontend/src/modules/settings/takeoutWatch.ts new file mode 100644 index 00000000..13855b2f --- /dev/null +++ b/frontend/src/modules/settings/takeoutWatch.ts @@ -0,0 +1,57 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { api, type TakeoutStatus } from "../../lib/api"; +import { qk } from "../../lib/queryKeys"; + +// A Takeout import, watched to its end: its status is asked again until no run is queued or going, +// then the lists it changed are read afresh. It lives outside the section on purpose: the user may +// leave Settings while the import runs, and the feed they go to must not keep showing as new what +// the import has just marked watched. + +const POLL_MS = 1500; + +export const importBusy = (s: TakeoutStatus | undefined): boolean => + !!s && (s.queued || s.run?.status === "running"); + +let watching = false; + +const pause = () => new Promise((done) => setTimeout(done, POLL_MS)); + +async function read(qc: QueryClient): Promise { + try { + return await qc.fetchQuery({ + queryKey: qk.takeout(), + queryFn: api.takeoutStatus, + staleTime: 0, + }); + } catch { + // A read that failed is the section's to say (its query holds the error); the watch asks again. + return qc.getQueryData(qk.takeout()); + } +} + +/** Mark stale everything an import may have changed: the feed and its counts, Watch later, and the + * playlists that show its size. */ +export function invalidateImported(qc: QueryClient): void { + for (const queryKey of [ + qk.feed(), + qk.feedSummary(), + qk.feedYears(), + qk.playlist(), + qk.playlists(), + ]) + void qc.invalidateQueries({ queryKey }); +} + +/** Watch the user's import until it is done (once: a second call while it watches is that watch), + * then mark stale what it changed. */ +export async function watchTakeout(qc: QueryClient): Promise { + if (watching) return; + watching = true; + let status = qc.getQueryData(qk.takeout()); + while (importBusy(status)) { + await pause(); + status = await read(qc); + } + watching = false; + invalidateImported(qc); +} diff --git a/scripts/e2e_seed.py b/scripts/e2e_seed.py index 5d545baf..8fa5236c 100644 --- a/scripts/e2e_seed.py +++ b/scripts/e2e_seed.py @@ -28,7 +28,7 @@ from app.db import SessionLocal import app.tables # noqa: F401 — every table, before the first query from app.catalog.models import Channel, Subscription, Video from app.models import AppState, User -from app.reader.models import Playlist, PlaylistItem, VideoState +from app.reader.models import Playlist, PlaylistItem, TakeoutItem, VideoState from app.security import hash_password EMAIL = os.environ.get("E2E_EMAIL", "e2e@siftlode.local") @@ -118,6 +118,9 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: # - one WATCHED video from the second channel, so the "watched" show-filter is a # real, non-empty narrowing of the feed (and the "all" vs "unwatched" split exists). db.execute(delete(VideoState).where(VideoState.user_id == user.id)) + # A Takeout import's kept videos too: a settled row keeps its outcome over a re-upload, so the + # import spec's counts would otherwise be the previous run's, not this one's. + db.execute(delete(TakeoutItem).where(TakeoutItem.user_id == user.id)) def _newest(channel_id: str, exclude: set[str]) -> Video | None: conds = [ @@ -178,6 +181,28 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: saved = svid.id db.commit() + # 5. Videos the Takeout import marks watched: from the subscribed channels, and already cleared by + # the Shorts probe — one not yet probed would wait for it, and the import spec would count + # nothing. One per browser project: they share this account, and a video imported once is not + # marked again. + known = ( + list( + db.scalars( + select(Video.id) + .where( + Video.channel_id.in_(channel_ids), + Video.shorts_probed.is_(True), + Video.is_short.is_(False), + Video.id.not_in(used | {saved} if saved else used), + ) + .order_by(Video.published_at.desc()) + .limit(5) + ) + ) + if channel_ids + else [] + ) + return { "user_id": user.id, "email": email, @@ -185,6 +210,7 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict: "in_progress": in_progress, "watched": watched, "saved": saved, + "takeout_known": known, }