From 7b8505af955973609afeb73689840008c4715b74 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 23 Oct 2026 17:04:54 +0200 Subject: [PATCH] feat(reader): a Google Takeout export becomes watch state and Watch later The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`). --- .../alembic/versions/0113_takeout_items.py | 51 +++ backend/app/bodylimit.py | 5 + backend/app/reader/models.py | 49 +++ backend/app/reader/queries.py | 69 +++- backend/app/reader/service.py | 122 +++++- backend/app/reader/takeout.py | 200 ++++++++++ backend/app/routes/feed.py | 63 ++- backend/app/scheduler.py | 39 +- backend/tests/test_body_limit.py | 1 + backend/tests/test_takeout.py | 362 ++++++++++++++++++ backend/tests/test_unit_fences.py | 4 + frontend/src/i18n/locales/en/errors.json | 4 +- frontend/src/i18n/locales/en/scheduler.json | 17 +- frontend/src/i18n/locales/hu/errors.json | 4 +- frontend/src/i18n/locales/hu/scheduler.json | 17 +- 15 files changed, 978 insertions(+), 29 deletions(-) create mode 100644 backend/alembic/versions/0113_takeout_items.py create mode 100644 backend/app/reader/takeout.py create mode 100644 backend/tests/test_takeout.py diff --git a/backend/alembic/versions/0113_takeout_items.py b/backend/alembic/versions/0113_takeout_items.py new file mode 100644 index 00000000..8a901789 --- /dev/null +++ b/backend/alembic/versions/0113_takeout_items.py @@ -0,0 +1,51 @@ +"""A user's Google Takeout export: what they watched on YouTube, and their Watch later list + +Revision ID: 0113_takeout_items +Revises: 0112_one_watch_later +Create Date: 2026-10-23 + +One row per video and kind. A video the catalog does not know yet waits with no outcome, so the id +is not a foreign key; the partial index is what a newly known video looks its waiting rows up by. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0113_takeout_items" +down_revision: str | None = "0112_one_watch_later" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.create_table( + "takeout_items", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "user_id", + sa.Integer(), + sa.ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("kind", sa.String(12), nullable=False), + sa.Column("video_id", sa.String(16), nullable=False), + sa.Column("channel_id", sa.String(32), nullable=True), + sa.Column("at", sa.DateTime(timezone=True), nullable=False), + sa.Column("outcome", sa.String(12), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + sa.CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + ) + op.create_index( + "ix_takeout_items_waiting", + "takeout_items", + ["video_id"], + postgresql_where=sa.text("outcome IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_takeout_items_waiting", table_name="takeout_items") + op.drop_table("takeout_items") diff --git a/backend/app/bodylimit.py b/backend/app/bodylimit.py index c0ecab4c..97b67eba 100644 --- a/backend/app/bodylimit.py +++ b/backend/app/bodylimit.py @@ -72,6 +72,10 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024 # A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why # `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the # default, so listing it could only widen what it accepts. +# A Google Takeout file (`reader.takeout`): a real watch history of 9500 views, the 18 months Google +# keeps by default, is 3.7 MB; four times that is still well inside one request. +TAKEOUT_MAX_BYTES = 16 * 1024 * 1024 + ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # Ingesting a local file. The body IS the file now — no multipart envelope to allow for — # so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s @@ -95,6 +99,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = { # sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports. ("POST", "/api/files/upload"): settings.download_upload_max_bytes, ("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES, + ("POST", "/api/takeout"): TAKEOUT_MAX_BYTES, } # The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders diff --git a/backend/app/reader/models.py b/backend/app/reader/models.py index f55bd23a..e5e3f031 100644 --- a/backend/app/reader/models.py +++ b/backend/app/reader/models.py @@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th edge, or asks this unit's service.""" from datetime import datetime +from enum import StrEnum from sqlalchemy import ( Boolean, @@ -213,3 +214,51 @@ class ComfortPin(Base, TimestampMixin): # 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder # endpoint rewrites wholesale beats a fractional index nobody would ever exercise here. position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False) + + +class TakeoutKind(StrEnum): + """What a Google Takeout row says the user did with a video.""" + + WATCHED = "watched" # history/watch-history.json + WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv + + +class TakeoutOutcome(StrEnum): + """What the import did with a row; NULL = still waiting for its video to be known (and, for a + watched one, probed for Shorts).""" + + WATCHED = "watched" # the video is marked watched at the Takeout time + KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays + SHORT = "short" # a Short: swiping one past is not watching it + ADDED = "added" # added to Watch later + PRESENT = "present" # already in Watch later + MISSING = "missing" # YouTube no longer has the video + + +class TakeoutItem(Base, TimestampMixin): + """One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch + later list (with when it was added). The rows are both the import job's input and its memory: a + video the catalog does not know yet waits here, with no outcome, until it enters the catalog. + The id is not a foreign key for that reason. The channel is kept for a later "channels you + watch but do not follow" list.""" + + __tablename__ = "takeout_items" + __table_args__ = ( + UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"), + CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"), + # The waiting rows, by video: what a newly known video looks up. + Index( + "ix_takeout_items_waiting", + "video_id", + postgresql_where=text("outcome IS NULL"), + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) + kind: Mapped[str] = mapped_column(String(12)) + video_id: Mapped[str] = mapped_column(String(16)) + channel_id: Mapped[str | None] = mapped_column(String(32)) + # Watched: the last time in the export; Watch later: when it was added. + at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + outcome: Mapped[str | None] = mapped_column(String(12)) diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index 453b7e88..ffb7302f 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -1,5 +1,5 @@ -"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and -for the other units that ask about them. +"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP +layer and for the other units that ask about them. Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and @@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t than joining Reader's tables itself. """ +from datetime import datetime + from sqlalchemy import ColumnElement, and_, func, or_, select from sqlalchemy.orm import InstrumentedAttribute, Session, aliased @@ -14,7 +16,16 @@ from app.collation import collated from app.feedquery import feed_columns, serialize from app.models import User from app.catalog.models import Channel, Subscription, Video -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + VideoState, +) from app.userscope import owned_by, owned_row # --- catalog, read along the edge ------------------------------------------------------------ @@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan owned_by(ChannelTag.user_id, user), ) ).scalar_one_or_none() + + +# --- Google Takeout -------------------------------------------------------------------------- + + +def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]: + """The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts: + (row id, video id, Takeout time, is Short).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short) + .join(Video, Video.id == TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == TakeoutKind.WATCHED, + TakeoutItem.outcome.is_(None), + Video.shorts_probed, + ) + ).tuples() + ) + + +def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]: + """The user's rows of `kind` the import has not settled, oldest first: (row id, video id).""" + return list( + db.execute( + select(TakeoutItem.id, TakeoutItem.video_id) + .where( + owned_by(TakeoutItem.user_id, user), + TakeoutItem.kind == kind, + TakeoutItem.outcome.is_(None), + ) + .order_by(TakeoutItem.at, TakeoutItem.id) + ).tuples() + ) + + +def known_video_ids(db: Session, ids: list[str]) -> set[str]: + """Which of `ids` the catalog holds.""" + if not ids: + return set() + return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars()) + + +def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]: + """How many of the user's rows each kind and outcome holds (None = waiting).""" + rows = db.execute( + select(TakeoutItem.kind, TakeoutItem.outcome, func.count()) + .where(owned_by(TakeoutItem.user_id, user)) + .group_by(TakeoutItem.kind, TakeoutItem.outcome) + ).all() + return {(kind, outcome): n for kind, outcome, n in rows} diff --git a/backend/app/reader/service.py b/backend/app/reader/service.py index ca16ab49..7113eaae 100644 --- a/backend/app/reader/service.py +++ b/backend/app/reader/service.py @@ -15,7 +15,17 @@ from sqlalchemy.orm import Session from app.models import User from app.reader import queries -from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState +from app.reader.models import ( + ChannelTag, + Playlist, + PlaylistItem, + SavedView, + Tag, + TakeoutItem, + TakeoutKind, + TakeoutOutcome, + VideoState, +) from app.sync.playlists import lock_playlists, recompute_dirty from app.userscope import owned_by @@ -495,3 +505,113 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No if row is not None: db.delete(row) db.flush() + + +# --- Google Takeout -------------------------------------------------------------------------- + +_TAKEOUT_BATCH = 1000 + + +def stage_takeout( + db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]] +) -> None: + """Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job. + A second upload of the same export, or a newer one, adds what is new: a video already kept keeps + the later time, and a row the import has settled keeps its outcome.""" + values = [ + {"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel} + for vid, (at, channel) in rows.items() + ] + for start in range(0, len(values), _TAKEOUT_BATCH): + stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH]) + db.execute( + stmt.on_conflict_do_update( + constraint="uq_takeout_item", + set_={ + "at": func.greatest(TakeoutItem.at, stmt.excluded.at), + "channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id), + }, + ) + ) + db.flush() + + +def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None: + for start in range(0, len(ids), _TAKEOUT_BATCH): + db.execute( + update(TakeoutItem) + .where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH])) + .values(outcome=outcome) + ) + + +def apply_takeout_watched(db: Session, user: User) -> dict[str, int]: + """Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed + for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one + past is not watching it), and a state the user set here wins: watched or hidden, or anything + changed after the Takeout time. The upsert's own condition is that rule, so a state set while + the import runs wins too.""" + short: list[int] = [] + todo: dict[str, tuple[int, datetime]] = {} + for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user): + if is_short: + short.append(row_id) + else: + todo[video_id] = (row_id, at) + marked: set[str] = set() + items = list(todo.items()) + now = datetime.now(timezone.utc) + for start in range(0, len(items), _TAKEOUT_BATCH): + stmt = pg_insert(VideoState).values( + [ + {"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at} + for vid, (_, at) in items[start : start + _TAKEOUT_BATCH] + ] + ) + marked.update( + db.execute( + stmt.on_conflict_do_update( + constraint="uq_user_video", + set_={ + "status": "watched", + "watched_at": stmt.excluded.watched_at, + "updated_at": now, + }, + where=(VideoState.status == "new") + & (VideoState.updated_at < stmt.excluded.watched_at), + ).returning(VideoState.video_id) + ).scalars() + ) + kept = [row_id for vid, (row_id, _) in items if vid not in marked] + _settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED) + _settle(db, kept, TakeoutOutcome.KEPT) + _settle(db, short, TakeoutOutcome.SHORT) + db.flush() + return {"watched": len(marked), "kept": len(kept), "shorts": len(short)} + + +def apply_takeout_watch_later(db: Session, user: User) -> dict[str, int]: + """Put the videos of the user's YouTube Watch later list into theirs here, oldest first, after + what it already holds. Called once the catalog has fetched what it could: a video still unknown + is one YouTube no longer has.""" + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if not waiting: + return {} + known = queries.known_video_ids(db, [vid for _, vid in waiting]) + pl = watch_later(db, user) + done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome} + for row_id, vid in waiting: + if vid not in known: + done[TakeoutOutcome.MISSING].append(row_id) + elif add_item(db, pl, vid, mark_dirty=False): + done[TakeoutOutcome.ADDED].append(row_id) + else: + done[TakeoutOutcome.PRESENT].append(row_id) + for outcome, ids in done.items(): + _settle(db, ids, outcome) + db.flush() + return { + "watch_later_added": len(done[TakeoutOutcome.ADDED]), + "watch_later_present": len(done[TakeoutOutcome.PRESENT]), + "watch_later_missing": len(done[TakeoutOutcome.MISSING]), + } diff --git a/backend/app/reader/takeout.py b/backend/app/reader/takeout.py new file mode 100644 index 00000000..9a752a1e --- /dev/null +++ b/backend/app/reader/takeout.py @@ -0,0 +1,200 @@ +"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their +YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the +only way in. + +Two files of the export are read, each uploaded on its own: `history/watch-history.json` and +`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies +what it can; a video the catalog does not know yet waits for it. + +The format, measured on a real export (2026-10): the history is a JSON list, one entry per view, +newest first. A view of a video has a `titleUrl` with `watch?v=` (on www.youtube.com or +music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles` +entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google +Ads". The title is in the account's language, so nothing here reads it. A Short is a plain +`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a +header row (in the account's language) and `,` rows. +""" + +import csv +import io +import json +import re +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime + +from app import quota +from app.bodylimit import TAKEOUT_MAX_BYTES +from app.db import SessionLocal +from app.models import User +from app.reader import queries, service +from app.reader.models import TakeoutKind +from app.sync.runner import get_service_user +from app.sync.videos import ensure_videos +from app.youtube.client import YouTubeClient + +# The largest file the upload takes (its edge cap and the route's own are one number). +MAX_BYTES = TAKEOUT_MAX_BYTES + +_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$") +_WATCH_URL = re.compile( + r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})" +) +_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})") + + +class TakeoutError(ValueError): + """The file is not one the import reads; `reason` is the client's word for why.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +@dataclass +class Parsed: + """One file's videos, `{video id: (time, channel id)}`, and what it left out, counted.""" + + kind: TakeoutKind + videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict) + entries: int = 0 + ads: int = 0 + other: int = 0 # not a view of a video (a post, a story) or a video with no link left + malformed: int = 0 + + +def _time(raw: object) -> datetime | None: + if not isinstance(raw, str): + return None + try: + at = datetime.fromisoformat(raw.replace("Z", "+00:00")) + # silent-ok: the row is counted as malformed, and the count is in the upload's answer. + except ValueError: + return None + return at if at.tzinfo is not None else None + + +def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None: + old = parsed.videos.get(vid) + if old is None or at > old[0]: + parsed.videos[vid] = (at, channel or (old[1] if old else None)) + + +def _history(data: list) -> Parsed: + parsed = Parsed(TakeoutKind.WATCHED) + for entry in data: + parsed.entries += 1 + if not isinstance(entry, dict): + parsed.malformed += 1 + continue + details = entry.get("details") + if isinstance(details, list) and any( + isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details + ): + parsed.ads += 1 + continue + url = entry.get("titleUrl") + match = _WATCH_URL.match(url) if isinstance(url, str) else None + if match is None: + parsed.other += 1 + continue + at = _time(entry.get("time")) + if at is None: + parsed.malformed += 1 + continue + channel = None + subtitles = entry.get("subtitles") + if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict): + found = _CHANNEL_URL.search(str(subtitles[0].get("url", ""))) + channel = found.group(1) if found else None + _keep(parsed, match.group(1), at, channel) + return parsed + + +def _watch_later(text: str) -> Parsed: + parsed = Parsed(TakeoutKind.WATCH_LATER) + rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)] + if not rows or len(rows[0]) < 2: + raise TakeoutError("not_takeout") + if not _VIDEO_ID.match(rows[0][0].strip()): + rows = rows[1:] # the header, in the account's language + for row in rows: + parsed.entries += 1 + vid = row[0].strip() if row else "" + at = _time(row[1].strip()) if len(row) > 1 else None + if not _VIDEO_ID.match(vid) or at is None: + parsed.malformed += 1 + continue + _keep(parsed, vid, at, None) + return parsed + + +def parse(body: bytes) -> Parsed: + """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told + apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file + with nothing to import in it.""" + try: + text = body.decode("utf-8-sig") + except UnicodeDecodeError: + raise TakeoutError("not_takeout") from None + stripped = text.lstrip() + if stripped.startswith("["): + try: + data = json.loads(stripped) + except json.JSONDecodeError: + raise TakeoutError("not_takeout") from None + if not isinstance(data, list): + raise TakeoutError("not_takeout") + parsed = _history(data) + elif stripped.startswith(("{", "<")): + # Another JSON file of the export, or the history's HTML variant. + raise TakeoutError("not_takeout") + else: + parsed = _watch_later(stripped) + if not parsed.videos: + raise TakeoutError("no_videos") + return parsed + + +# One import per user at a time: a second file uploaded while the first one's run is under way +# starts a run of its own (the first one's id has left the scheduler), and the two would settle the +# same rows. The second waits, then takes what the first left. One API process runs the scheduler, +# so a lock in this module holds it (as the channel backfill's does). +_user_locks: dict[int, threading.Lock] = {} +_user_locks_guard = threading.Lock() + + +@contextmanager +def _one_import(user_id: int) -> Iterator[None]: + with _user_locks_guard: + lock = _user_locks.setdefault(user_id, threading.Lock()) + with lock: + yield + + +def run_import(user_id: int) -> dict: + """The import job: apply what the user's kept Takeout rows can say now, as one recorded run. + The watched part commits before the Watch later part asks YouTube for the videos the catalog + lacks, so a YouTube failure there loses nothing already marked. Returns the counts.""" + with _one_import(user_id), SessionLocal() as db: + user = db.get(User, user_id) + if user is None: + return {"skipped": "no user"} + result = service.apply_takeout_watched(db, user) + db.commit() + waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER) + if waiting: + fetcher = get_service_user(db) + if fetcher is not None: + with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP): + with YouTubeClient(db, fetcher) as yt: + ensure_videos(db, yt, [vid for _, vid in waiting]) + result |= service.apply_takeout_watch_later(db, user) + db.commit() + outcomes = queries.takeout_outcomes(db, user) + result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0) + if outcomes.get((TakeoutKind.WATCH_LATER, None)): + result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)] + return result diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 001679a0..a8fd811a 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -1,18 +1,25 @@ from datetime import date +from functools import partial -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Request from pydantic import BaseModel from sqlalchemy.orm import Session from sqlalchemy.orm.exc import StaleDataError -from app import quota +from app import quota, scheduler from app.auth import current_user -from app.db import get_db +from app.db import get_db, release_read_connection from app.mailtext import lang_for_user from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years from app.models import User -from app.reader import queries, service -from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal +from app.reader import queries, service, takeout +from app.routes._common import ( + ProgressIn, + capped_body, + kept_position, + not_found, + youtube_refusal, +) from app.sync.videos import parse_iso8601_duration from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError @@ -293,3 +300,49 @@ def get_video_detail( items[0].get("contentDetails", {}).get("duration") ), } + + +# --- Google Takeout ------------------------------------------------------------------------------ + + +async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes: + # The sign-in check has run, and it was a SELECT: hand the connection back before a file of + # megabytes arrives, so a slow upload does not hold one `idle in transaction`. + release_read_connection(db) + return await capped_body(request, takeout.MAX_BYTES) + + +@router.post("/takeout") +def upload_takeout( + # The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before + # a byte of it is read. + user: User = Depends(current_user), + body: bytes = Depends(_takeout_body), + db: Session = Depends(get_db), +): + """One file of the user's Google Takeout export — the watch history or the Watch later list. + Its videos are kept and the import runs in the background; its outcome reaches the user's inbox.""" + try: + parsed = takeout.parse(body) + except takeout.TakeoutError as exc: + if exc.reason == "no_videos": + raise refused(422, "takeout_no_videos") from None + raise refused(422, "takeout_unreadable") from None + service.stage_takeout(db, user, parsed.kind, parsed.videos) + db.commit() + scheduler.run_ad_hoc( + scheduler.TAKEOUT_IMPORT, + partial(takeout.run_import, user.id), + actor_id=user.id, + target=f"user:{user.id}", + notify=True, + ) + return { + "ok": True, + "kind": parsed.kind, + "videos": len(parsed.videos), + "entries": parsed.entries, + "ads": parsed.ads, + "other": parsed.other, + "malformed": parsed.malformed, + } diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index c2a3ae09..dc5e2506 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -118,8 +118,9 @@ def _notify_done( summary: str | None, result: dict | None = None, ) -> None: - """Post a completion notice to the triggering admin's inbox (manual runs only). Best - effort — a notification failure must never mask the job's own outcome.""" + """Post a completion notice to the inbox of who started the run: an admin's manual run, or a + user's ad-hoc run that asks for one. Best effort — a notification failure must never mask the + job's own outcome.""" try: create_notification( db, @@ -608,14 +609,19 @@ def trigger_all(actor_id: int | None = None) -> list[str]: # one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and # spent is in `job_runs` like any job's, not a background task only the log ever heard of. CHANNEL_BACKFILL = "channel_backfill" +# A user's Google Takeout import (`reader.takeout`): its target is the user. +TAKEOUT_IMPORT = "takeout_import" # Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple). -AD_HOC_JOBS = (CHANNEL_BACKFILL,) +AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT) -def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None: +def run_ad_hoc( + job: str, fn: Callable[[], dict], *, actor_id: int, target: str, notify: bool = False +) -> None: """Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so - a second request before the first has even started is that same run.""" + a second request before the first has even started is that same run. `notify` posts its outcome + to the user's inbox, for work whose page does not watch it run.""" if job not in AD_HOC_JOBS: raise KeyError(job) # an unnamed job would show on the page as its bare id sched = _scheduler @@ -626,7 +632,7 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) sched.add_job( _ad_hoc, "date", - args=(job, fn, actor_id, target), + args=(job, fn, actor_id, target, notify), id=f"{job}@{target}", misfire_grace_time=None, ) @@ -635,7 +641,9 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) log.info("ad-hoc %s for %s already queued", job, target) -def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None: +def _ad_hoc( + job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False +) -> None: try: run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target) except Exception: @@ -644,12 +652,12 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non return _opened(run_id, job) try: - _ad_hoc_work(job, fn, run_id) + _ad_hoc_work(job, fn, run_id, actor_id if notify else None) finally: _closed(run_id) -def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: +def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | None) -> None: with logs.bind(run_id=run_id): # As in _run: only the work is inside the try. try: @@ -658,8 +666,19 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None: log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job)) err, failure = _failure(exc) _close(run_id, RunStatus.ERROR, error=err, failure=failure) + _notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure) return - _finish_ok(job, run_id, result) + summary, data = _finish_ok(job, run_id, result) + _notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data) + + +def _notify_ad_hoc( + user_id: int | None, job: str, status: RunStatus, summary: str | None, result: dict | None +) -> None: + if user_id is None: + return + with SessionLocal() as db: + _notify_done(db, user_id, job, status, summary, result) # Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of diff --git a/backend/tests/test_body_limit.py b/backend/tests/test_body_limit.py index dc308d01..ad1f72c2 100644 --- a/backend/tests/test_body_limit.py +++ b/backend/tests/test_body_limit.py @@ -221,6 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes(): CAPPED_BODY_SITES = { ("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"), ("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"), + ("feed.py", "_takeout_body", "capped_body", "takeout.MAX_BYTES"), ("files.py", "spool_upload", "capped_stream_to", "cap"), } diff --git a/backend/tests/test_takeout.py b/backend/tests/test_takeout.py new file mode 100644 index 00000000..39d65834 --- /dev/null +++ b/backend/tests/test_takeout.py @@ -0,0 +1,362 @@ +"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the +import job makes of them — watched here (never a Short, never over a state the user set), waiting for +a video the catalog does not know yet, and the Watch later list joined to the user's own.""" + +import json +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import select + +from app import scheduler +from app.catalog.models import Channel, Video +from app.models import Notification +from app.reader import service, takeout +from app.reader.models import ( + Playlist, + PlaylistItem, + TakeoutItem, + TakeoutKind, + VideoState, +) + +T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc) +CHANNEL = "UC" + "a" * 22 + + +def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra): + entry = { + "header": "YouTube", + "title": "Megnézted: valami", # the account's language: never read + "titleUrl": f"https://{host}.youtube.com/watch?v={vid}", + "time": at.isoformat().replace("+00:00", "Z"), + "products": ["YouTube"], + "activityControls": ["YouTube watch history"], + } + if channel: + entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}] + return entry | extra + + +def _history(*entries) -> bytes: + return json.dumps(list(entries)).encode() + + +# --- reading the files ------------------------------------------------------------------------- + + +def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel(): + later = T0 + timedelta(days=3) + parsed = takeout.parse( + _history( + _view("aaaaaaaaaaa", later), + _view("aaaaaaaaaaa", T0), + _view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel + _view("ccccccccccc", T0, host="music"), + # Out of order: the later view still wins. + _view("ddddddddddd", T0, channel=None), + _view("ddddddddddd", later), + ) + ) + assert parsed.kind == TakeoutKind.WATCHED + assert parsed.entries == 6 + assert parsed.videos == { + "aaaaaaaaaaa": (later, CHANNEL), + "bbbbbbbbbbb": (T0, None), + "ccccccccccc": (T0, CHANNEL), + "ddddddddddd": (later, CHANNEL), + } + + +def test_the_history_counts_what_it_leaves_out(): + ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}]) + post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"} + no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"} + bad_time = _view("ggggggggggg") | {"time": "yesterday"} + no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"} + parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk")) + assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2) + assert parsed.videos == { + "hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL) + } + + +def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none(): + body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n" + parsed = takeout.parse(("" + body).encode()) + assert parsed.kind == TakeoutKind.WATCH_LATER + assert parsed.videos == { + "Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None) + } + assert parsed.malformed == 1 + headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n") + assert list(headless.videos) == ["Ngb04xUK8uU"] + + +@pytest.mark.parametrize( + ("body", "reason"), + [ + (b'{"some": "other", "export": "file"}', "not_takeout"), + (b"watch history, as a page", "not_takeout"), + (b"[not json", "not_takeout"), + (b"\xff\xfe\x00", "not_takeout"), + (b"one column only\n", "not_takeout"), + (b"[]", "no_videos"), + ( + json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(), + "no_videos", + ), + ], +) +def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason): + with pytest.raises(takeout.TakeoutError) as caught: + takeout.parse(body) + assert caught.value.reason == reason + + +# --- keeping and applying ---------------------------------------------------------------------- + + +def _video(db, vid: str, *, probed=True, short=False): + if db.get(Channel, CHANNEL) is None: + db.add(Channel(id=CHANNEL, title="Ch")) + db.flush() + db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short)) + + +def _state(db, user, vid, status, *, changed, position=0, watched_at=None): + db.add( + VideoState( + user_id=user.id, + video_id=vid, + status=status, + position_seconds=position, + watched_at=watched_at, + updated_at=changed, + ) + ) + + +def _rows(db, user) -> dict[str, TakeoutItem]: + db.expire_all() + rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars() + return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows} + + +def _states(db, user) -> dict[str, VideoState]: + db.expire_all() + rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars() + return {r.video_id: r for r in rows} + + +def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept( + db, make_user +): + user = make_user("takeout@example.com") + for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"): + _video(db, vid, probed=vid != "unprobed000", short=vid == "short000000") + for vid in ("newer000000", "older000000"): + _video(db, vid) + earlier = T0 - timedelta(days=30) + _state(db, user, "hidden00000", "hidden", changed=earlier) + _state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier) + # In progress here, touched after the YouTube view: what the user did here is newer. + _state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40) + # In progress here, but last touched before the YouTube view: they finished it there. + _state(db, user, "older000000", "new", changed=earlier, position=40) + db.commit() + ids = [ + "plain000000", + "short000000", + "unprobed000", + "unknown0000", + "hidden00000", + "watched0000", + "newer000000", + "older000000", + ] + service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids}) + + counts = service.apply_takeout_watched(db, user) + db.commit() + + assert counts == {"watched": 2, "kept": 3, "shorts": 1} + rows, states = _rows(db, user), _states(db, user) + assert {v: rows[v].outcome for v in ids} == { + "plain000000": "watched", + "short000000": "short", + "unprobed000": None, + "unknown0000": None, + "hidden00000": "kept", + "watched0000": "kept", + "newer000000": "kept", + "older000000": "watched", + } + assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0) + assert "short000000" not in states + assert states["hidden00000"].status == "hidden" + assert states["watched0000"].watched_at == earlier + assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40) + assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40) + + +def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user): + user = make_user("again@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + service.apply_takeout_watched(db, user) + later = T0 + timedelta(days=1) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)}) + service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)}) + db.commit() + row = _rows(db, user)["plain000000"] + assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched") + + +def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user): + user = make_user("later@example.com") + for vid in ("held0000000", "old00000000", "new00000000"): + _video(db, vid) + db.commit() + pl = service.watch_later(db, user) + service.add_item(db, pl, "held0000000", mark_dirty=False) + service.stage_takeout( + db, + user, + TakeoutKind.WATCH_LATER, + { + "new00000000": (T0, None), + "held0000000": (T0, None), + "old00000000": (T0 - timedelta(days=9), None), + "gone0000000": (T0, None), # YouTube no longer has it + }, + ) + + counts = service.apply_takeout_watch_later(db, user) + db.commit() + + assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1} + items = db.execute( + select(PlaylistItem.video_id) + .where(PlaylistItem.playlist_id == pl.id) + .order_by(PlaylistItem.position) + ).scalars() + assert list(items) == ["held0000000", "old00000000", "new00000000"] + assert _rows(db, user)["WL:gone0000000"].outcome == "missing" + assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl] + + +def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits( + db, make_user, monkeypatch +): + user = make_user("job@example.com") + _video(db, "plain000000") + db.commit() + service.stage_takeout( + db, + user, + TakeoutKind.WATCHED, + {"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)}, + ) + service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)}) + db.commit() + asked = [] + + def fake_ensure(session, yt, ids): + asked.append(ids) + _video(session, "fetched0000") + session.commit() + + monkeypatch.setattr(takeout, "ensure_videos", fake_ensure) + monkeypatch.setattr(takeout, "get_service_user", lambda session: user) + + result = takeout.run_import(user.id) + + assert asked == [["fetched0000"]] + assert result == { + "watched": 1, + "kept": 0, + "shorts": 0, + "watch_later_added": 1, + "watch_later_present": 0, + "watch_later_missing": 0, + "waiting": 1, + } + + +# --- the upload -------------------------------------------------------------------------------- + + +@pytest.fixture +def queued(monkeypatch): + runs = [] + monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw))) + return runs + + +def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice( + db, make_user, as_user, queued +): + user = make_user("upload@example.com") + body = _history( + _view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}]) + ) + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 200, resp.text + assert resp.json() == { + "ok": True, + "kind": "watched", + "videos": 1, + "entries": 2, + "ads": 1, + "other": 0, + "malformed": 0, + } + assert list(_rows(db, user)) == ["aaaaaaaaaaa"] + assert queued == [ + ( + "takeout_import", + {"actor_id": user.id, "target": f"user:{user.id}", "notify": True}, + ) + ] + + +@pytest.mark.parametrize( + ("body", "reason"), + [(b"", "takeout_unreadable"), (b"[]", "takeout_no_videos")], +) +def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language( + db, make_user, as_user, queued, body, reason +): + user = make_user("refused@example.com") + resp = as_user(user).post("/api/takeout", content=body) + assert resp.status_code == 422 + assert resp.json()["detail"] == {"code": "refused", "reason": reason} + assert queued == [] and _rows(db, user) == {} + + +def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch): + user = make_user("big@example.com") + monkeypatch.setattr(takeout, "MAX_BYTES", 64) + resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))) + assert resp.status_code == 413 + assert _rows(db, user) == {} + + +def test_an_anonymous_upload_is_refused(client): + assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401 + + +# --- the notice -------------------------------------------------------------------------------- + + +def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user): + user = make_user("notice@example.com") + scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True) + scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x") + db.expire_all() + notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars() + assert [(n.data["job_id"], n.data["result"]) for n in notices] == [ + ("takeout_import", {"watched": 3}) + ] diff --git a/backend/tests/test_unit_fences.py b/backend/tests/test_unit_fences.py index 9bce2f17..f49b3a0d 100644 --- a/backend/tests/test_unit_fences.py +++ b/backend/tests/test_unit_fences.py @@ -44,6 +44,7 @@ UNIT_MODELS = { "PlaylistItem", "SavedView", "ComfortPin", + "TakeoutItem", "Tag", "ChannelTag", }, @@ -360,6 +361,9 @@ UNIT_COMMITS = { "sync/playlists.py:pull_user_playlists", "sync/playlists.py:merge_playlist", "sync/playlists.py:repull_playlist", + # The Takeout import is a recorded run with its own session: the watched part lands before + # the Watch later part asks YouTube (which commits the videos it fetched). + "reader/takeout.py:run_import", }, # The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page # fetched and stored is not lost when the next call fails, and an enrich that fails rolls back diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index 2c3afd48..d82745cc 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -249,6 +249,8 @@ "api_token_preset_grants": "The Hub token gets its folder by itself — leave the folders out", "api_token_no_grants": "Choose at least one folder the token may use", "api_token_name_required": "Give the token a name", - "api_token_limit": "You already have {{limit}} tokens — revoke one first" + "api_token_limit": "You already have {{limit}} tokens — revoke one first", + "takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.", + "takeout_no_videos": "This file has no videos to import" } } diff --git a/frontend/src/i18n/locales/en/scheduler.json b/frontend/src/i18n/locales/en/scheduler.json index f3a40d0a..7c1a7101 100644 --- a/frontend/src/i18n/locales/en/scheduler.json +++ b/frontend/src/i18n/locales/en/scheduler.json @@ -80,7 +80,8 @@ "log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.", "tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.", "people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.", - "channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running)." + "channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running).", + "takeout_import": "A user's Google Takeout import: what they watched on YouTube becomes watched here (not Shorts, never over a state they set here), and their YouTube Watch later list joins theirs. A video the catalogue does not know yet waits, and is marked once it arrives." }, "jobs": { "rss_poll": "RSS poll (new uploads)", @@ -100,7 +101,8 @@ "audit_gc": "Activity and quota-history cleanup", "log_gc": "Log cleanup", "people_enrich": "People details and portraits (Plex)", - "channel_backfill": "A channel's full history" + "channel_backfill": "A channel's full history", + "takeout_import": "Google Takeout import" }, "queue": { "recentPending": "Channels to sync", @@ -178,7 +180,13 @@ "deep": "deep", "bios": "biographies", "portraits": "portraits", - "cached": "cached" + "cached": "cached", + "kept": "kept as set here", + "waiting": "waiting for their video", + "watch_later_added": "added to Watch later", + "watch_later_present": "already in Watch later", + "watch_later_missing": "no longer on YouTube", + "watch_later_waiting": "Watch later waiting" }, "skip": { "disabled": "disabled", @@ -191,7 +199,8 @@ "busy": "another sync is running", "channelbusy": "this channel is already being backfilled", "atcapacity": "two full-history backfills are already running — the scheduler finishes this one", - "nothingtodo": "nothing left to fetch" + "nothingtodo": "nothing left to fetch", + "nouser": "the account is gone" }, "failure": { "rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.", diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index d0d54d8a..24351fc4 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -249,6 +249,8 @@ "api_token_preset_grants": "A Hub-token magától kapja a mappáját — ne adj meg mappát", "api_token_no_grants": "Válassz legalább egy mappát, amelyet a token használhat", "api_token_name_required": "Adj nevet a tokennek", - "api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet" + "api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet", + "takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.", + "takeout_no_videos": "Ebben a fájlban nincs importálható videó" } } diff --git a/frontend/src/i18n/locales/hu/scheduler.json b/frontend/src/i18n/locales/hu/scheduler.json index a6649a5f..244cf1b0 100644 --- a/frontend/src/i18n/locales/hu/scheduler.json +++ b/frontend/src/i18n/locales/hu/scheduler.json @@ -80,7 +80,8 @@ "log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.", "tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.", "people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.", - "channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be." + "channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be.", + "takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül az övébe. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz." }, "jobs": { "rss_poll": "RSS-lekérdezés (új feltöltések)", @@ -100,7 +101,8 @@ "audit_gc": "Napló- és kvótaelőzmény-takarítás", "log_gc": "Naplótakarítás", "people_enrich": "Személyadatok és portrék (Plex)", - "channel_backfill": "Egy csatorna teljes előzménye" + "channel_backfill": "Egy csatorna teljes előzménye", + "takeout_import": "Google Takeout import" }, "queue": { "recentPending": "Szinkronra váró csatorna", @@ -178,7 +180,13 @@ "deep": "mély", "bios": "életrajzok", "portraits": "portrék", - "cached": "elmentve" + "cached": "elmentve", + "kept": "itteni állapot megtartva", + "waiting": "a videójára vár", + "watch_later_added": "Watch laterbe téve", + "watch_later_present": "már a Watch laterben", + "watch_later_missing": "már nincs a YouTube-on", + "watch_later_waiting": "Watch later vár" }, "skip": { "disabled": "kikapcsolva", @@ -191,7 +199,8 @@ "busy": "egy másik szinkron fut", "channelbusy": "ennek a csatornának a betöltése már fut", "atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be", - "nothingtodo": "nincs több letöltendő" + "nothingtodo": "nincs több letöltendő", + "nouser": "a fiók már nem létezik" }, "failure": { "rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",