feat(reader): a Google Takeout export becomes watch state and Watch later
The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
@@ -0,0 +1,51 @@
|
|||||||
|
"""A user's Google Takeout export: what they watched on YouTube, and their Watch later list
|
||||||
|
|
||||||
|
Revision ID: 0113_takeout_items
|
||||||
|
Revises: 0112_one_watch_later
|
||||||
|
Create Date: 2026-10-23
|
||||||
|
|
||||||
|
One row per video and kind. A video the catalog does not know yet waits with no outcome, so the id
|
||||||
|
is not a foreign key; the partial index is what a newly known video looks its waiting rows up by.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0113_takeout_items"
|
||||||
|
down_revision: str | None = "0112_one_watch_later"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"takeout_items",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("kind", sa.String(12), nullable=False),
|
||||||
|
sa.Column("video_id", sa.String(16), nullable=False),
|
||||||
|
sa.Column("channel_id", sa.String(32), nullable=True),
|
||||||
|
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("outcome", sa.String(12), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
||||||
|
sa.CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_takeout_items_waiting",
|
||||||
|
"takeout_items",
|
||||||
|
["video_id"],
|
||||||
|
postgresql_where=sa.text("outcome IS NULL"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_takeout_items_waiting", table_name="takeout_items")
|
||||||
|
op.drop_table("takeout_items")
|
||||||
@@ -72,6 +72,10 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
|||||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||||
# default, so listing it could only widen what it accepts.
|
# default, so listing it could only widen what it accepts.
|
||||||
|
# A Google Takeout file (`reader.takeout`): a real watch history of 9500 views, the 18 months Google
|
||||||
|
# keeps by default, is 3.7 MB; four times that is still well inside one request.
|
||||||
|
TAKEOUT_MAX_BYTES = 16 * 1024 * 1024
|
||||||
|
|
||||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||||
# Ingesting a local file. The body IS the file now — no multipart envelope to allow for —
|
# Ingesting a local file. The body IS the file now — no multipart envelope to allow for —
|
||||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||||
@@ -95,6 +99,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
|||||||
# sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
|
# sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
|
||||||
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
|
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
|
||||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||||
|
("POST", "/api/takeout"): TAKEOUT_MAX_BYTES,
|
||||||
}
|
}
|
||||||
|
|
||||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th
|
|||||||
edge, or asks this unit's service."""
|
edge, or asks this unit's service."""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
Boolean,
|
Boolean,
|
||||||
@@ -213,3 +214,51 @@ class ComfortPin(Base, TimestampMixin):
|
|||||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class TakeoutKind(StrEnum):
|
||||||
|
"""What a Google Takeout row says the user did with a video."""
|
||||||
|
|
||||||
|
WATCHED = "watched" # history/watch-history.json
|
||||||
|
WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv
|
||||||
|
|
||||||
|
|
||||||
|
class TakeoutOutcome(StrEnum):
|
||||||
|
"""What the import did with a row; NULL = still waiting for its video to be known (and, for a
|
||||||
|
watched one, probed for Shorts)."""
|
||||||
|
|
||||||
|
WATCHED = "watched" # the video is marked watched at the Takeout time
|
||||||
|
KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays
|
||||||
|
SHORT = "short" # a Short: swiping one past is not watching it
|
||||||
|
ADDED = "added" # added to Watch later
|
||||||
|
PRESENT = "present" # already in Watch later
|
||||||
|
MISSING = "missing" # YouTube no longer has the video
|
||||||
|
|
||||||
|
|
||||||
|
class TakeoutItem(Base, TimestampMixin):
|
||||||
|
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
||||||
|
later list (with when it was added). The rows are both the import job's input and its memory: a
|
||||||
|
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
||||||
|
The id is not a foreign key for that reason. The channel is kept for a later "channels you
|
||||||
|
watch but do not follow" list."""
|
||||||
|
|
||||||
|
__tablename__ = "takeout_items"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
||||||
|
CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
||||||
|
# The waiting rows, by video: what a newly known video looks up.
|
||||||
|
Index(
|
||||||
|
"ix_takeout_items_waiting",
|
||||||
|
"video_id",
|
||||||
|
postgresql_where=text("outcome IS NULL"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
|
kind: Mapped[str] = mapped_column(String(12))
|
||||||
|
video_id: Mapped[str] = mapped_column(String(16))
|
||||||
|
channel_id: Mapped[str | None] = mapped_column(String(32))
|
||||||
|
# Watched: the last time in the export; Watch later: when it was added.
|
||||||
|
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
outcome: Mapped[str | None] = mapped_column(String(12))
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
|
"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP
|
||||||
for the other units that ask about them.
|
layer and for the other units that ask about them.
|
||||||
|
|
||||||
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
||||||
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
||||||
@@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t
|
|||||||
than joining Reader's tables itself.
|
than joining Reader's tables itself.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||||
|
|
||||||
@@ -14,7 +16,16 @@ from app.collation import collated
|
|||||||
from app.feedquery import feed_columns, serialize
|
from app.feedquery import feed_columns, serialize
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.catalog.models import Channel, Subscription, Video
|
from app.catalog.models import Channel, Subscription, Video
|
||||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
from app.reader.models import (
|
||||||
|
ChannelTag,
|
||||||
|
Playlist,
|
||||||
|
PlaylistItem,
|
||||||
|
SavedView,
|
||||||
|
Tag,
|
||||||
|
TakeoutItem,
|
||||||
|
TakeoutKind,
|
||||||
|
VideoState,
|
||||||
|
)
|
||||||
from app.userscope import owned_by, owned_row
|
from app.userscope import owned_by, owned_row
|
||||||
|
|
||||||
# --- catalog, read along the edge ------------------------------------------------------------
|
# --- catalog, read along the edge ------------------------------------------------------------
|
||||||
@@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
|
|||||||
owned_by(ChannelTag.user_id, user),
|
owned_by(ChannelTag.user_id, user),
|
||||||
)
|
)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Google Takeout --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
|
||||||
|
"""The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
|
||||||
|
(row id, video id, Takeout time, is Short)."""
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short)
|
||||||
|
.join(Video, Video.id == TakeoutItem.video_id)
|
||||||
|
.where(
|
||||||
|
owned_by(TakeoutItem.user_id, user),
|
||||||
|
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||||
|
TakeoutItem.outcome.is_(None),
|
||||||
|
Video.shorts_probed,
|
||||||
|
)
|
||||||
|
).tuples()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]:
|
||||||
|
"""The user's rows of `kind` the import has not settled, oldest first: (row id, video id)."""
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
select(TakeoutItem.id, TakeoutItem.video_id)
|
||||||
|
.where(
|
||||||
|
owned_by(TakeoutItem.user_id, user),
|
||||||
|
TakeoutItem.kind == kind,
|
||||||
|
TakeoutItem.outcome.is_(None),
|
||||||
|
)
|
||||||
|
.order_by(TakeoutItem.at, TakeoutItem.id)
|
||||||
|
).tuples()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
|
||||||
|
"""Which of `ids` the catalog holds."""
|
||||||
|
if not ids:
|
||||||
|
return set()
|
||||||
|
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
|
||||||
|
|
||||||
|
|
||||||
|
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
|
||||||
|
"""How many of the user's rows each kind and outcome holds (None = waiting)."""
|
||||||
|
rows = db.execute(
|
||||||
|
select(TakeoutItem.kind, TakeoutItem.outcome, func.count())
|
||||||
|
.where(owned_by(TakeoutItem.user_id, user))
|
||||||
|
.group_by(TakeoutItem.kind, TakeoutItem.outcome)
|
||||||
|
).all()
|
||||||
|
return {(kind, outcome): n for kind, outcome, n in rows}
|
||||||
|
|||||||
@@ -15,7 +15,17 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.reader import queries
|
from app.reader import queries
|
||||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
from app.reader.models import (
|
||||||
|
ChannelTag,
|
||||||
|
Playlist,
|
||||||
|
PlaylistItem,
|
||||||
|
SavedView,
|
||||||
|
Tag,
|
||||||
|
TakeoutItem,
|
||||||
|
TakeoutKind,
|
||||||
|
TakeoutOutcome,
|
||||||
|
VideoState,
|
||||||
|
)
|
||||||
from app.sync.playlists import lock_playlists, recompute_dirty
|
from app.sync.playlists import lock_playlists, recompute_dirty
|
||||||
from app.userscope import owned_by
|
from app.userscope import owned_by
|
||||||
|
|
||||||
@@ -495,3 +505,113 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No
|
|||||||
if row is not None:
|
if row is not None:
|
||||||
db.delete(row)
|
db.delete(row)
|
||||||
db.flush()
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Google Takeout --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_TAKEOUT_BATCH = 1000
|
||||||
|
|
||||||
|
|
||||||
|
def stage_takeout(
|
||||||
|
db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]]
|
||||||
|
) -> None:
|
||||||
|
"""Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job.
|
||||||
|
A second upload of the same export, or a newer one, adds what is new: a video already kept keeps
|
||||||
|
the later time, and a row the import has settled keeps its outcome."""
|
||||||
|
values = [
|
||||||
|
{"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel}
|
||||||
|
for vid, (at, channel) in rows.items()
|
||||||
|
]
|
||||||
|
for start in range(0, len(values), _TAKEOUT_BATCH):
|
||||||
|
stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH])
|
||||||
|
db.execute(
|
||||||
|
stmt.on_conflict_do_update(
|
||||||
|
constraint="uq_takeout_item",
|
||||||
|
set_={
|
||||||
|
"at": func.greatest(TakeoutItem.at, stmt.excluded.at),
|
||||||
|
"channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
|
||||||
|
for start in range(0, len(ids), _TAKEOUT_BATCH):
|
||||||
|
db.execute(
|
||||||
|
update(TakeoutItem)
|
||||||
|
.where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH]))
|
||||||
|
.values(outcome=outcome)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
|
||||||
|
"""Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
|
||||||
|
for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
|
||||||
|
past is not watching it), and a state the user set here wins: watched or hidden, or anything
|
||||||
|
changed after the Takeout time. The upsert's own condition is that rule, so a state set while
|
||||||
|
the import runs wins too."""
|
||||||
|
short: list[int] = []
|
||||||
|
todo: dict[str, tuple[int, datetime]] = {}
|
||||||
|
for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user):
|
||||||
|
if is_short:
|
||||||
|
short.append(row_id)
|
||||||
|
else:
|
||||||
|
todo[video_id] = (row_id, at)
|
||||||
|
marked: set[str] = set()
|
||||||
|
items = list(todo.items())
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for start in range(0, len(items), _TAKEOUT_BATCH):
|
||||||
|
stmt = pg_insert(VideoState).values(
|
||||||
|
[
|
||||||
|
{"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at}
|
||||||
|
for vid, (_, at) in items[start : start + _TAKEOUT_BATCH]
|
||||||
|
]
|
||||||
|
)
|
||||||
|
marked.update(
|
||||||
|
db.execute(
|
||||||
|
stmt.on_conflict_do_update(
|
||||||
|
constraint="uq_user_video",
|
||||||
|
set_={
|
||||||
|
"status": "watched",
|
||||||
|
"watched_at": stmt.excluded.watched_at,
|
||||||
|
"updated_at": now,
|
||||||
|
},
|
||||||
|
where=(VideoState.status == "new")
|
||||||
|
& (VideoState.updated_at < stmt.excluded.watched_at),
|
||||||
|
).returning(VideoState.video_id)
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
kept = [row_id for vid, (row_id, _) in items if vid not in marked]
|
||||||
|
_settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
|
||||||
|
_settle(db, kept, TakeoutOutcome.KEPT)
|
||||||
|
_settle(db, short, TakeoutOutcome.SHORT)
|
||||||
|
db.flush()
|
||||||
|
return {"watched": len(marked), "kept": len(kept), "shorts": len(short)}
|
||||||
|
|
||||||
|
|
||||||
|
def apply_takeout_watch_later(db: Session, user: User) -> dict[str, int]:
|
||||||
|
"""Put the videos of the user's YouTube Watch later list into theirs here, oldest first, after
|
||||||
|
what it already holds. Called once the catalog has fetched what it could: a video still unknown
|
||||||
|
is one YouTube no longer has."""
|
||||||
|
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||||
|
if not waiting:
|
||||||
|
return {}
|
||||||
|
known = queries.known_video_ids(db, [vid for _, vid in waiting])
|
||||||
|
pl = watch_later(db, user)
|
||||||
|
done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
|
||||||
|
for row_id, vid in waiting:
|
||||||
|
if vid not in known:
|
||||||
|
done[TakeoutOutcome.MISSING].append(row_id)
|
||||||
|
elif add_item(db, pl, vid, mark_dirty=False):
|
||||||
|
done[TakeoutOutcome.ADDED].append(row_id)
|
||||||
|
else:
|
||||||
|
done[TakeoutOutcome.PRESENT].append(row_id)
|
||||||
|
for outcome, ids in done.items():
|
||||||
|
_settle(db, ids, outcome)
|
||||||
|
db.flush()
|
||||||
|
return {
|
||||||
|
"watch_later_added": len(done[TakeoutOutcome.ADDED]),
|
||||||
|
"watch_later_present": len(done[TakeoutOutcome.PRESENT]),
|
||||||
|
"watch_later_missing": len(done[TakeoutOutcome.MISSING]),
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
||||||
|
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
||||||
|
only way in.
|
||||||
|
|
||||||
|
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
||||||
|
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
||||||
|
what it can; a video the catalog does not know yet waits for it.
|
||||||
|
|
||||||
|
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
||||||
|
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
||||||
|
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
||||||
|
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
||||||
|
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
||||||
|
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
||||||
|
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import threading
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from app import quota
|
||||||
|
from app.bodylimit import TAKEOUT_MAX_BYTES
|
||||||
|
from app.db import SessionLocal
|
||||||
|
from app.models import User
|
||||||
|
from app.reader import queries, service
|
||||||
|
from app.reader.models import TakeoutKind
|
||||||
|
from app.sync.runner import get_service_user
|
||||||
|
from app.sync.videos import ensure_videos
|
||||||
|
from app.youtube.client import YouTubeClient
|
||||||
|
|
||||||
|
# The largest file the upload takes (its edge cap and the route's own are one number).
|
||||||
|
MAX_BYTES = TAKEOUT_MAX_BYTES
|
||||||
|
|
||||||
|
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||||
|
_WATCH_URL = re.compile(
|
||||||
|
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
||||||
|
)
|
||||||
|
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
||||||
|
|
||||||
|
|
||||||
|
class TakeoutError(ValueError):
|
||||||
|
"""The file is not one the import reads; `reason` is the client's word for why."""
|
||||||
|
|
||||||
|
def __init__(self, reason: str):
|
||||||
|
super().__init__(reason)
|
||||||
|
self.reason = reason
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Parsed:
|
||||||
|
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
||||||
|
|
||||||
|
kind: TakeoutKind
|
||||||
|
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
||||||
|
entries: int = 0
|
||||||
|
ads: int = 0
|
||||||
|
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
||||||
|
malformed: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def _time(raw: object) -> datetime | None:
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||||
|
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return at if at.tzinfo is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
||||||
|
old = parsed.videos.get(vid)
|
||||||
|
if old is None or at > old[0]:
|
||||||
|
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
||||||
|
|
||||||
|
|
||||||
|
def _history(data: list) -> Parsed:
|
||||||
|
parsed = Parsed(TakeoutKind.WATCHED)
|
||||||
|
for entry in data:
|
||||||
|
parsed.entries += 1
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
parsed.malformed += 1
|
||||||
|
continue
|
||||||
|
details = entry.get("details")
|
||||||
|
if isinstance(details, list) and any(
|
||||||
|
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
||||||
|
):
|
||||||
|
parsed.ads += 1
|
||||||
|
continue
|
||||||
|
url = entry.get("titleUrl")
|
||||||
|
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
||||||
|
if match is None:
|
||||||
|
parsed.other += 1
|
||||||
|
continue
|
||||||
|
at = _time(entry.get("time"))
|
||||||
|
if at is None:
|
||||||
|
parsed.malformed += 1
|
||||||
|
continue
|
||||||
|
channel = None
|
||||||
|
subtitles = entry.get("subtitles")
|
||||||
|
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
||||||
|
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
||||||
|
channel = found.group(1) if found else None
|
||||||
|
_keep(parsed, match.group(1), at, channel)
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def _watch_later(text: str) -> Parsed:
|
||||||
|
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
||||||
|
rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
|
||||||
|
if not rows or len(rows[0]) < 2:
|
||||||
|
raise TakeoutError("not_takeout")
|
||||||
|
if not _VIDEO_ID.match(rows[0][0].strip()):
|
||||||
|
rows = rows[1:] # the header, in the account's language
|
||||||
|
for row in rows:
|
||||||
|
parsed.entries += 1
|
||||||
|
vid = row[0].strip() if row else ""
|
||||||
|
at = _time(row[1].strip()) if len(row) > 1 else None
|
||||||
|
if not _VIDEO_ID.match(vid) or at is None:
|
||||||
|
parsed.malformed += 1
|
||||||
|
continue
|
||||||
|
_keep(parsed, vid, at, None)
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def parse(body: bytes) -> Parsed:
|
||||||
|
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
||||||
|
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
||||||
|
with nothing to import in it."""
|
||||||
|
try:
|
||||||
|
text = body.decode("utf-8-sig")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise TakeoutError("not_takeout") from None
|
||||||
|
stripped = text.lstrip()
|
||||||
|
if stripped.startswith("["):
|
||||||
|
try:
|
||||||
|
data = json.loads(stripped)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
raise TakeoutError("not_takeout") from None
|
||||||
|
if not isinstance(data, list):
|
||||||
|
raise TakeoutError("not_takeout")
|
||||||
|
parsed = _history(data)
|
||||||
|
elif stripped.startswith(("{", "<")):
|
||||||
|
# Another JSON file of the export, or the history's HTML variant.
|
||||||
|
raise TakeoutError("not_takeout")
|
||||||
|
else:
|
||||||
|
parsed = _watch_later(stripped)
|
||||||
|
if not parsed.videos:
|
||||||
|
raise TakeoutError("no_videos")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
# One import per user at a time: a second file uploaded while the first one's run is under way
|
||||||
|
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
|
||||||
|
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
|
||||||
|
# so a lock in this module holds it (as the channel backfill's does).
|
||||||
|
_user_locks: dict[int, threading.Lock] = {}
|
||||||
|
_user_locks_guard = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _one_import(user_id: int) -> Iterator[None]:
|
||||||
|
with _user_locks_guard:
|
||||||
|
lock = _user_locks.setdefault(user_id, threading.Lock())
|
||||||
|
with lock:
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
def run_import(user_id: int) -> dict:
|
||||||
|
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||||
|
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||||
|
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
||||||
|
with _one_import(user_id), SessionLocal() as db:
|
||||||
|
user = db.get(User, user_id)
|
||||||
|
if user is None:
|
||||||
|
return {"skipped": "no user"}
|
||||||
|
result = service.apply_takeout_watched(db, user)
|
||||||
|
db.commit()
|
||||||
|
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||||
|
if waiting:
|
||||||
|
fetcher = get_service_user(db)
|
||||||
|
if fetcher is not None:
|
||||||
|
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||||
|
with YouTubeClient(db, fetcher) as yt:
|
||||||
|
ensure_videos(db, yt, [vid for _, vid in waiting])
|
||||||
|
result |= service.apply_takeout_watch_later(db, user)
|
||||||
|
db.commit()
|
||||||
|
outcomes = queries.takeout_outcomes(db, user)
|
||||||
|
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||||
|
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||||
|
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
||||||
|
return result
|
||||||
@@ -1,18 +1,25 @@
|
|||||||
from datetime import date
|
from datetime import date
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy.orm.exc import StaleDataError
|
from sqlalchemy.orm.exc import StaleDataError
|
||||||
|
|
||||||
from app import quota
|
from app import quota, scheduler
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db, release_read_connection
|
||||||
from app.mailtext import lang_for_user
|
from app.mailtext import lang_for_user
|
||||||
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.reader import queries, service
|
from app.reader import queries, service, takeout
|
||||||
from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal
|
from app.routes._common import (
|
||||||
|
ProgressIn,
|
||||||
|
capped_body,
|
||||||
|
kept_position,
|
||||||
|
not_found,
|
||||||
|
youtube_refusal,
|
||||||
|
)
|
||||||
from app.sync.videos import parse_iso8601_duration
|
from app.sync.videos import parse_iso8601_duration
|
||||||
from app.utils import iso
|
from app.utils import iso
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
@@ -293,3 +300,49 @@ def get_video_detail(
|
|||||||
items[0].get("contentDetails", {}).get("duration")
|
items[0].get("contentDetails", {}).get("duration")
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- Google Takeout ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
||||||
|
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
|
||||||
|
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
|
||||||
|
release_read_connection(db)
|
||||||
|
return await capped_body(request, takeout.MAX_BYTES)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/takeout")
|
||||||
|
def upload_takeout(
|
||||||
|
# The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before
|
||||||
|
# a byte of it is read.
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
body: bytes = Depends(_takeout_body),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
"""One file of the user's Google Takeout export — the watch history or the Watch later list.
|
||||||
|
Its videos are kept and the import runs in the background; its outcome reaches the user's inbox."""
|
||||||
|
try:
|
||||||
|
parsed = takeout.parse(body)
|
||||||
|
except takeout.TakeoutError as exc:
|
||||||
|
if exc.reason == "no_videos":
|
||||||
|
raise refused(422, "takeout_no_videos") from None
|
||||||
|
raise refused(422, "takeout_unreadable") from None
|
||||||
|
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
||||||
|
db.commit()
|
||||||
|
scheduler.run_ad_hoc(
|
||||||
|
scheduler.TAKEOUT_IMPORT,
|
||||||
|
partial(takeout.run_import, user.id),
|
||||||
|
actor_id=user.id,
|
||||||
|
target=f"user:{user.id}",
|
||||||
|
notify=True,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"kind": parsed.kind,
|
||||||
|
"videos": len(parsed.videos),
|
||||||
|
"entries": parsed.entries,
|
||||||
|
"ads": parsed.ads,
|
||||||
|
"other": parsed.other,
|
||||||
|
"malformed": parsed.malformed,
|
||||||
|
}
|
||||||
|
|||||||
+29
-10
@@ -118,8 +118,9 @@ def _notify_done(
|
|||||||
summary: str | None,
|
summary: str | None,
|
||||||
result: dict | None = None,
|
result: dict | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
"""Post a completion notice to the inbox of who started the run: an admin's manual run, or a
|
||||||
effort — a notification failure must never mask the job's own outcome."""
|
user's ad-hoc run that asks for one. Best effort — a notification failure must never mask the
|
||||||
|
job's own outcome."""
|
||||||
try:
|
try:
|
||||||
create_notification(
|
create_notification(
|
||||||
db,
|
db,
|
||||||
@@ -608,14 +609,19 @@ def trigger_all(actor_id: int | None = None) -> list[str]:
|
|||||||
# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and
|
# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and
|
||||||
# spent is in `job_runs` like any job's, not a background task only the log ever heard of.
|
# spent is in `job_runs` like any job's, not a background task only the log ever heard of.
|
||||||
CHANNEL_BACKFILL = "channel_backfill"
|
CHANNEL_BACKFILL = "channel_backfill"
|
||||||
|
# A user's Google Takeout import (`reader.takeout`): its target is the user.
|
||||||
|
TAKEOUT_IMPORT = "takeout_import"
|
||||||
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
|
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
|
||||||
AD_HOC_JOBS = (CHANNEL_BACKFILL,)
|
AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT)
|
||||||
|
|
||||||
|
|
||||||
def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None:
|
def run_ad_hoc(
|
||||||
|
job: str, fn: Callable[[], dict], *, actor_id: int, target: str, notify: bool = False
|
||||||
|
) -> None:
|
||||||
"""Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
|
"""Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
|
||||||
own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
|
own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
|
||||||
a second request before the first has even started is that same run."""
|
a second request before the first has even started is that same run. `notify` posts its outcome
|
||||||
|
to the user's inbox, for work whose page does not watch it run."""
|
||||||
if job not in AD_HOC_JOBS:
|
if job not in AD_HOC_JOBS:
|
||||||
raise KeyError(job) # an unnamed job would show on the page as its bare id
|
raise KeyError(job) # an unnamed job would show on the page as its bare id
|
||||||
sched = _scheduler
|
sched = _scheduler
|
||||||
@@ -626,7 +632,7 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str)
|
|||||||
sched.add_job(
|
sched.add_job(
|
||||||
_ad_hoc,
|
_ad_hoc,
|
||||||
"date",
|
"date",
|
||||||
args=(job, fn, actor_id, target),
|
args=(job, fn, actor_id, target, notify),
|
||||||
id=f"{job}@{target}",
|
id=f"{job}@{target}",
|
||||||
misfire_grace_time=None,
|
misfire_grace_time=None,
|
||||||
)
|
)
|
||||||
@@ -635,7 +641,9 @@ def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str)
|
|||||||
log.info("ad-hoc %s for %s already queued", job, target)
|
log.info("ad-hoc %s for %s already queued", job, target)
|
||||||
|
|
||||||
|
|
||||||
def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None:
|
def _ad_hoc(
|
||||||
|
job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False
|
||||||
|
) -> None:
|
||||||
try:
|
try:
|
||||||
run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
|
run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -644,12 +652,12 @@ def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> Non
|
|||||||
return
|
return
|
||||||
_opened(run_id, job)
|
_opened(run_id, job)
|
||||||
try:
|
try:
|
||||||
_ad_hoc_work(job, fn, run_id)
|
_ad_hoc_work(job, fn, run_id, actor_id if notify else None)
|
||||||
finally:
|
finally:
|
||||||
_closed(run_id)
|
_closed(run_id)
|
||||||
|
|
||||||
|
|
||||||
def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None:
|
def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | None) -> None:
|
||||||
with logs.bind(run_id=run_id):
|
with logs.bind(run_id=run_id):
|
||||||
# As in _run: only the work is inside the try.
|
# As in _run: only the work is inside the try.
|
||||||
try:
|
try:
|
||||||
@@ -658,8 +666,19 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None:
|
|||||||
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
||||||
err, failure = _failure(exc)
|
err, failure = _failure(exc)
|
||||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||||
|
_notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure)
|
||||||
return
|
return
|
||||||
_finish_ok(job, run_id, result)
|
summary, data = _finish_ok(job, run_id, result)
|
||||||
|
_notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data)
|
||||||
|
|
||||||
|
|
||||||
|
def _notify_ad_hoc(
|
||||||
|
user_id: int | None, job: str, status: RunStatus, summary: str | None, result: dict | None
|
||||||
|
) -> None:
|
||||||
|
if user_id is None:
|
||||||
|
return
|
||||||
|
with SessionLocal() as db:
|
||||||
|
_notify_done(db, user_id, job, status, summary, result)
|
||||||
|
|
||||||
|
|
||||||
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
|
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
|
||||||
|
|||||||
@@ -221,6 +221,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
|
|||||||
CAPPED_BODY_SITES = {
|
CAPPED_BODY_SITES = {
|
||||||
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
||||||
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
||||||
|
("feed.py", "_takeout_body", "capped_body", "takeout.MAX_BYTES"),
|
||||||
("files.py", "spool_upload", "capped_stream_to", "cap"),
|
("files.py", "spool_upload", "capped_stream_to", "cap"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,362 @@
|
|||||||
|
"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the
|
||||||
|
import job makes of them — watched here (never a Short, never over a state the user set), waiting for
|
||||||
|
a video the catalog does not know yet, and the Watch later list joined to the user's own."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app import scheduler
|
||||||
|
from app.catalog.models import Channel, Video
|
||||||
|
from app.models import Notification
|
||||||
|
from app.reader import service, takeout
|
||||||
|
from app.reader.models import (
|
||||||
|
Playlist,
|
||||||
|
PlaylistItem,
|
||||||
|
TakeoutItem,
|
||||||
|
TakeoutKind,
|
||||||
|
VideoState,
|
||||||
|
)
|
||||||
|
|
||||||
|
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||||||
|
CHANNEL = "UC" + "a" * 22
|
||||||
|
|
||||||
|
|
||||||
|
def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra):
|
||||||
|
entry = {
|
||||||
|
"header": "YouTube",
|
||||||
|
"title": "Megnézted: valami", # the account's language: never read
|
||||||
|
"titleUrl": f"https://{host}.youtube.com/watch?v={vid}",
|
||||||
|
"time": at.isoformat().replace("+00:00", "Z"),
|
||||||
|
"products": ["YouTube"],
|
||||||
|
"activityControls": ["YouTube watch history"],
|
||||||
|
}
|
||||||
|
if channel:
|
||||||
|
entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}]
|
||||||
|
return entry | extra
|
||||||
|
|
||||||
|
|
||||||
|
def _history(*entries) -> bytes:
|
||||||
|
return json.dumps(list(entries)).encode()
|
||||||
|
|
||||||
|
|
||||||
|
# --- reading the files -------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel():
|
||||||
|
later = T0 + timedelta(days=3)
|
||||||
|
parsed = takeout.parse(
|
||||||
|
_history(
|
||||||
|
_view("aaaaaaaaaaa", later),
|
||||||
|
_view("aaaaaaaaaaa", T0),
|
||||||
|
_view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel
|
||||||
|
_view("ccccccccccc", T0, host="music"),
|
||||||
|
# Out of order: the later view still wins.
|
||||||
|
_view("ddddddddddd", T0, channel=None),
|
||||||
|
_view("ddddddddddd", later),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert parsed.kind == TakeoutKind.WATCHED
|
||||||
|
assert parsed.entries == 6
|
||||||
|
assert parsed.videos == {
|
||||||
|
"aaaaaaaaaaa": (later, CHANNEL),
|
||||||
|
"bbbbbbbbbbb": (T0, None),
|
||||||
|
"ccccccccccc": (T0, CHANNEL),
|
||||||
|
"ddddddddddd": (later, CHANNEL),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_history_counts_what_it_leaves_out():
|
||||||
|
ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}])
|
||||||
|
post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"}
|
||||||
|
no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"}
|
||||||
|
bad_time = _view("ggggggggggg") | {"time": "yesterday"}
|
||||||
|
no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"}
|
||||||
|
parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk"))
|
||||||
|
assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2)
|
||||||
|
assert parsed.videos == {
|
||||||
|
"hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none():
|
||||||
|
body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n"
|
||||||
|
parsed = takeout.parse(("" + body).encode())
|
||||||
|
assert parsed.kind == TakeoutKind.WATCH_LATER
|
||||||
|
assert parsed.videos == {
|
||||||
|
"Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None)
|
||||||
|
}
|
||||||
|
assert parsed.malformed == 1
|
||||||
|
headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n")
|
||||||
|
assert list(headless.videos) == ["Ngb04xUK8uU"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("body", "reason"),
|
||||||
|
[
|
||||||
|
(b'{"some": "other", "export": "file"}', "not_takeout"),
|
||||||
|
(b"<html><body>watch history, as a page</body></html>", "not_takeout"),
|
||||||
|
(b"[not json", "not_takeout"),
|
||||||
|
(b"\xff\xfe\x00", "not_takeout"),
|
||||||
|
(b"one column only\n", "not_takeout"),
|
||||||
|
(b"[]", "no_videos"),
|
||||||
|
(
|
||||||
|
json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(),
|
||||||
|
"no_videos",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason):
|
||||||
|
with pytest.raises(takeout.TakeoutError) as caught:
|
||||||
|
takeout.parse(body)
|
||||||
|
assert caught.value.reason == reason
|
||||||
|
|
||||||
|
|
||||||
|
# --- keeping and applying ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _video(db, vid: str, *, probed=True, short=False):
|
||||||
|
if db.get(Channel, CHANNEL) is None:
|
||||||
|
db.add(Channel(id=CHANNEL, title="Ch"))
|
||||||
|
db.flush()
|
||||||
|
db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short))
|
||||||
|
|
||||||
|
|
||||||
|
def _state(db, user, vid, status, *, changed, position=0, watched_at=None):
|
||||||
|
db.add(
|
||||||
|
VideoState(
|
||||||
|
user_id=user.id,
|
||||||
|
video_id=vid,
|
||||||
|
status=status,
|
||||||
|
position_seconds=position,
|
||||||
|
watched_at=watched_at,
|
||||||
|
updated_at=changed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _rows(db, user) -> dict[str, TakeoutItem]:
|
||||||
|
db.expire_all()
|
||||||
|
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||||||
|
return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def _states(db, user) -> dict[str, VideoState]:
|
||||||
|
db.expire_all()
|
||||||
|
rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars()
|
||||||
|
return {r.video_id: r for r in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept(
|
||||||
|
db, make_user
|
||||||
|
):
|
||||||
|
user = make_user("takeout@example.com")
|
||||||
|
for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"):
|
||||||
|
_video(db, vid, probed=vid != "unprobed000", short=vid == "short000000")
|
||||||
|
for vid in ("newer000000", "older000000"):
|
||||||
|
_video(db, vid)
|
||||||
|
earlier = T0 - timedelta(days=30)
|
||||||
|
_state(db, user, "hidden00000", "hidden", changed=earlier)
|
||||||
|
_state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier)
|
||||||
|
# In progress here, touched after the YouTube view: what the user did here is newer.
|
||||||
|
_state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40)
|
||||||
|
# In progress here, but last touched before the YouTube view: they finished it there.
|
||||||
|
_state(db, user, "older000000", "new", changed=earlier, position=40)
|
||||||
|
db.commit()
|
||||||
|
ids = [
|
||||||
|
"plain000000",
|
||||||
|
"short000000",
|
||||||
|
"unprobed000",
|
||||||
|
"unknown0000",
|
||||||
|
"hidden00000",
|
||||||
|
"watched0000",
|
||||||
|
"newer000000",
|
||||||
|
"older000000",
|
||||||
|
]
|
||||||
|
service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids})
|
||||||
|
|
||||||
|
counts = service.apply_takeout_watched(db, user)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
assert counts == {"watched": 2, "kept": 3, "shorts": 1}
|
||||||
|
rows, states = _rows(db, user), _states(db, user)
|
||||||
|
assert {v: rows[v].outcome for v in ids} == {
|
||||||
|
"plain000000": "watched",
|
||||||
|
"short000000": "short",
|
||||||
|
"unprobed000": None,
|
||||||
|
"unknown0000": None,
|
||||||
|
"hidden00000": "kept",
|
||||||
|
"watched0000": "kept",
|
||||||
|
"newer000000": "kept",
|
||||||
|
"older000000": "watched",
|
||||||
|
}
|
||||||
|
assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0)
|
||||||
|
assert "short000000" not in states
|
||||||
|
assert states["hidden00000"].status == "hidden"
|
||||||
|
assert states["watched0000"].watched_at == earlier
|
||||||
|
assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40)
|
||||||
|
assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user):
|
||||||
|
user = make_user("again@example.com")
|
||||||
|
_video(db, "plain000000")
|
||||||
|
db.commit()
|
||||||
|
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||||||
|
service.apply_takeout_watched(db, user)
|
||||||
|
later = T0 + timedelta(days=1)
|
||||||
|
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)})
|
||||||
|
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||||||
|
db.commit()
|
||||||
|
row = _rows(db, user)["plain000000"]
|
||||||
|
assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user):
|
||||||
|
user = make_user("later@example.com")
|
||||||
|
for vid in ("held0000000", "old00000000", "new00000000"):
|
||||||
|
_video(db, vid)
|
||||||
|
db.commit()
|
||||||
|
pl = service.watch_later(db, user)
|
||||||
|
service.add_item(db, pl, "held0000000", mark_dirty=False)
|
||||||
|
service.stage_takeout(
|
||||||
|
db,
|
||||||
|
user,
|
||||||
|
TakeoutKind.WATCH_LATER,
|
||||||
|
{
|
||||||
|
"new00000000": (T0, None),
|
||||||
|
"held0000000": (T0, None),
|
||||||
|
"old00000000": (T0 - timedelta(days=9), None),
|
||||||
|
"gone0000000": (T0, None), # YouTube no longer has it
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
counts = service.apply_takeout_watch_later(db, user)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1}
|
||||||
|
items = db.execute(
|
||||||
|
select(PlaylistItem.video_id)
|
||||||
|
.where(PlaylistItem.playlist_id == pl.id)
|
||||||
|
.order_by(PlaylistItem.position)
|
||||||
|
).scalars()
|
||||||
|
assert list(items) == ["held0000000", "old00000000", "new00000000"]
|
||||||
|
assert _rows(db, user)["WL:gone0000000"].outcome == "missing"
|
||||||
|
assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits(
|
||||||
|
db, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user("job@example.com")
|
||||||
|
_video(db, "plain000000")
|
||||||
|
db.commit()
|
||||||
|
service.stage_takeout(
|
||||||
|
db,
|
||||||
|
user,
|
||||||
|
TakeoutKind.WATCHED,
|
||||||
|
{"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)},
|
||||||
|
)
|
||||||
|
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)})
|
||||||
|
db.commit()
|
||||||
|
asked = []
|
||||||
|
|
||||||
|
def fake_ensure(session, yt, ids):
|
||||||
|
asked.append(ids)
|
||||||
|
_video(session, "fetched0000")
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
monkeypatch.setattr(takeout, "ensure_videos", fake_ensure)
|
||||||
|
monkeypatch.setattr(takeout, "get_service_user", lambda session: user)
|
||||||
|
|
||||||
|
result = takeout.run_import(user.id)
|
||||||
|
|
||||||
|
assert asked == [["fetched0000"]]
|
||||||
|
assert result == {
|
||||||
|
"watched": 1,
|
||||||
|
"kept": 0,
|
||||||
|
"shorts": 0,
|
||||||
|
"watch_later_added": 1,
|
||||||
|
"watch_later_present": 0,
|
||||||
|
"watch_later_missing": 0,
|
||||||
|
"waiting": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- the upload --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def queued(monkeypatch):
|
||||||
|
runs = []
|
||||||
|
monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw)))
|
||||||
|
return runs
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice(
|
||||||
|
db, make_user, as_user, queued
|
||||||
|
):
|
||||||
|
user = make_user("upload@example.com")
|
||||||
|
body = _history(
|
||||||
|
_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}])
|
||||||
|
)
|
||||||
|
resp = as_user(user).post("/api/takeout", content=body)
|
||||||
|
assert resp.status_code == 200, resp.text
|
||||||
|
assert resp.json() == {
|
||||||
|
"ok": True,
|
||||||
|
"kind": "watched",
|
||||||
|
"videos": 1,
|
||||||
|
"entries": 2,
|
||||||
|
"ads": 1,
|
||||||
|
"other": 0,
|
||||||
|
"malformed": 0,
|
||||||
|
}
|
||||||
|
assert list(_rows(db, user)) == ["aaaaaaaaaaa"]
|
||||||
|
assert queued == [
|
||||||
|
(
|
||||||
|
"takeout_import",
|
||||||
|
{"actor_id": user.id, "target": f"user:{user.id}", "notify": True},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("body", "reason"),
|
||||||
|
[(b"<html></html>", "takeout_unreadable"), (b"[]", "takeout_no_videos")],
|
||||||
|
)
|
||||||
|
def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language(
|
||||||
|
db, make_user, as_user, queued, body, reason
|
||||||
|
):
|
||||||
|
user = make_user("refused@example.com")
|
||||||
|
resp = as_user(user).post("/api/takeout", content=body)
|
||||||
|
assert resp.status_code == 422
|
||||||
|
assert resp.json()["detail"] == {"code": "refused", "reason": reason}
|
||||||
|
assert queued == [] and _rows(db, user) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch):
|
||||||
|
user = make_user("big@example.com")
|
||||||
|
monkeypatch.setattr(takeout, "MAX_BYTES", 64)
|
||||||
|
resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
|
||||||
|
assert resp.status_code == 413
|
||||||
|
assert _rows(db, user) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_anonymous_upload_is_refused(client):
|
||||||
|
assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401
|
||||||
|
|
||||||
|
|
||||||
|
# --- the notice --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user):
|
||||||
|
user = make_user("notice@example.com")
|
||||||
|
scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True)
|
||||||
|
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x")
|
||||||
|
db.expire_all()
|
||||||
|
notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars()
|
||||||
|
assert [(n.data["job_id"], n.data["result"]) for n in notices] == [
|
||||||
|
("takeout_import", {"watched": 3})
|
||||||
|
]
|
||||||
@@ -44,6 +44,7 @@ UNIT_MODELS = {
|
|||||||
"PlaylistItem",
|
"PlaylistItem",
|
||||||
"SavedView",
|
"SavedView",
|
||||||
"ComfortPin",
|
"ComfortPin",
|
||||||
|
"TakeoutItem",
|
||||||
"Tag",
|
"Tag",
|
||||||
"ChannelTag",
|
"ChannelTag",
|
||||||
},
|
},
|
||||||
@@ -360,6 +361,9 @@ UNIT_COMMITS = {
|
|||||||
"sync/playlists.py:pull_user_playlists",
|
"sync/playlists.py:pull_user_playlists",
|
||||||
"sync/playlists.py:merge_playlist",
|
"sync/playlists.py:merge_playlist",
|
||||||
"sync/playlists.py:repull_playlist",
|
"sync/playlists.py:repull_playlist",
|
||||||
|
# The Takeout import is a recorded run with its own session: the watched part lands before
|
||||||
|
# the Watch later part asks YouTube (which commits the videos it fetched).
|
||||||
|
"reader/takeout.py:run_import",
|
||||||
},
|
},
|
||||||
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
||||||
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
||||||
|
|||||||
@@ -249,6 +249,8 @@
|
|||||||
"api_token_preset_grants": "The Hub token gets its folder by itself — leave the folders out",
|
"api_token_preset_grants": "The Hub token gets its folder by itself — leave the folders out",
|
||||||
"api_token_no_grants": "Choose at least one folder the token may use",
|
"api_token_no_grants": "Choose at least one folder the token may use",
|
||||||
"api_token_name_required": "Give the token a name",
|
"api_token_name_required": "Give the token a name",
|
||||||
"api_token_limit": "You already have {{limit}} tokens — revoke one first"
|
"api_token_limit": "You already have {{limit}} tokens — revoke one first",
|
||||||
|
"takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.",
|
||||||
|
"takeout_no_videos": "This file has no videos to import"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,7 +80,8 @@
|
|||||||
"log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.",
|
"log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.",
|
||||||
"tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.",
|
"tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.",
|
||||||
"people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.",
|
"people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.",
|
||||||
"channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running)."
|
"channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running).",
|
||||||
|
"takeout_import": "A user's Google Takeout import: what they watched on YouTube becomes watched here (not Shorts, never over a state they set here), and their YouTube Watch later list joins theirs. A video the catalogue does not know yet waits, and is marked once it arrives."
|
||||||
},
|
},
|
||||||
"jobs": {
|
"jobs": {
|
||||||
"rss_poll": "RSS poll (new uploads)",
|
"rss_poll": "RSS poll (new uploads)",
|
||||||
@@ -100,7 +101,8 @@
|
|||||||
"audit_gc": "Activity and quota-history cleanup",
|
"audit_gc": "Activity and quota-history cleanup",
|
||||||
"log_gc": "Log cleanup",
|
"log_gc": "Log cleanup",
|
||||||
"people_enrich": "People details and portraits (Plex)",
|
"people_enrich": "People details and portraits (Plex)",
|
||||||
"channel_backfill": "A channel's full history"
|
"channel_backfill": "A channel's full history",
|
||||||
|
"takeout_import": "Google Takeout import"
|
||||||
},
|
},
|
||||||
"queue": {
|
"queue": {
|
||||||
"recentPending": "Channels to sync",
|
"recentPending": "Channels to sync",
|
||||||
@@ -178,7 +180,13 @@
|
|||||||
"deep": "deep",
|
"deep": "deep",
|
||||||
"bios": "biographies",
|
"bios": "biographies",
|
||||||
"portraits": "portraits",
|
"portraits": "portraits",
|
||||||
"cached": "cached"
|
"cached": "cached",
|
||||||
|
"kept": "kept as set here",
|
||||||
|
"waiting": "waiting for their video",
|
||||||
|
"watch_later_added": "added to Watch later",
|
||||||
|
"watch_later_present": "already in Watch later",
|
||||||
|
"watch_later_missing": "no longer on YouTube",
|
||||||
|
"watch_later_waiting": "Watch later waiting"
|
||||||
},
|
},
|
||||||
"skip": {
|
"skip": {
|
||||||
"disabled": "disabled",
|
"disabled": "disabled",
|
||||||
@@ -191,7 +199,8 @@
|
|||||||
"busy": "another sync is running",
|
"busy": "another sync is running",
|
||||||
"channelbusy": "this channel is already being backfilled",
|
"channelbusy": "this channel is already being backfilled",
|
||||||
"atcapacity": "two full-history backfills are already running — the scheduler finishes this one",
|
"atcapacity": "two full-history backfills are already running — the scheduler finishes this one",
|
||||||
"nothingtodo": "nothing left to fetch"
|
"nothingtodo": "nothing left to fetch",
|
||||||
|
"nouser": "the account is gone"
|
||||||
},
|
},
|
||||||
"failure": {
|
"failure": {
|
||||||
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
|
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
|
||||||
|
|||||||
@@ -249,6 +249,8 @@
|
|||||||
"api_token_preset_grants": "A Hub-token magától kapja a mappáját — ne adj meg mappát",
|
"api_token_preset_grants": "A Hub-token magától kapja a mappáját — ne adj meg mappát",
|
||||||
"api_token_no_grants": "Válassz legalább egy mappát, amelyet a token használhat",
|
"api_token_no_grants": "Válassz legalább egy mappát, amelyet a token használhat",
|
||||||
"api_token_name_required": "Adj nevet a tokennek",
|
"api_token_name_required": "Adj nevet a tokennek",
|
||||||
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet"
|
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet",
|
||||||
|
"takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.",
|
||||||
|
"takeout_no_videos": "Ebben a fájlban nincs importálható videó"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,7 +80,8 @@
|
|||||||
"log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.",
|
"log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.",
|
||||||
"tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.",
|
"tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.",
|
||||||
"people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.",
|
"people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.",
|
||||||
"channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be."
|
"channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be.",
|
||||||
|
"takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül az övébe. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz."
|
||||||
},
|
},
|
||||||
"jobs": {
|
"jobs": {
|
||||||
"rss_poll": "RSS-lekérdezés (új feltöltések)",
|
"rss_poll": "RSS-lekérdezés (új feltöltések)",
|
||||||
@@ -100,7 +101,8 @@
|
|||||||
"audit_gc": "Napló- és kvótaelőzmény-takarítás",
|
"audit_gc": "Napló- és kvótaelőzmény-takarítás",
|
||||||
"log_gc": "Naplótakarítás",
|
"log_gc": "Naplótakarítás",
|
||||||
"people_enrich": "Személyadatok és portrék (Plex)",
|
"people_enrich": "Személyadatok és portrék (Plex)",
|
||||||
"channel_backfill": "Egy csatorna teljes előzménye"
|
"channel_backfill": "Egy csatorna teljes előzménye",
|
||||||
|
"takeout_import": "Google Takeout import"
|
||||||
},
|
},
|
||||||
"queue": {
|
"queue": {
|
||||||
"recentPending": "Szinkronra váró csatorna",
|
"recentPending": "Szinkronra váró csatorna",
|
||||||
@@ -178,7 +180,13 @@
|
|||||||
"deep": "mély",
|
"deep": "mély",
|
||||||
"bios": "életrajzok",
|
"bios": "életrajzok",
|
||||||
"portraits": "portrék",
|
"portraits": "portrék",
|
||||||
"cached": "elmentve"
|
"cached": "elmentve",
|
||||||
|
"kept": "itteni állapot megtartva",
|
||||||
|
"waiting": "a videójára vár",
|
||||||
|
"watch_later_added": "Watch laterbe téve",
|
||||||
|
"watch_later_present": "már a Watch laterben",
|
||||||
|
"watch_later_missing": "már nincs a YouTube-on",
|
||||||
|
"watch_later_waiting": "Watch later vár"
|
||||||
},
|
},
|
||||||
"skip": {
|
"skip": {
|
||||||
"disabled": "kikapcsolva",
|
"disabled": "kikapcsolva",
|
||||||
@@ -191,7 +199,8 @@
|
|||||||
"busy": "egy másik szinkron fut",
|
"busy": "egy másik szinkron fut",
|
||||||
"channelbusy": "ennek a csatornának a betöltése már fut",
|
"channelbusy": "ennek a csatornának a betöltése már fut",
|
||||||
"atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be",
|
"atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be",
|
||||||
"nothingtodo": "nincs több letöltendő"
|
"nothingtodo": "nincs több letöltendő",
|
||||||
|
"nouser": "a fiók már nem létezik"
|
||||||
},
|
},
|
||||||
"failure": {
|
"failure": {
|
||||||
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
||||||
|
|||||||
Reference in New Issue
Block a user