feat(reader): a Google Takeout export becomes watch state and Watch later
The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
@@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th
|
||||
edge, or asks this unit's service."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
@@ -213,3 +214,51 @@ class ComfortPin(Base, TimestampMixin):
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
|
||||
class TakeoutKind(StrEnum):
|
||||
"""What a Google Takeout row says the user did with a video."""
|
||||
|
||||
WATCHED = "watched" # history/watch-history.json
|
||||
WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv
|
||||
|
||||
|
||||
class TakeoutOutcome(StrEnum):
|
||||
"""What the import did with a row; NULL = still waiting for its video to be known (and, for a
|
||||
watched one, probed for Shorts)."""
|
||||
|
||||
WATCHED = "watched" # the video is marked watched at the Takeout time
|
||||
KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays
|
||||
SHORT = "short" # a Short: swiping one past is not watching it
|
||||
ADDED = "added" # added to Watch later
|
||||
PRESENT = "present" # already in Watch later
|
||||
MISSING = "missing" # YouTube no longer has the video
|
||||
|
||||
|
||||
class TakeoutItem(Base, TimestampMixin):
|
||||
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
||||
later list (with when it was added). The rows are both the import job's input and its memory: a
|
||||
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
||||
The id is not a foreign key for that reason. The channel is kept for a later "channels you
|
||||
watch but do not follow" list."""
|
||||
|
||||
__tablename__ = "takeout_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
||||
CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
||||
# The waiting rows, by video: what a newly known video looks up.
|
||||
Index(
|
||||
"ix_takeout_items_waiting",
|
||||
"video_id",
|
||||
postgresql_where=text("outcome IS NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
kind: Mapped[str] = mapped_column(String(12))
|
||||
video_id: Mapped[str] = mapped_column(String(16))
|
||||
channel_id: Mapped[str | None] = mapped_column(String(32))
|
||||
# Watched: the last time in the export; Watch later: when it was added.
|
||||
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
outcome: Mapped[str | None] = mapped_column(String(12))
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
|
||||
for the other units that ask about them.
|
||||
"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP
|
||||
layer and for the other units that ask about them.
|
||||
|
||||
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
||||
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
||||
@@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t
|
||||
than joining Reader's tables itself.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||
|
||||
@@ -14,7 +16,16 @@ from app.collation import collated
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.reader.models import (
|
||||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SavedView,
|
||||
Tag,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
VideoState,
|
||||
)
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
# --- catalog, read along the edge ------------------------------------------------------------
|
||||
@@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
|
||||
owned_by(ChannelTag.user_id, user),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- Google Takeout --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
|
||||
"""The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
|
||||
(row id, video id, Takeout time, is Short)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short)
|
||||
.join(Video, Video.id == TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
Video.shorts_probed,
|
||||
)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]:
|
||||
"""The user's rows of `kind` the import has not settled, oldest first: (row id, video id)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == kind,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
)
|
||||
.order_by(TakeoutItem.at, TakeoutItem.id)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
|
||||
"""Which of `ids` the catalog holds."""
|
||||
if not ids:
|
||||
return set()
|
||||
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
|
||||
|
||||
|
||||
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
|
||||
"""How many of the user's rows each kind and outcome holds (None = waiting)."""
|
||||
rows = db.execute(
|
||||
select(TakeoutItem.kind, TakeoutItem.outcome, func.count())
|
||||
.where(owned_by(TakeoutItem.user_id, user))
|
||||
.group_by(TakeoutItem.kind, TakeoutItem.outcome)
|
||||
).all()
|
||||
return {(kind, outcome): n for kind, outcome, n in rows}
|
||||
|
||||
@@ -15,7 +15,17 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import User
|
||||
from app.reader import queries
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.reader.models import (
|
||||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SavedView,
|
||||
Tag,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
TakeoutOutcome,
|
||||
VideoState,
|
||||
)
|
||||
from app.sync.playlists import lock_playlists, recompute_dirty
|
||||
from app.userscope import owned_by
|
||||
|
||||
@@ -495,3 +505,113 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- Google Takeout --------------------------------------------------------------------------
|
||||
|
||||
_TAKEOUT_BATCH = 1000
|
||||
|
||||
|
||||
def stage_takeout(
|
||||
db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]]
|
||||
) -> None:
|
||||
"""Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job.
|
||||
A second upload of the same export, or a newer one, adds what is new: a video already kept keeps
|
||||
the later time, and a row the import has settled keeps its outcome."""
|
||||
values = [
|
||||
{"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel}
|
||||
for vid, (at, channel) in rows.items()
|
||||
]
|
||||
for start in range(0, len(values), _TAKEOUT_BATCH):
|
||||
stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH])
|
||||
db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
constraint="uq_takeout_item",
|
||||
set_={
|
||||
"at": func.greatest(TakeoutItem.at, stmt.excluded.at),
|
||||
"channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id),
|
||||
},
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
|
||||
for start in range(0, len(ids), _TAKEOUT_BATCH):
|
||||
db.execute(
|
||||
update(TakeoutItem)
|
||||
.where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH]))
|
||||
.values(outcome=outcome)
|
||||
)
|
||||
|
||||
|
||||
def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
|
||||
"""Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
|
||||
for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
|
||||
past is not watching it), and a state the user set here wins: watched or hidden, or anything
|
||||
changed after the Takeout time. The upsert's own condition is that rule, so a state set while
|
||||
the import runs wins too."""
|
||||
short: list[int] = []
|
||||
todo: dict[str, tuple[int, datetime]] = {}
|
||||
for row_id, video_id, at, is_short in queries.takeout_watched_ready(db, user):
|
||||
if is_short:
|
||||
short.append(row_id)
|
||||
else:
|
||||
todo[video_id] = (row_id, at)
|
||||
marked: set[str] = set()
|
||||
items = list(todo.items())
|
||||
now = datetime.now(timezone.utc)
|
||||
for start in range(0, len(items), _TAKEOUT_BATCH):
|
||||
stmt = pg_insert(VideoState).values(
|
||||
[
|
||||
{"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at}
|
||||
for vid, (_, at) in items[start : start + _TAKEOUT_BATCH]
|
||||
]
|
||||
)
|
||||
marked.update(
|
||||
db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
constraint="uq_user_video",
|
||||
set_={
|
||||
"status": "watched",
|
||||
"watched_at": stmt.excluded.watched_at,
|
||||
"updated_at": now,
|
||||
},
|
||||
where=(VideoState.status == "new")
|
||||
& (VideoState.updated_at < stmt.excluded.watched_at),
|
||||
).returning(VideoState.video_id)
|
||||
).scalars()
|
||||
)
|
||||
kept = [row_id for vid, (row_id, _) in items if vid not in marked]
|
||||
_settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
|
||||
_settle(db, kept, TakeoutOutcome.KEPT)
|
||||
_settle(db, short, TakeoutOutcome.SHORT)
|
||||
db.flush()
|
||||
return {"watched": len(marked), "kept": len(kept), "shorts": len(short)}
|
||||
|
||||
|
||||
def apply_takeout_watch_later(db: Session, user: User) -> dict[str, int]:
|
||||
"""Put the videos of the user's YouTube Watch later list into theirs here, oldest first, after
|
||||
what it already holds. Called once the catalog has fetched what it could: a video still unknown
|
||||
is one YouTube no longer has."""
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if not waiting:
|
||||
return {}
|
||||
known = queries.known_video_ids(db, [vid for _, vid in waiting])
|
||||
pl = watch_later(db, user)
|
||||
done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
|
||||
for row_id, vid in waiting:
|
||||
if vid not in known:
|
||||
done[TakeoutOutcome.MISSING].append(row_id)
|
||||
elif add_item(db, pl, vid, mark_dirty=False):
|
||||
done[TakeoutOutcome.ADDED].append(row_id)
|
||||
else:
|
||||
done[TakeoutOutcome.PRESENT].append(row_id)
|
||||
for outcome, ids in done.items():
|
||||
_settle(db, ids, outcome)
|
||||
db.flush()
|
||||
return {
|
||||
"watch_later_added": len(done[TakeoutOutcome.ADDED]),
|
||||
"watch_later_present": len(done[TakeoutOutcome.PRESENT]),
|
||||
"watch_later_missing": len(done[TakeoutOutcome.MISSING]),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
||||
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
||||
only way in.
|
||||
|
||||
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
||||
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
||||
what it can; a video the catalog does not know yet waits for it.
|
||||
|
||||
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
||||
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
||||
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
||||
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
||||
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
||||
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
||||
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from app import quota
|
||||
from app.bodylimit import TAKEOUT_MAX_BYTES
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.reader.models import TakeoutKind
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.videos import ensure_videos
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
# The largest file the upload takes (its edge cap and the route's own are one number).
|
||||
MAX_BYTES = TAKEOUT_MAX_BYTES
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
_WATCH_URL = re.compile(
|
||||
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
||||
)
|
||||
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
||||
|
||||
|
||||
class TakeoutError(ValueError):
|
||||
"""The file is not one the import reads; `reason` is the client's word for why."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass
|
||||
class Parsed:
|
||||
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
||||
|
||||
kind: TakeoutKind
|
||||
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
||||
entries: int = 0
|
||||
ads: int = 0
|
||||
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
||||
malformed: int = 0
|
||||
|
||||
|
||||
def _time(raw: object) -> datetime | None:
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
try:
|
||||
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo is not None else None
|
||||
|
||||
|
||||
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
||||
old = parsed.videos.get(vid)
|
||||
if old is None or at > old[0]:
|
||||
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
||||
|
||||
|
||||
def _history(data: list) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCHED)
|
||||
for entry in data:
|
||||
parsed.entries += 1
|
||||
if not isinstance(entry, dict):
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
details = entry.get("details")
|
||||
if isinstance(details, list) and any(
|
||||
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
||||
):
|
||||
parsed.ads += 1
|
||||
continue
|
||||
url = entry.get("titleUrl")
|
||||
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
||||
if match is None:
|
||||
parsed.other += 1
|
||||
continue
|
||||
at = _time(entry.get("time"))
|
||||
if at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
channel = None
|
||||
subtitles = entry.get("subtitles")
|
||||
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
||||
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
||||
channel = found.group(1) if found else None
|
||||
_keep(parsed, match.group(1), at, channel)
|
||||
return parsed
|
||||
|
||||
|
||||
def _watch_later(text: str) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
||||
rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
|
||||
if not rows or len(rows[0]) < 2:
|
||||
raise TakeoutError("not_takeout")
|
||||
if not _VIDEO_ID.match(rows[0][0].strip()):
|
||||
rows = rows[1:] # the header, in the account's language
|
||||
for row in rows:
|
||||
parsed.entries += 1
|
||||
vid = row[0].strip() if row else ""
|
||||
at = _time(row[1].strip()) if len(row) > 1 else None
|
||||
if not _VIDEO_ID.match(vid) or at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
_keep(parsed, vid, at, None)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse(body: bytes) -> Parsed:
|
||||
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
||||
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
||||
with nothing to import in it."""
|
||||
try:
|
||||
text = body.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith("["):
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
if not isinstance(data, list):
|
||||
raise TakeoutError("not_takeout")
|
||||
parsed = _history(data)
|
||||
elif stripped.startswith(("{", "<")):
|
||||
# Another JSON file of the export, or the history's HTML variant.
|
||||
raise TakeoutError("not_takeout")
|
||||
else:
|
||||
parsed = _watch_later(stripped)
|
||||
if not parsed.videos:
|
||||
raise TakeoutError("no_videos")
|
||||
return parsed
|
||||
|
||||
|
||||
# One import per user at a time: a second file uploaded while the first one's run is under way
|
||||
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
|
||||
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
|
||||
# so a lock in this module holds it (as the channel backfill's does).
|
||||
_user_locks: dict[int, threading.Lock] = {}
|
||||
_user_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _one_import(user_id: int) -> Iterator[None]:
|
||||
with _user_locks_guard:
|
||||
lock = _user_locks.setdefault(user_id, threading.Lock())
|
||||
with lock:
|
||||
yield
|
||||
|
||||
|
||||
def run_import(user_id: int) -> dict:
|
||||
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
||||
with _one_import(user_id), SessionLocal() as db:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return {"skipped": "no user"}
|
||||
result = service.apply_takeout_watched(db, user)
|
||||
db.commit()
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if waiting:
|
||||
fetcher = get_service_user(db)
|
||||
if fetcher is not None:
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
ensure_videos(db, yt, [vid for _, vid in waiting])
|
||||
result |= service.apply_takeout_watch_later(db, user)
|
||||
db.commit()
|
||||
outcomes = queries.takeout_outcomes(db, user)
|
||||
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
||||
return result
|
||||
Reference in New Issue
Block a user