feat(reader): a Google Takeout export becomes watch state and Watch later

The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout
export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or
Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the
user's `takeout_import` run, whose outcome reaches their inbox.

The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short
is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by
the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later
videos are fetched if unknown and appended oldest first.

An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
2026-10-23 17:04:54 +02:00
parent bf5ceec8a1
commit 7b8505af95
15 changed files with 978 additions and 29 deletions
+66 -3
View File
@@ -1,5 +1,5 @@
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
for the other units that ask about them.
"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP
layer and for the other units that ask about them.
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
@@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t
than joining Reader's tables itself.
"""
from datetime import datetime
from sqlalchemy import ColumnElement, and_, func, or_, select
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
@@ -14,7 +16,16 @@ from app.collation import collated
from app.feedquery import feed_columns, serialize
from app.models import User
from app.catalog.models import Channel, Subscription, Video
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
from app.reader.models import (
ChannelTag,
Playlist,
PlaylistItem,
SavedView,
Tag,
TakeoutItem,
TakeoutKind,
VideoState,
)
from app.userscope import owned_by, owned_row
# --- catalog, read along the edge ------------------------------------------------------------
@@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
owned_by(ChannelTag.user_id, user),
)
).scalar_one_or_none()
# --- Google Takeout --------------------------------------------------------------------------
def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
"""The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
(row id, video id, Takeout time, is Short)."""
return list(
db.execute(
select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short)
.join(Video, Video.id == TakeoutItem.video_id)
.where(
owned_by(TakeoutItem.user_id, user),
TakeoutItem.kind == TakeoutKind.WATCHED,
TakeoutItem.outcome.is_(None),
Video.shorts_probed,
)
).tuples()
)
def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]:
"""The user's rows of `kind` the import has not settled, oldest first: (row id, video id)."""
return list(
db.execute(
select(TakeoutItem.id, TakeoutItem.video_id)
.where(
owned_by(TakeoutItem.user_id, user),
TakeoutItem.kind == kind,
TakeoutItem.outcome.is_(None),
)
.order_by(TakeoutItem.at, TakeoutItem.id)
).tuples()
)
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
"""Which of `ids` the catalog holds."""
if not ids:
return set()
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
"""How many of the user's rows each kind and outcome holds (None = waiting)."""
rows = db.execute(
select(TakeoutItem.kind, TakeoutItem.outcome, func.count())
.where(owned_by(TakeoutItem.user_id, user))
.group_by(TakeoutItem.kind, TakeoutItem.outcome)
).all()
return {(kind, outcome): n for kind, outcome, n in rows}