feat(reader): a Google Takeout export becomes watch state and Watch later
The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
|
||||
for the other units that ask about them.
|
||||
"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP
|
||||
layer and for the other units that ask about them.
|
||||
|
||||
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
||||
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
||||
@@ -7,6 +7,8 @@ length, whether a video exists); another unit asks the questions at the end of t
|
||||
than joining Reader's tables itself.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||
|
||||
@@ -14,7 +16,16 @@ from app.collation import collated
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.reader.models import (
|
||||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SavedView,
|
||||
Tag,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
VideoState,
|
||||
)
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
# --- catalog, read along the edge ------------------------------------------------------------
|
||||
@@ -359,3 +370,55 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
|
||||
owned_by(ChannelTag.user_id, user),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- Google Takeout --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
|
||||
"""The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
|
||||
(row id, video id, Takeout time, is Short)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short)
|
||||
.join(Video, Video.id == TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
Video.shorts_probed,
|
||||
)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]:
|
||||
"""The user's rows of `kind` the import has not settled, oldest first: (row id, video id)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == kind,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
)
|
||||
.order_by(TakeoutItem.at, TakeoutItem.id)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
|
||||
"""Which of `ids` the catalog holds."""
|
||||
if not ids:
|
||||
return set()
|
||||
return set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars())
|
||||
|
||||
|
||||
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
|
||||
"""How many of the user's rows each kind and outcome holds (None = waiting)."""
|
||||
rows = db.execute(
|
||||
select(TakeoutItem.kind, TakeoutItem.outcome, func.count())
|
||||
.where(owned_by(TakeoutItem.user_id, user))
|
||||
.group_by(TakeoutItem.kind, TakeoutItem.outcome)
|
||||
).all()
|
||||
return {(kind, outcome): n for kind, outcome, n in rows}
|
||||
|
||||
Reference in New Issue
Block a user