feat(reader): a Google Takeout export becomes watch state and Watch later
The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
||||
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
||||
only way in.
|
||||
|
||||
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
||||
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
||||
what it can; a video the catalog does not know yet waits for it.
|
||||
|
||||
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
||||
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
||||
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
||||
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
||||
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
||||
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
||||
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from app import quota
|
||||
from app.bodylimit import TAKEOUT_MAX_BYTES
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.reader.models import TakeoutKind
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.videos import ensure_videos
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
# The largest file the upload takes (its edge cap and the route's own are one number).
|
||||
MAX_BYTES = TAKEOUT_MAX_BYTES
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
_WATCH_URL = re.compile(
|
||||
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
||||
)
|
||||
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
||||
|
||||
|
||||
class TakeoutError(ValueError):
|
||||
"""The file is not one the import reads; `reason` is the client's word for why."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass
|
||||
class Parsed:
|
||||
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
||||
|
||||
kind: TakeoutKind
|
||||
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
||||
entries: int = 0
|
||||
ads: int = 0
|
||||
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
||||
malformed: int = 0
|
||||
|
||||
|
||||
def _time(raw: object) -> datetime | None:
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
try:
|
||||
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo is not None else None
|
||||
|
||||
|
||||
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
||||
old = parsed.videos.get(vid)
|
||||
if old is None or at > old[0]:
|
||||
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
||||
|
||||
|
||||
def _history(data: list) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCHED)
|
||||
for entry in data:
|
||||
parsed.entries += 1
|
||||
if not isinstance(entry, dict):
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
details = entry.get("details")
|
||||
if isinstance(details, list) and any(
|
||||
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
||||
):
|
||||
parsed.ads += 1
|
||||
continue
|
||||
url = entry.get("titleUrl")
|
||||
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
||||
if match is None:
|
||||
parsed.other += 1
|
||||
continue
|
||||
at = _time(entry.get("time"))
|
||||
if at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
channel = None
|
||||
subtitles = entry.get("subtitles")
|
||||
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
||||
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
||||
channel = found.group(1) if found else None
|
||||
_keep(parsed, match.group(1), at, channel)
|
||||
return parsed
|
||||
|
||||
|
||||
def _watch_later(text: str) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
||||
rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
|
||||
if not rows or len(rows[0]) < 2:
|
||||
raise TakeoutError("not_takeout")
|
||||
if not _VIDEO_ID.match(rows[0][0].strip()):
|
||||
rows = rows[1:] # the header, in the account's language
|
||||
for row in rows:
|
||||
parsed.entries += 1
|
||||
vid = row[0].strip() if row else ""
|
||||
at = _time(row[1].strip()) if len(row) > 1 else None
|
||||
if not _VIDEO_ID.match(vid) or at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
_keep(parsed, vid, at, None)
|
||||
return parsed
|
||||
|
||||
|
||||
def parse(body: bytes) -> Parsed:
|
||||
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
||||
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
||||
with nothing to import in it."""
|
||||
try:
|
||||
text = body.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
stripped = text.lstrip()
|
||||
if stripped.startswith("["):
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
raise TakeoutError("not_takeout") from None
|
||||
if not isinstance(data, list):
|
||||
raise TakeoutError("not_takeout")
|
||||
parsed = _history(data)
|
||||
elif stripped.startswith(("{", "<")):
|
||||
# Another JSON file of the export, or the history's HTML variant.
|
||||
raise TakeoutError("not_takeout")
|
||||
else:
|
||||
parsed = _watch_later(stripped)
|
||||
if not parsed.videos:
|
||||
raise TakeoutError("no_videos")
|
||||
return parsed
|
||||
|
||||
|
||||
# One import per user at a time: a second file uploaded while the first one's run is under way
|
||||
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
|
||||
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
|
||||
# so a lock in this module holds it (as the channel backfill's does).
|
||||
_user_locks: dict[int, threading.Lock] = {}
|
||||
_user_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _one_import(user_id: int) -> Iterator[None]:
|
||||
with _user_locks_guard:
|
||||
lock = _user_locks.setdefault(user_id, threading.Lock())
|
||||
with lock:
|
||||
yield
|
||||
|
||||
|
||||
def run_import(user_id: int) -> dict:
|
||||
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
||||
with _one_import(user_id), SessionLocal() as db:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return {"skipped": "no user"}
|
||||
result = service.apply_takeout_watched(db, user)
|
||||
db.commit()
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if waiting:
|
||||
fetcher = get_service_user(db)
|
||||
if fetcher is not None:
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
ensure_videos(db, yt, [vid for _, vid in waiting])
|
||||
result |= service.apply_takeout_watch_later(db, user)
|
||||
db.commit()
|
||||
outcomes = queries.takeout_outcomes(db, user)
|
||||
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
||||
return result
|
||||
Reference in New Issue
Block a user