peter
9f580fd8c8
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
324 lines
13 KiB
Python
324 lines
13 KiB
Python
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
|
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
|
only way in.
|
|
|
|
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
|
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
|
what it can; a video the catalog does not know yet waits for it.
|
|
|
|
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
|
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
|
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
|
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
|
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
|
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
|
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
|
"""
|
|
|
|
import csv
|
|
import io
|
|
import json
|
|
import re
|
|
import threading
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from itertools import chain
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import quota
|
|
from app.bodylimit import TAKEOUT_MAX_BYTES
|
|
from app.db import SessionLocal
|
|
from app.models import User
|
|
from app.ratelimit import RateLimiter
|
|
from app.reader import queries, service
|
|
from app.reader.models import TakeoutKind
|
|
from app.sync.runner import get_service_user
|
|
from app.sync.videos import ensure_videos
|
|
from app.youtube.client import YouTubeClient
|
|
|
|
# The largest file the upload takes (its edge cap and the route's own are one number).
|
|
MAX_BYTES = TAKEOUT_MAX_BYTES
|
|
# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
|
|
# looked up on the shared API budget, so a longer list is not one YouTube made.
|
|
WATCH_LATER_MAX = 5000
|
|
# The most Takeout videos one user keeps: twenty times the real export measured here (~9,000 videos
|
|
# across both files), so it only stops uploads of made-up ones piling up for good.
|
|
ROWS_MAX = 200_000
|
|
|
|
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
|
_WATCH_URL = re.compile(
|
|
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
|
)
|
|
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
|
|
|
|
|
class TakeoutError(ValueError):
|
|
"""The file is not one the import reads; `reason` is the client's word for why."""
|
|
|
|
def __init__(self, reason: str):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
|
|
|
|
@dataclass
|
|
class Parsed:
|
|
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
|
|
|
kind: TakeoutKind
|
|
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
|
entries: int = 0
|
|
ads: int = 0
|
|
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
|
malformed: int = 0
|
|
|
|
|
|
def _time(raw: object) -> datetime | None:
|
|
if not isinstance(raw, str):
|
|
return None
|
|
try:
|
|
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
|
except ValueError:
|
|
return None
|
|
return at if at.tzinfo is not None else None
|
|
|
|
|
|
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
|
old = parsed.videos.get(vid)
|
|
if old is None or at > old[0]:
|
|
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
|
|
|
|
|
def _history(data: Iterator[object]) -> Parsed:
|
|
parsed = Parsed(TakeoutKind.WATCHED)
|
|
for entry in data:
|
|
parsed.entries += 1
|
|
if not isinstance(entry, dict):
|
|
parsed.malformed += 1
|
|
continue
|
|
details = entry.get("details")
|
|
if isinstance(details, list) and any(
|
|
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
|
):
|
|
parsed.ads += 1
|
|
continue
|
|
url = entry.get("titleUrl")
|
|
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
|
if match is None:
|
|
parsed.other += 1
|
|
continue
|
|
at = _time(entry.get("time"))
|
|
if at is None:
|
|
parsed.malformed += 1
|
|
continue
|
|
channel = None
|
|
subtitles = entry.get("subtitles")
|
|
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
|
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
|
channel = found.group(1) if found else None
|
|
_keep(parsed, match.group(1), at, channel)
|
|
return parsed
|
|
|
|
|
|
def _watch_later(text: str) -> Parsed:
|
|
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
|
rows = (row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row))
|
|
first = next(rows, None)
|
|
if first is None or len(first) < 2:
|
|
raise TakeoutError("not_takeout")
|
|
# The first row is the header, in the account's language, unless it is already a video.
|
|
for row in chain([first], rows) if _VIDEO_ID.match(first[0].strip()) else rows:
|
|
parsed.entries += 1
|
|
vid = row[0].strip()
|
|
at = _time(row[1].strip()) if len(row) > 1 else None
|
|
if not _VIDEO_ID.match(vid) or at is None:
|
|
parsed.malformed += 1
|
|
continue
|
|
_keep(parsed, vid, at, None)
|
|
if len(parsed.videos) > WATCH_LATER_MAX:
|
|
raise TakeoutError("too_many")
|
|
return parsed
|
|
|
|
|
|
def _skip(text: str, at: int) -> int:
|
|
"""The first index at or after `at` that is not JSON whitespace."""
|
|
while at < len(text) and text[at] in " \t\n\r":
|
|
at += 1
|
|
return at
|
|
|
|
|
|
def _json_list(text: str) -> Iterator[object]:
|
|
"""The entries of a JSON list, one at a time: the whole tree is never built, so a file of a
|
|
million tiny entries costs its text and one entry, not a million dicts at once."""
|
|
decoder = json.JSONDecoder()
|
|
at = _skip(text, _skip(text, 0) + 1) # past the "[" the caller saw
|
|
if text.startswith("]", at):
|
|
at += 1
|
|
else:
|
|
while True:
|
|
entry, at = decoder.raw_decode(text, at)
|
|
yield entry
|
|
at = _skip(text, at)
|
|
if text.startswith(",", at):
|
|
at = _skip(text, at + 1)
|
|
elif text.startswith("]", at):
|
|
at += 1
|
|
break
|
|
else:
|
|
raise ValueError("not a JSON list")
|
|
if _skip(text, at) != len(text):
|
|
raise ValueError("text after the JSON list")
|
|
|
|
|
|
def parse(body: bytes) -> Parsed:
|
|
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
|
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
|
with nothing to import in it."""
|
|
try:
|
|
text = body.decode("utf-8-sig")
|
|
except UnicodeDecodeError:
|
|
raise TakeoutError("not_takeout") from None
|
|
stripped = text.lstrip()
|
|
if stripped.startswith(("{", "<")):
|
|
# Another JSON file of the export, or the history's HTML variant.
|
|
raise TakeoutError("not_takeout")
|
|
try:
|
|
parsed = (
|
|
_history(_json_list(stripped)) if stripped.startswith("[") else _watch_later(stripped)
|
|
)
|
|
# A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
|
|
# long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
|
|
except (ValueError, RecursionError, csv.Error) as exc:
|
|
if isinstance(exc, TakeoutError):
|
|
raise
|
|
raise TakeoutError("not_takeout") from None
|
|
if not parsed.videos:
|
|
raise TakeoutError("no_videos")
|
|
return parsed
|
|
|
|
|
|
# One import per user at a time: a second file uploaded while the first one's run is under way
|
|
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
|
|
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
|
|
# so a lock in this module holds it (as the channel backfill's does).
|
|
_user_locks: dict[int, threading.Lock] = {}
|
|
_user_locks_guard = threading.Lock()
|
|
|
|
|
|
@contextmanager
|
|
def _one_import(user_id: int) -> Iterator[None]:
|
|
with _user_locks_guard:
|
|
lock = _user_locks.setdefault(user_id, threading.Lock())
|
|
with lock:
|
|
yield
|
|
|
|
|
|
def run_target(user_id: int) -> str:
|
|
"""The import run's target, `job_runs.target`: one user's import."""
|
|
return f"user:{user_id}"
|
|
|
|
|
|
def target_title(db: Session, target: str) -> str | None:
|
|
"""Who an import run's target is, for the admin's run lists: the user's name, else their email."""
|
|
_, _, raw = target.partition("user:")
|
|
user = db.get(User, int(raw)) if raw.isdigit() else None
|
|
return (user.display_name or user.email) if user is not None else None
|
|
|
|
|
|
class TooMany(Exception):
|
|
"""The upload would take the user past `ROWS_MAX` kept videos."""
|
|
|
|
|
|
def stage(db: Session, user: User, parsed: Parsed) -> None:
|
|
"""Keep an uploaded file's videos for the import, unless that would take the user past
|
|
`ROWS_MAX` (`TooMany`). The caller commits."""
|
|
service.lock_takeout(db, user) # the count and the insert are one user's, not two uploads' each
|
|
if (
|
|
queries.takeout_rows_besides(db, user, parsed.kind, parsed.videos) + len(parsed.videos)
|
|
> ROWS_MAX
|
|
):
|
|
raise TooMany
|
|
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
|
|
|
|
|
# How many unknown Watch later videos one user may have looked up on the shared API budget in a day:
|
|
# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs.
|
|
_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
|
|
|
|
|
def _put_in_watch_later(db: Session, user: User) -> dict:
|
|
"""Put the user's waiting Watch later videos into theirs here: the ones the catalog knows at once,
|
|
the others once YouTube was asked for them — when a YouTube-connected account can ask, today's
|
|
budget is above the backfill reserve, and the user's own daily share of lookups allows it. A video
|
|
not asked about yet waits for a later run. The lookup commits what it fetched; the caller commits
|
|
the rest."""
|
|
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
|
if not waiting:
|
|
return {}
|
|
ids = [vid for _, vid in waiting]
|
|
known = queries.known_video_ids(db, ids)
|
|
ask = [vid for vid in ids if vid not in known][:WATCH_LATER_MAX]
|
|
asked: set[str] = set()
|
|
fetcher = get_service_user(db) if ask else None
|
|
if (
|
|
fetcher is not None
|
|
and not quota.backfill_reserve_reached()
|
|
and _lookups.allow(str(user.id), cost=len(ask))
|
|
):
|
|
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
|
|
with YouTubeClient(db, fetcher) as yt:
|
|
ensure_videos(db, yt, ask)
|
|
asked = set(ask)
|
|
# Only the rows read above: one a second upload staged meanwhile was never asked about.
|
|
return service.apply_takeout_watch_later(db, user, waiting, asked)
|
|
|
|
|
|
def run_import(user_id: int) -> dict:
|
|
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
|
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
|
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
|
with _one_import(user_id), SessionLocal() as db:
|
|
user = db.get(User, user_id)
|
|
if user is None:
|
|
return {"skipped": "no user"}
|
|
result = service.apply_takeout_watched(db, user)
|
|
db.commit()
|
|
result |= _put_in_watch_later(db, user)
|
|
db.commit()
|
|
outcomes = queries.takeout_outcomes(db, user)
|
|
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
|
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
|
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
|
return result
|
|
|
|
|
|
def apply_waiting(db: Session) -> dict:
|
|
"""Settle what every user's import left waiting and can be settled now, one commit each: a
|
|
watched video the catalog has come to know and the Shorts probe has cleared (by its job or a
|
|
search's), and the Watch later videos a run could not put in yet. Run after each run of the
|
|
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
|
|
nothing was waiting that could move."""
|
|
marked = added = 0
|
|
ready = set(queries.takeout_users_ready(db))
|
|
later = set(queries.takeout_users_watch_later_waiting(db))
|
|
for user_id in sorted(ready | later):
|
|
user = db.get(User, user_id)
|
|
if user is None:
|
|
continue
|
|
with _one_import(user_id):
|
|
if user_id in ready:
|
|
marked += service.apply_takeout_watched(db, user)["watched"]
|
|
db.commit()
|
|
if user_id in later:
|
|
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
|
|
db.commit()
|
|
out = {}
|
|
if ready:
|
|
out["takeout_watched"] = marked
|
|
if added:
|
|
out["takeout_watch_later_added"] = added
|
|
return out
|