peter
5d9952ab1e
A watched video the catalog did not know at import time waits in `takeout_items`. The Shorts job now settles those rows right after its probe (the only place a video's Short-ness becomes known), every user's, a commit each: `reader.takeout.apply_waiting`, its count in the job's result. `GET /api/takeout` is what the settings page reads: the user's last import run (`jobruns.last_run`) and the kept videos by kind and outcome, "waiting" for one not known yet.
225 lines
8.7 KiB
Python
225 lines
8.7 KiB
Python
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
|
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
|
only way in.
|
|
|
|
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
|
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
|
what it can; a video the catalog does not know yet waits for it.
|
|
|
|
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
|
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
|
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
|
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
|
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
|
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
|
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
|
"""
|
|
|
|
import csv
|
|
import io
|
|
import json
|
|
import re
|
|
import threading
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import quota
|
|
from app.bodylimit import TAKEOUT_MAX_BYTES
|
|
from app.db import SessionLocal
|
|
from app.models import User
|
|
from app.reader import queries, service
|
|
from app.reader.models import TakeoutKind
|
|
from app.sync.runner import get_service_user
|
|
from app.sync.videos import ensure_videos
|
|
from app.youtube.client import YouTubeClient
|
|
|
|
# The largest file the upload takes (its edge cap and the route's own are one number).
|
|
MAX_BYTES = TAKEOUT_MAX_BYTES
|
|
|
|
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
|
_WATCH_URL = re.compile(
|
|
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
|
)
|
|
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
|
|
|
|
|
class TakeoutError(ValueError):
|
|
"""The file is not one the import reads; `reason` is the client's word for why."""
|
|
|
|
def __init__(self, reason: str):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
|
|
|
|
@dataclass
|
|
class Parsed:
|
|
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
|
|
|
kind: TakeoutKind
|
|
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
|
entries: int = 0
|
|
ads: int = 0
|
|
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
|
malformed: int = 0
|
|
|
|
|
|
def _time(raw: object) -> datetime | None:
|
|
if not isinstance(raw, str):
|
|
return None
|
|
try:
|
|
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
|
except ValueError:
|
|
return None
|
|
return at if at.tzinfo is not None else None
|
|
|
|
|
|
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
|
old = parsed.videos.get(vid)
|
|
if old is None or at > old[0]:
|
|
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
|
|
|
|
|
def _history(data: list) -> Parsed:
|
|
parsed = Parsed(TakeoutKind.WATCHED)
|
|
for entry in data:
|
|
parsed.entries += 1
|
|
if not isinstance(entry, dict):
|
|
parsed.malformed += 1
|
|
continue
|
|
details = entry.get("details")
|
|
if isinstance(details, list) and any(
|
|
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
|
):
|
|
parsed.ads += 1
|
|
continue
|
|
url = entry.get("titleUrl")
|
|
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
|
if match is None:
|
|
parsed.other += 1
|
|
continue
|
|
at = _time(entry.get("time"))
|
|
if at is None:
|
|
parsed.malformed += 1
|
|
continue
|
|
channel = None
|
|
subtitles = entry.get("subtitles")
|
|
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
|
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
|
channel = found.group(1) if found else None
|
|
_keep(parsed, match.group(1), at, channel)
|
|
return parsed
|
|
|
|
|
|
def _watch_later(text: str) -> Parsed:
|
|
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
|
rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
|
|
if not rows or len(rows[0]) < 2:
|
|
raise TakeoutError("not_takeout")
|
|
if not _VIDEO_ID.match(rows[0][0].strip()):
|
|
rows = rows[1:] # the header, in the account's language
|
|
for row in rows:
|
|
parsed.entries += 1
|
|
vid = row[0].strip() if row else ""
|
|
at = _time(row[1].strip()) if len(row) > 1 else None
|
|
if not _VIDEO_ID.match(vid) or at is None:
|
|
parsed.malformed += 1
|
|
continue
|
|
_keep(parsed, vid, at, None)
|
|
return parsed
|
|
|
|
|
|
def parse(body: bytes) -> Parsed:
|
|
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
|
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
|
with nothing to import in it."""
|
|
try:
|
|
text = body.decode("utf-8-sig")
|
|
except UnicodeDecodeError:
|
|
raise TakeoutError("not_takeout") from None
|
|
stripped = text.lstrip()
|
|
if stripped.startswith("["):
|
|
try:
|
|
data = json.loads(stripped)
|
|
except json.JSONDecodeError:
|
|
raise TakeoutError("not_takeout") from None
|
|
if not isinstance(data, list):
|
|
raise TakeoutError("not_takeout")
|
|
parsed = _history(data)
|
|
elif stripped.startswith(("{", "<")):
|
|
# Another JSON file of the export, or the history's HTML variant.
|
|
raise TakeoutError("not_takeout")
|
|
else:
|
|
parsed = _watch_later(stripped)
|
|
if not parsed.videos:
|
|
raise TakeoutError("no_videos")
|
|
return parsed
|
|
|
|
|
|
# One import per user at a time: a second file uploaded while the first one's run is under way
|
|
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
|
|
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
|
|
# so a lock in this module holds it (as the channel backfill's does).
|
|
_user_locks: dict[int, threading.Lock] = {}
|
|
_user_locks_guard = threading.Lock()
|
|
|
|
|
|
@contextmanager
|
|
def _one_import(user_id: int) -> Iterator[None]:
|
|
with _user_locks_guard:
|
|
lock = _user_locks.setdefault(user_id, threading.Lock())
|
|
with lock:
|
|
yield
|
|
|
|
|
|
def run_target(user_id: int) -> str:
|
|
"""The import run's target, `job_runs.target`: one user's import."""
|
|
return f"user:{user_id}"
|
|
|
|
|
|
def run_import(user_id: int) -> dict:
|
|
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
|
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
|
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
|
with _one_import(user_id), SessionLocal() as db:
|
|
user = db.get(User, user_id)
|
|
if user is None:
|
|
return {"skipped": "no user"}
|
|
result = service.apply_takeout_watched(db, user)
|
|
db.commit()
|
|
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
|
if waiting:
|
|
fetcher = get_service_user(db)
|
|
if fetcher is not None:
|
|
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
|
|
with YouTubeClient(db, fetcher) as yt:
|
|
ensure_videos(db, yt, [vid for _, vid in waiting])
|
|
result |= service.apply_takeout_watch_later(db, user)
|
|
db.commit()
|
|
outcomes = queries.takeout_outcomes(db, user)
|
|
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
|
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
|
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
|
return result
|
|
|
|
|
|
def apply_waiting(db: Session) -> dict:
|
|
"""Mark watched the waiting Takeout rows whose video the catalog has come to know and the Shorts
|
|
probe has cleared since the import — every user's, one commit each. Run after each Shorts probe
|
|
(the scheduler's `shorts` job), the only place a video's Short-ness becomes known. Returns its
|
|
counts, or an empty dict when nothing was waiting on a probed video."""
|
|
users = queries.takeout_users_ready(db)
|
|
marked = 0
|
|
for user_id in users:
|
|
user = db.get(User, user_id)
|
|
if user is None:
|
|
continue
|
|
with _one_import(user_id):
|
|
marked += service.apply_takeout_watched(db, user)["watched"]
|
|
db.commit()
|
|
return {"takeout_watched": marked} if users else {}
|