siftlode/backend/app/reader/takeout.py

"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
only way in.

Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
what it can; a video the catalog does not know yet waits for it.

The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
header row (in the account's language) and `<id>,<ISO time added>` rows.
"""

import csv
import io
import json
import re
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime

from app import quota
from app.bodylimit import TAKEOUT_MAX_BYTES
from app.db import SessionLocal
from app.models import User
from app.reader import queries, service
from app.reader.models import TakeoutKind
from app.sync.runner import get_service_user
from app.sync.videos import ensure_videos
from app.youtube.client import YouTubeClient

# The largest file the upload takes (its edge cap and the route's own are one number).
MAX_BYTES = TAKEOUT_MAX_BYTES

_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
_WATCH_URL = re.compile(
    r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
)
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")


class TakeoutError(ValueError):
    """The file is not one the import reads; `reason` is the client's word for why."""

    def __init__(self, reason: str):
        super().__init__(reason)
        self.reason = reason


@dataclass
class Parsed:
    """One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""

    kind: TakeoutKind
    videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
    entries: int = 0
    ads: int = 0
    other: int = 0  # not a view of a video (a post, a story) or a video with no link left
    malformed: int = 0


def _time(raw: object) -> datetime | None:
    if not isinstance(raw, str):
        return None
    try:
        at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
    # silent-ok: the row is counted as malformed, and the count is in the upload's answer.
    except ValueError:
        return None
    return at if at.tzinfo is not None else None


def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
    old = parsed.videos.get(vid)
    if old is None or at > old[0]:
        parsed.videos[vid] = (at, channel or (old[1] if old else None))


def _history(data: list) -> Parsed:
    parsed = Parsed(TakeoutKind.WATCHED)
    for entry in data:
        parsed.entries += 1
        if not isinstance(entry, dict):
            parsed.malformed += 1
            continue
        details = entry.get("details")
        if isinstance(details, list) and any(
            isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
        ):
            parsed.ads += 1
            continue
        url = entry.get("titleUrl")
        match = _WATCH_URL.match(url) if isinstance(url, str) else None
        if match is None:
            parsed.other += 1
            continue
        at = _time(entry.get("time"))
        if at is None:
            parsed.malformed += 1
            continue
        channel = None
        subtitles = entry.get("subtitles")
        if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
            found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
            channel = found.group(1) if found else None
        _keep(parsed, match.group(1), at, channel)
    return parsed


def _watch_later(text: str) -> Parsed:
    parsed = Parsed(TakeoutKind.WATCH_LATER)
    rows = [row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row)]
    if not rows or len(rows[0]) < 2:
        raise TakeoutError("not_takeout")
    if not _VIDEO_ID.match(rows[0][0].strip()):
        rows = rows[1:]  # the header, in the account's language
    for row in rows:
        parsed.entries += 1
        vid = row[0].strip() if row else ""
        at = _time(row[1].strip()) if len(row) > 1 else None
        if not _VIDEO_ID.match(vid) or at is None:
            parsed.malformed += 1
            continue
        _keep(parsed, vid, at, None)
    return parsed


def parse(body: bytes) -> Parsed:
    """Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
    apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
    with nothing to import in it."""
    try:
        text = body.decode("utf-8-sig")
    except UnicodeDecodeError:
        raise TakeoutError("not_takeout") from None
    stripped = text.lstrip()
    if stripped.startswith("["):
        try:
            data = json.loads(stripped)
        except json.JSONDecodeError:
            raise TakeoutError("not_takeout") from None
        if not isinstance(data, list):
            raise TakeoutError("not_takeout")
        parsed = _history(data)
    elif stripped.startswith(("{", "<")):
        # Another JSON file of the export, or the history's HTML variant.
        raise TakeoutError("not_takeout")
    else:
        parsed = _watch_later(stripped)
    if not parsed.videos:
        raise TakeoutError("no_videos")
    return parsed


# One import per user at a time: a second file uploaded while the first one's run is under way
# starts a run of its own (the first one's id has left the scheduler), and the two would settle the
# same rows. The second waits, then takes what the first left. One API process runs the scheduler,
# so a lock in this module holds it (as the channel backfill's does).
_user_locks: dict[int, threading.Lock] = {}
_user_locks_guard = threading.Lock()


@contextmanager
def _one_import(user_id: int) -> Iterator[None]:
    with _user_locks_guard:
        lock = _user_locks.setdefault(user_id, threading.Lock())
    with lock:
        yield


def run_import(user_id: int) -> dict:
    """The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
    The watched part commits before the Watch later part asks YouTube for the videos the catalog
    lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
    with _one_import(user_id), SessionLocal() as db:
        user = db.get(User, user_id)
        if user is None:
            return {"skipped": "no user"}
        result = service.apply_takeout_watched(db, user)
        db.commit()
        waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
        if waiting:
            fetcher = get_service_user(db)
            if fetcher is not None:
                with quota.attribute(user_id, quota.QuotaAction.VIDEOS_LOOKUP):
                    with YouTubeClient(db, fetcher) as yt:
                        ensure_videos(db, yt, [vid for _, vid in waiting])
                result |= service.apply_takeout_watch_later(db, user)
                db.commit()
        outcomes = queries.takeout_outcomes(db, user)
        result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
        if outcomes.get((TakeoutKind.WATCH_LATER, None)):
            result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
        return result