siftlode/backend/tests/test_takeout.py

"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the
import job makes of them — watched here (never a Short, never over a state the user set), waiting for
a video the catalog does not know yet, and the Watch later list joined to the user's own."""

import json
from datetime import datetime, timedelta, timezone

import pytest
from sqlalchemy import select

from app import scheduler
from app.catalog.models import Channel, Video
from app.models import Notification
from app.reader import service, takeout
from app.reader.models import (
    Playlist,
    PlaylistItem,
    TakeoutItem,
    TakeoutKind,
    VideoState,
)

T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
CHANNEL = "UC" + "a" * 22


def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra):
    entry = {
        "header": "YouTube",
        "title": "Megnézted: valami",  # the account's language: never read
        "titleUrl": f"https://{host}.youtube.com/watch?v={vid}",
        "time": at.isoformat().replace("+00:00", "Z"),
        "products": ["YouTube"],
        "activityControls": ["YouTube watch history"],
    }
    if channel:
        entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}]
    return entry | extra


def _history(*entries) -> bytes:
    return json.dumps(list(entries)).encode()


# --- reading the files -------------------------------------------------------------------------


def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel():
    later = T0 + timedelta(days=3)
    parsed = takeout.parse(
        _history(
            _view("aaaaaaaaaaa", later),
            _view("aaaaaaaaaaa", T0),
            _view("bbbbbbbbbbb", T0, channel=None),  # a video since removed names no channel
            _view("ccccccccccc", T0, host="music"),
            # Out of order: the later view still wins.
            _view("ddddddddddd", T0, channel=None),
            _view("ddddddddddd", later),
        )
    )
    assert parsed.kind == TakeoutKind.WATCHED
    assert parsed.entries == 6
    assert parsed.videos == {
        "aaaaaaaaaaa": (later, CHANNEL),
        "bbbbbbbbbbb": (T0, None),
        "ccccccccccc": (T0, CHANNEL),
        "ddddddddddd": (later, CHANNEL),
    }


def test_the_history_counts_what_it_leaves_out():
    ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}])
    post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"}
    no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"}
    bad_time = _view("ggggggggggg") | {"time": "yesterday"}
    no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"}
    parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk"))
    assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2)
    assert parsed.videos == {
        "hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL)
    }


def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none():
    body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n"
    parsed = takeout.parse(("" + body).encode())
    assert parsed.kind == TakeoutKind.WATCH_LATER
    assert parsed.videos == {
        "Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None)
    }
    assert parsed.malformed == 1
    headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n")
    assert list(headless.videos) == ["Ngb04xUK8uU"]


@pytest.mark.parametrize(
    ("body", "reason"),
    [
        (b'{"some": "other", "export": "file"}', "not_takeout"),
        (b"<html><body>watch history, as a page</body></html>", "not_takeout"),
        (b"[not json", "not_takeout"),
        (b"\xff\xfe\x00", "not_takeout"),
        (b"one column only\n", "not_takeout"),
        (b"[]", "no_videos"),
        (
            json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(),
            "no_videos",
        ),
    ],
)
def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason):
    with pytest.raises(takeout.TakeoutError) as caught:
        takeout.parse(body)
    assert caught.value.reason == reason


# --- keeping and applying ----------------------------------------------------------------------


def _video(db, vid: str, *, probed=True, short=False):
    if db.get(Channel, CHANNEL) is None:
        db.add(Channel(id=CHANNEL, title="Ch"))
        db.flush()
    db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short))


def _state(db, user, vid, status, *, changed, position=0, watched_at=None):
    db.add(
        VideoState(
            user_id=user.id,
            video_id=vid,
            status=status,
            position_seconds=position,
            watched_at=watched_at,
            updated_at=changed,
        )
    )


def _rows(db, user) -> dict[str, TakeoutItem]:
    db.expire_all()
    rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
    return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows}


def _states(db, user) -> dict[str, VideoState]:
    db.expire_all()
    rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars()
    return {r.video_id: r for r in rows}


def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept(
    db, make_user
):
    user = make_user("takeout@example.com")
    for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"):
        _video(db, vid, probed=vid != "unprobed000", short=vid == "short000000")
    for vid in ("newer000000", "older000000"):
        _video(db, vid)
    earlier = T0 - timedelta(days=30)
    _state(db, user, "hidden00000", "hidden", changed=earlier)
    _state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier)
    # In progress here, touched after the YouTube view: what the user did here is newer.
    _state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40)
    # In progress here, but last touched before the YouTube view: they finished it there.
    _state(db, user, "older000000", "new", changed=earlier, position=40)
    db.commit()
    ids = [
        "plain000000",
        "short000000",
        "unprobed000",
        "unknown0000",
        "hidden00000",
        "watched0000",
        "newer000000",
        "older000000",
    ]
    service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids})

    counts = service.apply_takeout_watched(db, user)
    db.commit()

    assert counts == {"watched": 2, "kept": 3, "shorts": 1}
    rows, states = _rows(db, user), _states(db, user)
    assert {v: rows[v].outcome for v in ids} == {
        "plain000000": "watched",
        "short000000": "short",
        "unprobed000": None,
        "unknown0000": None,
        "hidden00000": "kept",
        "watched0000": "kept",
        "newer000000": "kept",
        "older000000": "watched",
    }
    assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0)
    assert "short000000" not in states
    assert states["hidden00000"].status == "hidden"
    assert states["watched0000"].watched_at == earlier
    assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40)
    assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40)


def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user):
    user = make_user("again@example.com")
    _video(db, "plain000000")
    db.commit()
    service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
    service.apply_takeout_watched(db, user)
    later = T0 + timedelta(days=1)
    service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)})
    service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
    db.commit()
    row = _rows(db, user)["plain000000"]
    assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched")


def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user):
    user = make_user("later@example.com")
    for vid in ("held0000000", "old00000000", "new00000000"):
        _video(db, vid)
    db.commit()
    pl = service.watch_later(db, user)
    service.add_item(db, pl, "held0000000", mark_dirty=False)
    service.stage_takeout(
        db,
        user,
        TakeoutKind.WATCH_LATER,
        {
            "new00000000": (T0, None),
            "held0000000": (T0, None),
            "old00000000": (T0 - timedelta(days=9), None),
            "gone0000000": (T0, None),  # YouTube no longer has it
        },
    )

    counts = service.apply_takeout_watch_later(db, user)
    db.commit()

    assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1}
    items = db.execute(
        select(PlaylistItem.video_id)
        .where(PlaylistItem.playlist_id == pl.id)
        .order_by(PlaylistItem.position)
    ).scalars()
    assert list(items) == ["held0000000", "old00000000", "new00000000"]
    assert _rows(db, user)["WL:gone0000000"].outcome == "missing"
    assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl]


def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits(
    db, make_user, monkeypatch
):
    user = make_user("job@example.com")
    _video(db, "plain000000")
    db.commit()
    service.stage_takeout(
        db,
        user,
        TakeoutKind.WATCHED,
        {"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)},
    )
    service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)})
    db.commit()
    asked = []

    def fake_ensure(session, yt, ids):
        asked.append(ids)
        _video(session, "fetched0000")
        session.commit()

    monkeypatch.setattr(takeout, "ensure_videos", fake_ensure)
    monkeypatch.setattr(takeout, "get_service_user", lambda session: user)

    result = takeout.run_import(user.id)

    assert asked == [["fetched0000"]]
    assert result == {
        "watched": 1,
        "kept": 0,
        "shorts": 0,
        "watch_later_added": 1,
        "watch_later_present": 0,
        "watch_later_missing": 0,
        "waiting": 1,
    }


# --- the upload --------------------------------------------------------------------------------


@pytest.fixture
def queued(monkeypatch):
    runs = []
    monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw)))
    return runs


def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice(
    db, make_user, as_user, queued
):
    user = make_user("upload@example.com")
    body = _history(
        _view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}])
    )
    resp = as_user(user).post("/api/takeout", content=body)
    assert resp.status_code == 200, resp.text
    assert resp.json() == {
        "ok": True,
        "kind": "watched",
        "videos": 1,
        "entries": 2,
        "ads": 1,
        "other": 0,
        "malformed": 0,
    }
    assert list(_rows(db, user)) == ["aaaaaaaaaaa"]
    assert queued == [
        (
            "takeout_import",
            {"actor_id": user.id, "target": f"user:{user.id}", "notify": True},
        )
    ]


@pytest.mark.parametrize(
    ("body", "reason"),
    [(b"<html></html>", "takeout_unreadable"), (b"[]", "takeout_no_videos")],
)
def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language(
    db, make_user, as_user, queued, body, reason
):
    user = make_user("refused@example.com")
    resp = as_user(user).post("/api/takeout", content=body)
    assert resp.status_code == 422
    assert resp.json()["detail"] == {"code": "refused", "reason": reason}
    assert queued == [] and _rows(db, user) == {}


def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch):
    user = make_user("big@example.com")
    monkeypatch.setattr(takeout, "MAX_BYTES", 64)
    resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
    assert resp.status_code == 413
    assert _rows(db, user) == {}


def test_an_anonymous_upload_is_refused(client):
    assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401


# --- the notice --------------------------------------------------------------------------------


def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user):
    user = make_user("notice@example.com")
    scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True)
    scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x")
    db.expire_all()
    notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars()
    assert [(n.data["job_id"], n.data["result"]) for n in notices] == [
        ("takeout_import", {"watched": 3})
    ]