feat(reader): a Google Takeout export becomes watch state and Watch later
The YouTube API reads neither a user's watch history nor their Watch later list, so the Takeout export is the only way in. `POST /api/takeout` takes one file of it (watch-history.json or Watch later-videos.csv, told apart by content), keeps its videos in `takeout_items` and queues the user's `takeout_import` run, whose outcome reaches their inbox. The run marks watched the known videos the Shorts probe has cleared, at the Takeout time; a Short is not marked, and a state the user set here (watched, hidden, or changed after that time) wins, by the upsert's own condition. A video the catalog does not know yet waits with no outcome. Watch later videos are fetched if unknown and appended oldest first. An ad-hoc run can now ask for a completion notice (`run_ad_hoc(..., notify=True)`).
This commit is contained in:
@@ -0,0 +1,362 @@
|
||||
"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the
|
||||
import job makes of them — watched here (never a Short, never over a state the user set), waiting for
|
||||
a video the catalog does not know yet, and the Watch later list joined to the user's own."""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import scheduler
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.models import Notification
|
||||
from app.reader import service, takeout
|
||||
from app.reader.models import (
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
VideoState,
|
||||
)
|
||||
|
||||
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||||
CHANNEL = "UC" + "a" * 22
|
||||
|
||||
|
||||
def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra):
|
||||
entry = {
|
||||
"header": "YouTube",
|
||||
"title": "Megnézted: valami", # the account's language: never read
|
||||
"titleUrl": f"https://{host}.youtube.com/watch?v={vid}",
|
||||
"time": at.isoformat().replace("+00:00", "Z"),
|
||||
"products": ["YouTube"],
|
||||
"activityControls": ["YouTube watch history"],
|
||||
}
|
||||
if channel:
|
||||
entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}]
|
||||
return entry | extra
|
||||
|
||||
|
||||
def _history(*entries) -> bytes:
|
||||
return json.dumps(list(entries)).encode()
|
||||
|
||||
|
||||
# --- reading the files -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel():
|
||||
later = T0 + timedelta(days=3)
|
||||
parsed = takeout.parse(
|
||||
_history(
|
||||
_view("aaaaaaaaaaa", later),
|
||||
_view("aaaaaaaaaaa", T0),
|
||||
_view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel
|
||||
_view("ccccccccccc", T0, host="music"),
|
||||
# Out of order: the later view still wins.
|
||||
_view("ddddddddddd", T0, channel=None),
|
||||
_view("ddddddddddd", later),
|
||||
)
|
||||
)
|
||||
assert parsed.kind == TakeoutKind.WATCHED
|
||||
assert parsed.entries == 6
|
||||
assert parsed.videos == {
|
||||
"aaaaaaaaaaa": (later, CHANNEL),
|
||||
"bbbbbbbbbbb": (T0, None),
|
||||
"ccccccccccc": (T0, CHANNEL),
|
||||
"ddddddddddd": (later, CHANNEL),
|
||||
}
|
||||
|
||||
|
||||
def test_the_history_counts_what_it_leaves_out():
|
||||
ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}])
|
||||
post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"}
|
||||
no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"}
|
||||
bad_time = _view("ggggggggggg") | {"time": "yesterday"}
|
||||
no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"}
|
||||
parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk"))
|
||||
assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2)
|
||||
assert parsed.videos == {
|
||||
"hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL)
|
||||
}
|
||||
|
||||
|
||||
def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none():
|
||||
body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n"
|
||||
parsed = takeout.parse(("" + body).encode())
|
||||
assert parsed.kind == TakeoutKind.WATCH_LATER
|
||||
assert parsed.videos == {
|
||||
"Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None)
|
||||
}
|
||||
assert parsed.malformed == 1
|
||||
headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n")
|
||||
assert list(headless.videos) == ["Ngb04xUK8uU"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "reason"),
|
||||
[
|
||||
(b'{"some": "other", "export": "file"}', "not_takeout"),
|
||||
(b"<html><body>watch history, as a page</body></html>", "not_takeout"),
|
||||
(b"[not json", "not_takeout"),
|
||||
(b"\xff\xfe\x00", "not_takeout"),
|
||||
(b"one column only\n", "not_takeout"),
|
||||
(b"[]", "no_videos"),
|
||||
(
|
||||
json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(),
|
||||
"no_videos",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason):
|
||||
with pytest.raises(takeout.TakeoutError) as caught:
|
||||
takeout.parse(body)
|
||||
assert caught.value.reason == reason
|
||||
|
||||
|
||||
# --- keeping and applying ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _video(db, vid: str, *, probed=True, short=False):
|
||||
if db.get(Channel, CHANNEL) is None:
|
||||
db.add(Channel(id=CHANNEL, title="Ch"))
|
||||
db.flush()
|
||||
db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short))
|
||||
|
||||
|
||||
def _state(db, user, vid, status, *, changed, position=0, watched_at=None):
|
||||
db.add(
|
||||
VideoState(
|
||||
user_id=user.id,
|
||||
video_id=vid,
|
||||
status=status,
|
||||
position_seconds=position,
|
||||
watched_at=watched_at,
|
||||
updated_at=changed,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _rows(db, user) -> dict[str, TakeoutItem]:
|
||||
db.expire_all()
|
||||
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||||
return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows}
|
||||
|
||||
|
||||
def _states(db, user) -> dict[str, VideoState]:
|
||||
db.expire_all()
|
||||
rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars()
|
||||
return {r.video_id: r for r in rows}
|
||||
|
||||
|
||||
def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept(
|
||||
db, make_user
|
||||
):
|
||||
user = make_user("takeout@example.com")
|
||||
for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"):
|
||||
_video(db, vid, probed=vid != "unprobed000", short=vid == "short000000")
|
||||
for vid in ("newer000000", "older000000"):
|
||||
_video(db, vid)
|
||||
earlier = T0 - timedelta(days=30)
|
||||
_state(db, user, "hidden00000", "hidden", changed=earlier)
|
||||
_state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier)
|
||||
# In progress here, touched after the YouTube view: what the user did here is newer.
|
||||
_state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40)
|
||||
# In progress here, but last touched before the YouTube view: they finished it there.
|
||||
_state(db, user, "older000000", "new", changed=earlier, position=40)
|
||||
db.commit()
|
||||
ids = [
|
||||
"plain000000",
|
||||
"short000000",
|
||||
"unprobed000",
|
||||
"unknown0000",
|
||||
"hidden00000",
|
||||
"watched0000",
|
||||
"newer000000",
|
||||
"older000000",
|
||||
]
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids})
|
||||
|
||||
counts = service.apply_takeout_watched(db, user)
|
||||
db.commit()
|
||||
|
||||
assert counts == {"watched": 2, "kept": 3, "shorts": 1}
|
||||
rows, states = _rows(db, user), _states(db, user)
|
||||
assert {v: rows[v].outcome for v in ids} == {
|
||||
"plain000000": "watched",
|
||||
"short000000": "short",
|
||||
"unprobed000": None,
|
||||
"unknown0000": None,
|
||||
"hidden00000": "kept",
|
||||
"watched0000": "kept",
|
||||
"newer000000": "kept",
|
||||
"older000000": "watched",
|
||||
}
|
||||
assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0)
|
||||
assert "short000000" not in states
|
||||
assert states["hidden00000"].status == "hidden"
|
||||
assert states["watched0000"].watched_at == earlier
|
||||
assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40)
|
||||
assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40)
|
||||
|
||||
|
||||
def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user):
|
||||
user = make_user("again@example.com")
|
||||
_video(db, "plain000000")
|
||||
db.commit()
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||||
service.apply_takeout_watched(db, user)
|
||||
later = T0 + timedelta(days=1)
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)})
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||||
db.commit()
|
||||
row = _rows(db, user)["plain000000"]
|
||||
assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched")
|
||||
|
||||
|
||||
def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user):
|
||||
user = make_user("later@example.com")
|
||||
for vid in ("held0000000", "old00000000", "new00000000"):
|
||||
_video(db, vid)
|
||||
db.commit()
|
||||
pl = service.watch_later(db, user)
|
||||
service.add_item(db, pl, "held0000000", mark_dirty=False)
|
||||
service.stage_takeout(
|
||||
db,
|
||||
user,
|
||||
TakeoutKind.WATCH_LATER,
|
||||
{
|
||||
"new00000000": (T0, None),
|
||||
"held0000000": (T0, None),
|
||||
"old00000000": (T0 - timedelta(days=9), None),
|
||||
"gone0000000": (T0, None), # YouTube no longer has it
|
||||
},
|
||||
)
|
||||
|
||||
counts = service.apply_takeout_watch_later(db, user)
|
||||
db.commit()
|
||||
|
||||
assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1}
|
||||
items = db.execute(
|
||||
select(PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position)
|
||||
).scalars()
|
||||
assert list(items) == ["held0000000", "old00000000", "new00000000"]
|
||||
assert _rows(db, user)["WL:gone0000000"].outcome == "missing"
|
||||
assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl]
|
||||
|
||||
|
||||
def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
user = make_user("job@example.com")
|
||||
_video(db, "plain000000")
|
||||
db.commit()
|
||||
service.stage_takeout(
|
||||
db,
|
||||
user,
|
||||
TakeoutKind.WATCHED,
|
||||
{"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)},
|
||||
)
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)})
|
||||
db.commit()
|
||||
asked = []
|
||||
|
||||
def fake_ensure(session, yt, ids):
|
||||
asked.append(ids)
|
||||
_video(session, "fetched0000")
|
||||
session.commit()
|
||||
|
||||
monkeypatch.setattr(takeout, "ensure_videos", fake_ensure)
|
||||
monkeypatch.setattr(takeout, "get_service_user", lambda session: user)
|
||||
|
||||
result = takeout.run_import(user.id)
|
||||
|
||||
assert asked == [["fetched0000"]]
|
||||
assert result == {
|
||||
"watched": 1,
|
||||
"kept": 0,
|
||||
"shorts": 0,
|
||||
"watch_later_added": 1,
|
||||
"watch_later_present": 0,
|
||||
"watch_later_missing": 0,
|
||||
"waiting": 1,
|
||||
}
|
||||
|
||||
|
||||
# --- the upload --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def queued(monkeypatch):
|
||||
runs = []
|
||||
monkeypatch.setattr(scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, kw)))
|
||||
return runs
|
||||
|
||||
|
||||
def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice(
|
||||
db, make_user, as_user, queued
|
||||
):
|
||||
user = make_user("upload@example.com")
|
||||
body = _history(
|
||||
_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}])
|
||||
)
|
||||
resp = as_user(user).post("/api/takeout", content=body)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json() == {
|
||||
"ok": True,
|
||||
"kind": "watched",
|
||||
"videos": 1,
|
||||
"entries": 2,
|
||||
"ads": 1,
|
||||
"other": 0,
|
||||
"malformed": 0,
|
||||
}
|
||||
assert list(_rows(db, user)) == ["aaaaaaaaaaa"]
|
||||
assert queued == [
|
||||
(
|
||||
"takeout_import",
|
||||
{"actor_id": user.id, "target": f"user:{user.id}", "notify": True},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "reason"),
|
||||
[(b"<html></html>", "takeout_unreadable"), (b"[]", "takeout_no_videos")],
|
||||
)
|
||||
def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language(
|
||||
db, make_user, as_user, queued, body, reason
|
||||
):
|
||||
user = make_user("refused@example.com")
|
||||
resp = as_user(user).post("/api/takeout", content=body)
|
||||
assert resp.status_code == 422
|
||||
assert resp.json()["detail"] == {"code": "refused", "reason": reason}
|
||||
assert queued == [] and _rows(db, user) == {}
|
||||
|
||||
|
||||
def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch):
|
||||
user = make_user("big@example.com")
|
||||
monkeypatch.setattr(takeout, "MAX_BYTES", 64)
|
||||
resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
|
||||
assert resp.status_code == 413
|
||||
assert _rows(db, user) == {}
|
||||
|
||||
|
||||
def test_an_anonymous_upload_is_refused(client):
|
||||
assert client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 401
|
||||
|
||||
|
||||
# --- the notice --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user):
|
||||
user = make_user("notice@example.com")
|
||||
scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True)
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x")
|
||||
db.expire_all()
|
||||
notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars()
|
||||
assert [(n.data["job_id"], n.data["result"]) for n in notices] == [
|
||||
("takeout_import", {"watched": 3})
|
||||
]
|
||||
Reference in New Issue
Block a user