peter
9c5f81fd94
- An ad-hoc request made while its target's run is under way is the run after it: the scheduler dropped a second fire of a running id, so a second Takeout file was never imported and its key stayed queued until a restart. - Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view no longer marks it watched again; un-marks take the user's Takeout lock. - One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the others are settled, and a failed lookup spends none of the daily share. - The upload reads the user's id before giving the read connection back. - A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a mark never pins it against the cleanup jobs. - The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and two uploads at once across all users (takeout_server_busy). - A Watch later list goes in as one presence read and one insert.
1020 lines
40 KiB
Python
1020 lines
40 KiB
Python
"""The Google Takeout import: reading the two files of the export, keeping their videos, and what the
|
||
import job makes of them — watched here (never a Short, never over a state the user set), waiting for
|
||
a video the catalog does not know yet, and the Watch later list joined to the user's own."""
|
||
|
||
import json
|
||
import tracemalloc
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import pytest
|
||
from fastapi import Depends
|
||
from sqlalchemy import select, text
|
||
from sqlalchemy.exc import OperationalError
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app import jobruns, quota, scheduler
|
||
from app.auth import current_user
|
||
from app.db import SessionLocal, get_db
|
||
from app.catalog.models import Channel, Subscription, Video
|
||
from app.models import JobRun, Notification, User
|
||
from app.runstatus import RunStatus, RunTrigger
|
||
from app.reader import queries, service, takeout
|
||
from app.routes import feed as feed_routes
|
||
from app.reader.models import (
|
||
Playlist,
|
||
PlaylistItem,
|
||
TakeoutItem,
|
||
TakeoutKind,
|
||
VideoState,
|
||
)
|
||
|
||
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||
CHANNEL = "UC" + "a" * 22
|
||
|
||
|
||
def _view(vid: str, at: datetime = T0, *, channel: str | None = CHANNEL, host="www", **extra):
|
||
entry = {
|
||
"header": "YouTube",
|
||
"title": "Megnézted: valami", # the account's language: never read
|
||
"titleUrl": f"https://{host}.youtube.com/watch?v={vid}",
|
||
"time": at.isoformat().replace("+00:00", "Z"),
|
||
"products": ["YouTube"],
|
||
"activityControls": ["YouTube watch history"],
|
||
}
|
||
if channel:
|
||
entry["subtitles"] = [{"name": "Ch", "url": f"https://www.youtube.com/channel/{channel}"}]
|
||
return entry | extra
|
||
|
||
|
||
def _history(*entries) -> bytes:
|
||
return json.dumps(list(entries)).encode()
|
||
|
||
|
||
# --- reading the files -------------------------------------------------------------------------
|
||
|
||
|
||
def test_the_history_keeps_each_video_once_at_its_last_view_with_its_channel():
|
||
later = T0 + timedelta(days=3)
|
||
parsed = takeout.parse(
|
||
_history(
|
||
_view("aaaaaaaaaaa", later),
|
||
_view("aaaaaaaaaaa", T0),
|
||
_view("bbbbbbbbbbb", T0, channel=None), # a video since removed names no channel
|
||
_view("ccccccccccc", T0, host="music"),
|
||
# Out of order: the later view still wins.
|
||
_view("ddddddddddd", T0, channel=None),
|
||
_view("ddddddddddd", later),
|
||
)
|
||
)
|
||
assert parsed.kind == TakeoutKind.WATCHED
|
||
assert parsed.entries == 6
|
||
assert parsed.videos == {
|
||
"aaaaaaaaaaa": (later, CHANNEL),
|
||
"bbbbbbbbbbb": (T0, None),
|
||
"ccccccccccc": (T0, CHANNEL),
|
||
"ddddddddddd": (later, CHANNEL),
|
||
}
|
||
|
||
|
||
def test_the_history_counts_what_it_leaves_out():
|
||
ad = _view("ddddddddddd", details=[{"name": "From Google Ads"}])
|
||
post = _view("eeeeeeeeeee") | {"titleUrl": "https://www.youtube.com/post/Ugkx123"}
|
||
no_link = {k: v for k, v in _view("fffffffffff").items() if k != "titleUrl"}
|
||
bad_time = _view("ggggggggggg") | {"time": "yesterday"}
|
||
no_ms = _view("hhhhhhhhhhh") | {"time": "2025-04-19T21:29:22Z"}
|
||
parsed = takeout.parse(_history(ad, post, no_link, bad_time, no_ms, "junk"))
|
||
assert (parsed.ads, parsed.other, parsed.malformed) == (1, 2, 2)
|
||
assert parsed.videos == {
|
||
"hhhhhhhhhhh": (datetime(2025, 4, 19, 21, 29, 22, tzinfo=timezone.utc), CHANNEL)
|
||
}
|
||
|
||
|
||
def test_the_watch_later_csv_is_read_with_a_header_in_any_language_or_none():
|
||
body = "Videóazonosító,Létrehozás ideje\nNgb04xUK8uU,2019-02-07T15:23:30+00:00\nbad,x\n"
|
||
parsed = takeout.parse(("" + body).encode())
|
||
assert parsed.kind == TakeoutKind.WATCH_LATER
|
||
assert parsed.videos == {
|
||
"Ngb04xUK8uU": (datetime(2019, 2, 7, 15, 23, 30, tzinfo=timezone.utc), None)
|
||
}
|
||
assert parsed.malformed == 1
|
||
headless = takeout.parse(b"Ngb04xUK8uU,2019-02-07T15:23:30+00:00\n")
|
||
assert list(headless.videos) == ["Ngb04xUK8uU"]
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("body", "reason"),
|
||
[
|
||
(b'{"some": "other", "export": "file"}', "not_takeout"),
|
||
(b"<html><body>watch history, as a page</body></html>", "not_takeout"),
|
||
(b"[not json", "not_takeout"),
|
||
(b"\xff\xfe\x00", "not_takeout"),
|
||
(b"one column only\n", "not_takeout"),
|
||
(b"[]", "no_videos"),
|
||
(
|
||
json.dumps([_view("aaaaaaaaaaa", details=[{"name": "From Google Ads"}])]).encode(),
|
||
"no_videos",
|
||
),
|
||
# Read an entry at a time, the list must still be one: nothing after it, nothing missing.
|
||
(_history(_view("aaaaaaaaaaa")) + b" []", "not_takeout"),
|
||
(_history(_view("aaaaaaaaaaa"))[:-1], "not_takeout"),
|
||
(b"[" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" x]", "not_takeout"),
|
||
# However a hostile file fails, it is one the import cannot read, not a server error.
|
||
(b"[" * 100_000, "not_takeout"),
|
||
(b"[" + b"1" * 5000 + b"]", "not_takeout"),
|
||
(b"Ngb04xUK8uU," + b"x" * 200_000 + b"\n", "not_takeout"),
|
||
(
|
||
"".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode(),
|
||
"too_many",
|
||
),
|
||
],
|
||
)
|
||
def test_a_file_the_import_cannot_use_is_refused_with_its_reason(body, reason):
|
||
with pytest.raises(takeout.TakeoutError) as caught:
|
||
takeout.parse(body)
|
||
assert caught.value.reason == reason
|
||
|
||
|
||
def test_the_history_is_read_an_entry_at_a_time_not_as_one_tree():
|
||
"""A file of a million tiny entries must cost about its own size, not a million dicts at once:
|
||
the whole tree built first measured ~25 times the file."""
|
||
body = b"[" + b",".join([b"{}"] * 1_000_000) + b"]" # 3 MB
|
||
tracemalloc.start()
|
||
try:
|
||
with pytest.raises(takeout.TakeoutError):
|
||
takeout.parse(body)
|
||
_, peak = tracemalloc.get_traced_memory()
|
||
finally:
|
||
tracemalloc.stop()
|
||
assert peak < 5 * len(body), f"parsing held {peak / len(body):.1f} times the file"
|
||
|
||
|
||
def test_a_history_with_an_emoji_costs_less_than_its_own_size_to_read(monkeypatch):
|
||
"""Decoded whole, one emoji makes the text four bytes a character: a 32 MiB history held some
|
||
128 MiB of it on top of its bytes. Read a window at a time, it holds a window (a few times its
|
||
bytes as text) whatever the file's size — a small one here, so a 9 MB file can show it."""
|
||
monkeypatch.setattr(takeout, "_WINDOW", 64 * 1024)
|
||
view = _view("aaaaaaaaaaa", title="Watched " + "😀" * 40)
|
||
body = b"[" + b",".join([json.dumps(view, ensure_ascii=False).encode()] * 20_000) + b"]"
|
||
tracemalloc.start()
|
||
try:
|
||
parsed = takeout.parse(body)
|
||
_, peak = tracemalloc.get_traced_memory()
|
||
finally:
|
||
tracemalloc.stop()
|
||
assert parsed.entries == 20_000
|
||
assert peak < len(body) / 2, f"parsing held {peak / len(body):.1f} times the file"
|
||
|
||
|
||
@pytest.mark.parametrize("split", [1, 7, 100, 4096])
|
||
def test_the_history_reads_the_same_whatever_its_window(monkeypatch, split):
|
||
"""An entry, a character or a number cut by the window's edge is read whole."""
|
||
monkeypatch.setattr(takeout, "_WINDOW", split)
|
||
views = [_view(f"{c * 11}", title="ő😀" * 3) for c in "abcdef"]
|
||
body = " [\n" + ",\n".join(json.dumps(v, ensure_ascii=False) for v in views) + "\n]\n"
|
||
parsed = takeout.parse(body.encode())
|
||
assert (parsed.entries, sorted(parsed.videos)) == (6, [c * 11 for c in "abcdef"])
|
||
numbers = takeout._json_list(b"[123456789, 2]", 0)
|
||
assert list(numbers) == [123456789, 2]
|
||
|
||
|
||
def test_a_watch_later_file_far_larger_than_any_list_is_refused_unread():
|
||
body = b"Video ID,Time\n" + b"x" * (takeout.CSV_MAX_BYTES)
|
||
with pytest.raises(takeout.TakeoutError) as caught:
|
||
takeout.parse(body)
|
||
assert caught.value.reason == "too_many"
|
||
|
||
|
||
def test_the_id_lists_are_asked_in_chunks(db, make_user, monkeypatch):
|
||
user = make_user("chunks@example.com")
|
||
ids = [f"chunk{n:06d}" for n in range(5)]
|
||
for vid in ids[:3]:
|
||
_video(db, vid)
|
||
db.commit()
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {vid: (T0, None) for vid in ids[:4]})
|
||
db.commit()
|
||
monkeypatch.setattr(queries, "_IN_CHUNK", 2)
|
||
assert queries.known_video_ids(db, ids) == set(ids[:3])
|
||
assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids) == 0
|
||
assert queries.takeout_rows_besides(db, user, TakeoutKind.WATCHED, ids[:1]) == 3
|
||
|
||
|
||
def test_the_history_is_read_whatever_its_whitespace():
|
||
body = b" \n[ \n" + json.dumps(_view("aaaaaaaaaaa")).encode() + b" ,\n\t"
|
||
body += json.dumps(_view("bbbbbbbbbbb")).encode() + b"\n] \n"
|
||
assert sorted(takeout.parse(body).videos) == ["aaaaaaaaaaa", "bbbbbbbbbbb"]
|
||
full = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5000))
|
||
assert len(takeout.parse(full.encode()).videos) == 5000 # YouTube's own cap still fits
|
||
|
||
|
||
# --- keeping and applying ----------------------------------------------------------------------
|
||
|
||
|
||
def _video(db, vid: str, *, probed=True, short=False):
|
||
if db.get(Channel, CHANNEL) is None:
|
||
db.add(Channel(id=CHANNEL, title="Ch"))
|
||
db.flush()
|
||
db.add(Video(id=vid, channel_id=CHANNEL, published_at=T0, shorts_probed=probed, is_short=short))
|
||
|
||
|
||
def _state(db, user, vid, status, *, changed, position=0, watched_at=None):
|
||
db.add(
|
||
VideoState(
|
||
user_id=user.id,
|
||
video_id=vid,
|
||
status=status,
|
||
position_seconds=position,
|
||
watched_at=watched_at,
|
||
updated_at=changed,
|
||
)
|
||
)
|
||
|
||
|
||
def _rows(db, user) -> dict[str, TakeoutItem]:
|
||
db.expire_all()
|
||
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||
return {(r.video_id if r.kind == TakeoutKind.WATCHED else "WL:" + r.video_id): r for r in rows}
|
||
|
||
|
||
def _states(db, user) -> dict[str, VideoState]:
|
||
db.expire_all()
|
||
rows = db.execute(select(VideoState).where(VideoState.user_id == user.id)).scalars()
|
||
return {r.video_id: r for r in rows}
|
||
|
||
|
||
def test_a_known_video_is_marked_watched_at_its_takeout_time_and_the_rest_wait_or_are_kept(
|
||
db, make_user
|
||
):
|
||
user = make_user("takeout@example.com")
|
||
for vid in ("plain000000", "short000000", "unprobed000", "hidden00000", "watched0000"):
|
||
_video(db, vid, probed=vid != "unprobed000", short=vid == "short000000")
|
||
for vid in ("newer000000", "older000000"):
|
||
_video(db, vid)
|
||
earlier = T0 - timedelta(days=30)
|
||
_state(db, user, "hidden00000", "hidden", changed=earlier)
|
||
_state(db, user, "watched0000", "watched", changed=earlier, watched_at=earlier)
|
||
# In progress here, touched after the YouTube view: what the user did here is newer.
|
||
_state(db, user, "newer000000", "new", changed=T0 + timedelta(hours=1), position=40)
|
||
# In progress here, but last touched before the YouTube view: they finished it there.
|
||
_state(db, user, "older000000", "new", changed=earlier, position=40)
|
||
db.commit()
|
||
ids = [
|
||
"plain000000",
|
||
"short000000",
|
||
"unprobed000",
|
||
"unknown0000",
|
||
"hidden00000",
|
||
"watched0000",
|
||
"newer000000",
|
||
"older000000",
|
||
]
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {v: (T0, CHANNEL) for v in ids})
|
||
|
||
counts = service.apply_takeout_watched(db, user)
|
||
db.commit()
|
||
|
||
assert counts == {"watched": 2, "kept": 3, "shorts": 1}
|
||
rows, states = _rows(db, user), _states(db, user)
|
||
assert {v: rows[v].outcome for v in ids} == {
|
||
"plain000000": "watched",
|
||
"short000000": "short",
|
||
"unprobed000": None,
|
||
"unknown0000": None,
|
||
"hidden00000": "kept",
|
||
"watched0000": "kept",
|
||
"newer000000": "kept",
|
||
"older000000": "watched",
|
||
}
|
||
assert (states["plain000000"].status, states["plain000000"].watched_at) == ("watched", T0)
|
||
assert "short000000" not in states
|
||
assert states["hidden00000"].status == "hidden"
|
||
assert states["watched0000"].watched_at == earlier
|
||
assert (states["newer000000"].status, states["newer000000"].position_seconds) == ("new", 40)
|
||
assert (states["older000000"].status, states["older000000"].position_seconds) == ("watched", 40)
|
||
|
||
|
||
@pytest.mark.parametrize("undo", ["unmark", "clear_state"])
|
||
def test_a_video_the_user_un_marked_here_is_not_marked_again_by_an_older_view(db, make_user, undo):
|
||
"""Un-marking (or resetting) a video leaves no state row, and the import inserted a watched one
|
||
over the user's own choice. A view on YouTube after the un-mark is a new view, and counts."""
|
||
user = make_user(f"{undo}@example.com")
|
||
for vid in ("undone00000", "again000000"):
|
||
_video(db, vid)
|
||
db.commit()
|
||
for vid in ("undone00000", "again000000"):
|
||
service.mark(db, user, vid, "watched")
|
||
getattr(service, undo)(db, user, vid)
|
||
db.commit()
|
||
assert _states(db, user) == {}
|
||
later = datetime.now(timezone.utc) + timedelta(minutes=5)
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCHED,
|
||
{"undone00000": (T0, None), "again000000": (later, None)},
|
||
)
|
||
db.commit()
|
||
|
||
assert takeout.run_import(user.id)["watched"] == 1
|
||
assert set(_states(db, user)) == {"again000000"}
|
||
assert _rows(db, user)["undone00000"].outcome == "kept"
|
||
|
||
|
||
def test_a_second_upload_keeps_the_later_time_and_what_the_import_settled(db, make_user):
|
||
user = make_user("again@example.com")
|
||
_video(db, "plain000000")
|
||
db.commit()
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||
service.apply_takeout_watched(db, user)
|
||
later = T0 + timedelta(days=1)
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (later, CHANNEL)})
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||
db.commit()
|
||
row = _rows(db, user)["plain000000"]
|
||
assert (row.at, row.channel_id, row.outcome) == (later, CHANNEL, "watched")
|
||
|
||
|
||
def test_the_watch_later_list_joins_the_users_own_oldest_first_after_what_it_holds(db, make_user):
|
||
user = make_user("later@example.com")
|
||
for vid in ("held0000000", "old00000000", "new00000000"):
|
||
_video(db, vid)
|
||
db.commit()
|
||
pl = service.watch_later(db, user)
|
||
service.add_item(db, pl, "held0000000", mark_dirty=False)
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCH_LATER,
|
||
{
|
||
"new00000000": (T0, None),
|
||
"held0000000": (T0, None),
|
||
"old00000000": (T0 - timedelta(days=9), None),
|
||
"gone0000000": (T0, None), # YouTube no longer has it
|
||
},
|
||
)
|
||
|
||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unasked0000": (T0, None)})
|
||
rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||
|
||
counts = service.apply_takeout_watch_later(db, user, rows, asked={"gone0000000"})
|
||
db.commit()
|
||
|
||
assert counts == {"watch_later_added": 2, "watch_later_present": 1, "watch_later_missing": 1}
|
||
# Unknown and not asked about: it waits, it is not called gone.
|
||
assert _rows(db, user)["WL:unasked0000"].outcome is None
|
||
items = db.execute(
|
||
select(PlaylistItem.video_id)
|
||
.where(PlaylistItem.playlist_id == pl.id)
|
||
.order_by(PlaylistItem.position)
|
||
).scalars()
|
||
assert list(items) == ["held0000000", "old00000000", "new00000000"]
|
||
assert _rows(db, user)["WL:gone0000000"].outcome == "missing"
|
||
assert db.execute(select(Playlist).where(Playlist.user_id == user.id)).scalars().all() == [pl]
|
||
|
||
|
||
@pytest.mark.parametrize("n", [3, 40])
|
||
def test_a_watch_later_list_goes_in_in_statements_that_do_not_grow_with_it(db, make_user, n):
|
||
"""A video at a time, a full list of 5000 was some 20,000 statements under the user's lock."""
|
||
from sqlalchemy import event
|
||
|
||
user = make_user(f"many{n}@example.com")
|
||
ids = [f"wl{i:09d}" for i in range(n)]
|
||
for vid in ids:
|
||
_video(db, vid)
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCH_LATER,
|
||
{v: (T0 + timedelta(minutes=i), None) for i, v in enumerate(ids)},
|
||
)
|
||
db.commit()
|
||
rows = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||
service.watch_later(db, user) # the list made beforehand: its own insert is not this count's
|
||
db.flush()
|
||
statements = []
|
||
engine = db.get_bind()
|
||
listener = lambda *a: statements.append(a[2]) # noqa: E731 — the event's own signature
|
||
event.listen(engine, "before_cursor_execute", listener)
|
||
try:
|
||
assert service.apply_takeout_watch_later(db, user, rows, set())["watch_later_added"] == n
|
||
finally:
|
||
event.remove(engine, "before_cursor_execute", listener)
|
||
db.commit()
|
||
assert len(statements) <= 9, statements
|
||
held = [i.video_id for i in queries.items(db, service.watch_later(db, user).id)]
|
||
assert held == ids
|
||
|
||
|
||
def test_the_job_fetches_the_unknown_watch_later_videos_and_says_what_waits(
|
||
db, make_user, monkeypatch
|
||
):
|
||
user = make_user("job@example.com")
|
||
_video(db, "plain000000")
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCHED,
|
||
{"plain000000": (T0, CHANNEL), "unknown0000": (T0, CHANNEL)},
|
||
)
|
||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"fetched0000": (T0, None)})
|
||
db.commit()
|
||
asked = []
|
||
|
||
def fake_ensure(session, yt, ids):
|
||
# The lookup is the importing user's spend, on the import's own action.
|
||
asked.append((ids, quota._actor_id.get(), quota._action.get()))
|
||
_video(session, "fetched0000")
|
||
session.flush()
|
||
|
||
monkeypatch.setattr(takeout, "ensure_videos", fake_ensure)
|
||
monkeypatch.setattr(takeout, "get_service_user", lambda session: user)
|
||
|
||
result = takeout.run_import(user.id)
|
||
|
||
assert asked == [(["fetched0000"], user.id, quota.QuotaAction.VIDEOS_LOOKUP)]
|
||
assert result == {
|
||
"watched": 1,
|
||
"kept": 0,
|
||
"shorts": 0,
|
||
"watch_later_added": 1,
|
||
"watch_later_present": 0,
|
||
"watch_later_missing": 0,
|
||
"waiting": 1,
|
||
}
|
||
|
||
|
||
def _wl_job(db, make_user, monkeypatch, email, *, fetcher=True, reserve=False):
|
||
user = make_user(email)
|
||
_video(db, "known000000")
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCH_LATER,
|
||
{"known000000": (T0, None), "unknown0000": (T0 + timedelta(minutes=1), None)},
|
||
)
|
||
db.commit()
|
||
asked = []
|
||
monkeypatch.setattr(takeout, "ensure_videos", lambda session, yt, ids: asked.append(ids))
|
||
monkeypatch.setattr(takeout, "get_service_user", lambda session: user if fetcher else None)
|
||
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: reserve)
|
||
return user, asked
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"why", ["no account to ask with", "the budget's reserve", "the user's share"]
|
||
)
|
||
def test_a_watch_later_video_not_asked_about_waits_and_the_known_ones_go_in(
|
||
db, make_user, monkeypatch, why
|
||
):
|
||
user, asked = _wl_job(
|
||
db,
|
||
make_user,
|
||
monkeypatch,
|
||
"wait@example.com",
|
||
fetcher=why != "no account to ask with",
|
||
reserve=why == "the budget's reserve",
|
||
)
|
||
if why == "the user's share":
|
||
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(0, 3600))
|
||
|
||
result = takeout.run_import(user.id)
|
||
|
||
assert asked == []
|
||
assert (result["watch_later_added"], result["watch_later_missing"]) == (1, 0)
|
||
assert result["watch_later_waiting"] == 1
|
||
assert _rows(db, user)["WL:unknown0000"].outcome is None
|
||
|
||
|
||
def test_the_shorts_job_puts_in_the_watch_later_videos_a_run_could_not(db, make_user, monkeypatch):
|
||
user, asked = _wl_job(db, make_user, monkeypatch, "retry@example.com", fetcher=False)
|
||
takeout.run_import(user.id)
|
||
assert asked == []
|
||
|
||
# A YouTube-connected account exists now; the next probe run asks for what waits.
|
||
monkeypatch.setattr(takeout, "get_service_user", lambda session: user)
|
||
|
||
def fetched(session, yt, ids):
|
||
asked.append(ids)
|
||
_video(session, "unknown0000")
|
||
session.flush()
|
||
|
||
monkeypatch.setattr(takeout, "ensure_videos", fetched)
|
||
assert takeout.apply_waiting(db) == {"takeout_watch_later_added": 1}
|
||
assert asked == [["unknown0000"]]
|
||
assert _rows(db, user)["WL:unknown0000"].outcome == "added"
|
||
assert takeout.apply_waiting(db) == {}
|
||
|
||
|
||
def test_one_users_failure_in_the_shorts_job_leaves_the_others_and_the_probe_alone(
|
||
db, make_user, monkeypatch
|
||
):
|
||
"""A Watch later lookup that raised (a revoked token, a refused quota) escaped the Shorts job:
|
||
its run was recorded as failed with its own counts lost, and every user after this one waited."""
|
||
first, second = make_user("first@example.com"), make_user("second@example.com")
|
||
_video(db, "ready000000")
|
||
db.commit()
|
||
service.stage_takeout(db, first, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)})
|
||
service.stage_takeout(db, second, TakeoutKind.WATCHED, {"ready000000": (T0, None)})
|
||
db.commit()
|
||
|
||
def refused(session, yt, ids):
|
||
raise RuntimeError("invalid_grant")
|
||
|
||
monkeypatch.setattr(takeout, "ensure_videos", refused)
|
||
monkeypatch.setattr(takeout, "get_service_user", lambda session: first)
|
||
monkeypatch.setattr(scheduler, "run_shorts", lambda session: {"probed": 4})
|
||
|
||
scheduler._shorts_job()
|
||
|
||
db.expire_all()
|
||
run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one()
|
||
assert run.status == "ok"
|
||
assert run.result == {"probed": 4, "takeout_watched": 1, "takeout_failed": 1}
|
||
assert _states(db, second)["ready000000"].status == "watched"
|
||
assert _rows(db, first)["WL:unknown0000"].outcome is None
|
||
# A lookup that failed spent none of the user's daily share.
|
||
assert not takeout._lookups.exhausted(str(first.id))
|
||
assert takeout._lookups.room(str(first.id)) == 2 * takeout.WATCH_LATER_MAX
|
||
|
||
|
||
def test_a_watch_later_row_staged_while_youtube_is_asked_is_left_for_the_next_run(
|
||
db, make_user, monkeypatch
|
||
):
|
||
user, _ = _wl_job(db, make_user, monkeypatch, "race@example.com")
|
||
|
||
def meanwhile(session, yt, ids):
|
||
# A second upload lands while the lookup runs: its video was never asked about.
|
||
with SessionLocal() as other:
|
||
service.stage_takeout(
|
||
other,
|
||
other.get(User, user.id),
|
||
TakeoutKind.WATCH_LATER,
|
||
{"second00000": (T0, None)},
|
||
)
|
||
other.commit()
|
||
|
||
monkeypatch.setattr(takeout, "ensure_videos", meanwhile)
|
||
takeout.run_import(user.id)
|
||
|
||
rows = _rows(db, user)
|
||
assert rows["WL:unknown0000"].outcome == "missing" # asked, and YouTube has it no longer
|
||
assert rows["WL:second00000"].outcome is None
|
||
|
||
|
||
def test_a_youtube_failure_keeps_what_the_watched_part_already_marked(db, make_user, monkeypatch):
|
||
user = make_user("fails@example.com")
|
||
_video(db, "plain000000")
|
||
db.commit()
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"plain000000": (T0, CHANNEL)})
|
||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"unknown0000": (T0, None)})
|
||
db.commit()
|
||
|
||
def refused(session, yt, ids):
|
||
raise RuntimeError("quota exceeded")
|
||
|
||
monkeypatch.setattr(takeout, "ensure_videos", refused)
|
||
monkeypatch.setattr(takeout, "get_service_user", lambda session: user)
|
||
with pytest.raises(RuntimeError):
|
||
takeout.run_import(user.id)
|
||
|
||
with SessionLocal() as fresh:
|
||
state = fresh.execute(select(VideoState).where(VideoState.user_id == user.id)).scalar_one()
|
||
assert (state.video_id, state.status) == ("plain000000", "watched")
|
||
outcome = fresh.execute(
|
||
select(TakeoutItem.outcome).where(TakeoutItem.video_id == "plain000000")
|
||
).scalar_one()
|
||
assert outcome == "watched"
|
||
|
||
|
||
def test_staging_and_settling_take_the_users_one_lock(db, make_user):
|
||
"""An upload's staging and an import's settling lock the same rows in no common order: one lock
|
||
over both is what keeps the two from deadlocking."""
|
||
user = make_user("lock@example.com")
|
||
db.commit()
|
||
steps = {
|
||
"stage": lambda s, u: service.stage_takeout(
|
||
s, u, TakeoutKind.WATCHED, {"aaaaaaaaaaa": (T0, None)}
|
||
),
|
||
"watched": lambda s, u: service.apply_takeout_watched(s, u),
|
||
"watch later": lambda s, u: service.apply_takeout_watch_later(
|
||
s, u, [(1, "aaaaaaaaaaa")], set()
|
||
),
|
||
# Between the import's read of the un-marks and its marking, one would be overwritten.
|
||
"unmark": lambda s, u: service.unmark(s, u, "aaaaaaaaaaa"),
|
||
"reset": lambda s, u: service.clear_state(s, u, "aaaaaaaaaaa"),
|
||
}
|
||
with SessionLocal() as holder:
|
||
service.lock_takeout(holder, holder.get(User, user.id))
|
||
for name, step in steps.items():
|
||
with SessionLocal() as other:
|
||
other.execute(text("SET lock_timeout = '200ms'"))
|
||
with pytest.raises(OperationalError, match="lock timeout"):
|
||
step(other, other.get(User, user.id))
|
||
pytest.fail(f"{name} did not wait for the user's lock")
|
||
holder.rollback()
|
||
|
||
|
||
# --- the upload --------------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture
|
||
def queued(monkeypatch):
|
||
runs = []
|
||
monkeypatch.setattr(
|
||
scheduler, "run_ad_hoc", lambda job, fn, **kw: runs.append((job, fn.func, fn.args, kw))
|
||
)
|
||
return runs
|
||
|
||
|
||
def test_an_upload_keeps_its_videos_and_queues_the_users_import_with_a_notice(
|
||
db, make_user, as_user, queued
|
||
):
|
||
user = make_user("upload@example.com")
|
||
body = _history(
|
||
_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb", details=[{"name": "From Google Ads"}])
|
||
)
|
||
resp = as_user(user).post("/api/takeout", content=body)
|
||
assert resp.status_code == 200, resp.text
|
||
assert resp.json() == {
|
||
"ok": True,
|
||
"kind": "watched",
|
||
"videos": 1,
|
||
"entries": 2,
|
||
"ads": 1,
|
||
"other": 0,
|
||
"malformed": 0,
|
||
}
|
||
assert list(_rows(db, user)) == ["aaaaaaaaaaa"]
|
||
assert queued == [
|
||
(
|
||
"takeout_import",
|
||
takeout.run_import,
|
||
(user.id,),
|
||
{"actor_id": user.id, "target": f"user:{user.id}", "notify": True},
|
||
)
|
||
]
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("body", "reason"),
|
||
[(b"<html></html>", "takeout_unreadable"), (b"[]", "takeout_no_videos")],
|
||
)
|
||
def test_an_upload_the_import_cannot_use_is_refused_in_the_readers_language(
|
||
db, make_user, as_user, queued, body, reason
|
||
):
|
||
user = make_user("refused@example.com")
|
||
resp = as_user(user).post("/api/takeout", content=body)
|
||
assert resp.status_code == 422
|
||
assert resp.json()["detail"] == {"code": "refused", "reason": reason}
|
||
assert queued == [] and _rows(db, user) == {}
|
||
|
||
|
||
def test_an_upload_over_the_cap_is_refused_while_it_arrives(db, make_user, as_user, monkeypatch):
|
||
user = make_user("big@example.com")
|
||
monkeypatch.setattr(takeout, "MAX_BYTES", 64)
|
||
resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
|
||
assert resp.status_code == 413
|
||
assert _rows(db, user) == {}
|
||
|
||
|
||
def _counting_body(chunks: int = 4, size: int = 1024):
|
||
"""A body that records how much of itself the server pulled (test_upload.py's probe)."""
|
||
pulled: list[int] = []
|
||
|
||
def body():
|
||
for _ in range(chunks):
|
||
pulled.append(size)
|
||
yield b"x" * size
|
||
|
||
return body(), pulled
|
||
|
||
|
||
def test_an_anonymous_upload_is_refused_before_a_byte_of_it_is_read(client):
|
||
body, pulled = _counting_body()
|
||
assert client.post("/api/takeout", content=body).status_code == 401
|
||
assert pulled == [], f"the server read {sum(pulled)} bytes from an anonymous caller"
|
||
|
||
|
||
def test_a_signed_in_upload_has_its_body_read(db, make_user, as_user, queued):
|
||
"""The mutation check for the test above: a route reading nobody's body would pass it too."""
|
||
body, pulled = _counting_body()
|
||
c = as_user(make_user("read@example.com"))
|
||
assert c.post("/api/takeout", content=body).status_code == 422
|
||
assert len(pulled) == 4
|
||
|
||
|
||
def test_the_upload_arrives_with_no_connection_held(
|
||
db, make_user, as_user, override_dep, queued, monkeypatch
|
||
):
|
||
"""The read connection is given back before the body: reading the user's id after that reloaded
|
||
the user and took one straight back out, `idle in transaction` for the whole upload."""
|
||
user = make_user("idle@example.com")
|
||
client = as_user(user)
|
||
|
||
def signed_in(session: Session = Depends(get_db)) -> User:
|
||
# As the real sign-in check: a SELECT on the request's session, which the release ends —
|
||
# and expires what it loaded. (`as_user`'s own resolver reads nothing.)
|
||
session.expire_all()
|
||
return session.get(User, user.id)
|
||
|
||
override_dep(current_user, signed_in)
|
||
sessions, held = [], []
|
||
release, capped = feed_routes.release_read_connection, feed_routes.capped_body
|
||
|
||
def releasing(session):
|
||
sessions.append(session)
|
||
release(session)
|
||
|
||
async def reading(request, limit):
|
||
held.extend(s.in_transaction() for s in sessions)
|
||
return await capped(request, limit)
|
||
|
||
monkeypatch.setattr(feed_routes, "release_read_connection", releasing)
|
||
monkeypatch.setattr(feed_routes, "capped_body", reading)
|
||
resp = client.post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
|
||
assert resp.status_code == 200, resp.text
|
||
assert held == [False]
|
||
|
||
|
||
def test_uploads_past_the_hourly_limit_are_refused_after_their_body_arrived(
|
||
db, make_user, as_user, queued
|
||
):
|
||
c = as_user(make_user("many@example.com"))
|
||
for _ in range(10):
|
||
assert c.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200
|
||
body, pulled = _counting_body()
|
||
resp = c.post("/api/takeout", content=body)
|
||
assert resp.status_code == 429
|
||
assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_cooldown"}
|
||
# Read to the end first: refused mid-upload, the browser would say "network error" instead.
|
||
assert len(pulled) == 4
|
||
assert len(queued) == 10
|
||
|
||
|
||
def test_a_second_upload_while_one_of_the_users_is_still_arriving_is_refused(
|
||
db, make_user, as_user, queued, monkeypatch
|
||
):
|
||
user = make_user("busy@example.com")
|
||
monkeypatch.setattr(feed_routes, "_takeout_uploading", {user.id})
|
||
resp = as_user(user).post("/api/takeout", content=_history(_view("aaaaaaaaaaa")))
|
||
assert resp.status_code == 429
|
||
assert resp.json()["detail"]["reason"] == "takeout_busy"
|
||
assert queued == []
|
||
# Another user's upload is not held up by it, and lets go of its own place when it is done.
|
||
other = as_user(make_user("free@example.com"))
|
||
assert other.post("/api/takeout", content=_history(_view("aaaaaaaaaaa"))).status_code == 200
|
||
assert feed_routes._takeout_uploading == {user.id}
|
||
|
||
|
||
def test_a_third_upload_across_users_while_two_arrive_is_refused_after_its_body(
|
||
db, make_user, as_user, queued, monkeypatch
|
||
):
|
||
"""Each upload's bytes are held whole while it arrives: with no bound across users, a few at
|
||
once could take the process past its memory."""
|
||
monkeypatch.setattr(feed_routes, "_takeout_uploading", {9001, 9002})
|
||
body, pulled = _counting_body()
|
||
resp = as_user(make_user("third@example.com")).post("/api/takeout", content=body)
|
||
assert resp.status_code == 429
|
||
assert resp.json()["detail"]["reason"] == "takeout_server_busy"
|
||
assert len(pulled) == 4
|
||
assert queued == [] and feed_routes._takeout_uploading == {9001, 9002}
|
||
|
||
|
||
def test_an_upload_that_would_keep_too_many_videos_is_refused(
|
||
db, make_user, as_user, queued, monkeypatch
|
||
):
|
||
user = make_user("cap@example.com")
|
||
c = as_user(user)
|
||
monkeypatch.setattr(takeout, "ROWS_MAX", 3)
|
||
two = _history(_view("aaaaaaaaaaa"), _view("bbbbbbbbbbb"))
|
||
assert c.post("/api/takeout", content=two).status_code == 200
|
||
assert c.post("/api/takeout", content=two).status_code == 200 # the same videos again: no more
|
||
resp = c.post("/api/takeout", content=_history(_view("ccccccccccc"), _view("ddddddddddd")))
|
||
assert resp.status_code == 422
|
||
assert resp.json()["detail"] == {"code": "refused", "reason": "takeout_too_many", "limit": 3}
|
||
assert sorted(_rows(db, user)) == ["aaaaaaaaaaa", "bbbbbbbbbbb"]
|
||
assert len(queued) == 2
|
||
|
||
|
||
def test_a_watch_later_file_longer_than_youtube_allows_is_refused(db, make_user, as_user, queued):
|
||
body = "".join(f"v{n:010d},2019-02-07T15:23:30Z\n" for n in range(5001)).encode()
|
||
resp = as_user(make_user("long@example.com")).post("/api/takeout", content=body)
|
||
assert resp.status_code == 422
|
||
assert resp.json()["detail"] == {
|
||
"code": "refused",
|
||
"reason": "takeout_list_too_long",
|
||
"limit": 5000,
|
||
}
|
||
|
||
|
||
# --- the notice --------------------------------------------------------------------------------
|
||
|
||
|
||
def test_an_ad_hoc_run_that_asks_for_it_tells_its_user_how_it_went(db, make_user):
|
||
user = make_user("notice@example.com")
|
||
scheduler._ad_hoc("takeout_import", lambda: {"watched": 3}, user.id, "user:1", True)
|
||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_x")
|
||
db.expire_all()
|
||
notices = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars()
|
||
assert [(n.data["job_id"], n.data["result"]) for n in notices] == [
|
||
("takeout_import", {"watched": 3})
|
||
]
|
||
|
||
|
||
def test_a_failed_run_tells_its_user_the_failure_never_the_upstreams_text(db, make_user):
|
||
user = make_user("failnotice@example.com")
|
||
|
||
def fails():
|
||
raise RuntimeError("(psycopg.errors.Something) [SQL: SELECT secret]")
|
||
|
||
scheduler._ad_hoc("takeout_import", fails, user.id, "user:1", True)
|
||
scheduler._ad_hoc("channel_backfill", fails, user.id, "UC_x")
|
||
db.expire_all()
|
||
(notice,) = db.execute(select(Notification).where(Notification.user_id == user.id)).scalars()
|
||
assert notice.body is None
|
||
assert notice.data == {
|
||
"job_id": "takeout_import",
|
||
"status": "error",
|
||
"summary": None,
|
||
"result": {"failure": "unexpected"},
|
||
}
|
||
# The admin's run row keeps the diagnostic.
|
||
run = db.execute(select(JobRun).where(JobRun.job == "takeout_import")).scalar_one()
|
||
assert "SELECT secret" in run.error
|
||
|
||
|
||
def test_a_queued_import_is_told_apart_from_the_last_one_until_its_run_is_recorded(
|
||
db, make_user, as_user, monkeypatch
|
||
):
|
||
user = make_user("queued@example.com")
|
||
added = []
|
||
|
||
class Held:
|
||
def add_job(self, *args, **kwargs):
|
||
added.append(kwargs["id"])
|
||
|
||
monkeypatch.setattr(scheduler, "_scheduler", Held())
|
||
target = takeout.run_target(user.id)
|
||
scheduler.run_ad_hoc("takeout_import", dict, actor_id=user.id, target=target)
|
||
assert scheduler.ad_hoc_queued("takeout_import", target)
|
||
assert as_user(user).get("/api/takeout").json()["queued"] is True
|
||
|
||
scheduler._ad_hoc_fired("takeout_import", target)
|
||
assert not scheduler.ad_hoc_queued("takeout_import", target)
|
||
assert as_user(user).get("/api/takeout").json()["queued"] is False
|
||
assert added == [f"takeout_import@{target}"]
|
||
|
||
|
||
def test_an_upload_while_its_import_starts_stays_queued_for_the_run_after(
|
||
db, make_user, monkeypatch
|
||
):
|
||
"""A request landing while the run before it writes its row is the run after that one: still
|
||
queued, so the page does not take the run starting now for it."""
|
||
user = make_user("starting@example.com")
|
||
added, ran = [], []
|
||
|
||
class Held:
|
||
def add_job(self, *args, **kwargs):
|
||
added.append(kwargs["id"])
|
||
|
||
monkeypatch.setattr(scheduler, "_scheduler", Held())
|
||
target = takeout.run_target(user.id)
|
||
start = jobruns.start
|
||
|
||
def starting(*args, **kwargs):
|
||
scheduler.run_ad_hoc(
|
||
"takeout_import", lambda: ran.append(2) or {}, actor_id=user.id, target=target
|
||
)
|
||
return start(*args, **kwargs)
|
||
|
||
scheduler.run_ad_hoc(
|
||
"takeout_import", lambda: ran.append(1) or {}, actor_id=user.id, target=target
|
||
)
|
||
monkeypatch.setattr(scheduler.jobruns, "start", starting)
|
||
scheduler._ad_hoc_fired("takeout_import", target)
|
||
assert ran == [1]
|
||
assert scheduler.ad_hoc_queued("takeout_import", target)
|
||
assert added == [f"takeout_import@{target}"] * 2 # the run after it, asked for as it ended
|
||
|
||
monkeypatch.setattr(scheduler.jobruns, "start", start)
|
||
scheduler._ad_hoc_fired("takeout_import", target)
|
||
assert ran == [1, 2]
|
||
assert not scheduler.ad_hoc_queued("takeout_import", target)
|
||
|
||
|
||
# --- the videos that wait ----------------------------------------------------------------------
|
||
|
||
|
||
def test_waiting_videos_are_marked_once_the_shorts_probe_has_cleared_them(db, make_user):
|
||
a, b = make_user("a@example.com"), make_user("b@example.com")
|
||
_video(db, "later000000", probed=False)
|
||
_video(db, "notyet00000", probed=False)
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db, a, TakeoutKind.WATCHED, {"later000000": (T0, None), "notyet00000": (T0, None)}
|
||
)
|
||
service.stage_takeout(db, b, TakeoutKind.WATCHED, {"later000000": (T0, None)})
|
||
db.commit()
|
||
assert takeout.apply_waiting(db) == {}
|
||
|
||
# The probe clears one of them; the other is still unprobed.
|
||
db.get(Video, "later000000").shorts_probed = True
|
||
db.commit()
|
||
assert takeout.apply_waiting(db) == {"takeout_watched": 2}
|
||
|
||
assert _states(db, a)["later000000"].watched_at == T0
|
||
assert _states(db, b)["later000000"].status == "watched"
|
||
assert _rows(db, a)["notyet00000"].outcome is None
|
||
assert takeout.apply_waiting(db) == {}
|
||
|
||
|
||
def test_a_discovery_video_is_not_marked_until_the_catalog_keeps_it(db, make_user):
|
||
"""A mark is a VideoState, which the cleanup jobs read as a reader keeping the video: on a video
|
||
only someone's search or explore brought in, it kept that video (and its channel) for good."""
|
||
user = make_user("discover@example.com")
|
||
for vid, flag in (("search00000", "via_search"), ("explore0000", "via_explore")):
|
||
_video(db, vid)
|
||
db.flush()
|
||
setattr(db.get(Video, vid), flag, True)
|
||
_video(db, "plain000000")
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCHED,
|
||
{v: (T0, None) for v in ("search00000", "explore0000", "plain000000")},
|
||
)
|
||
db.commit()
|
||
|
||
assert takeout.apply_waiting(db) == {"takeout_watched": 1}
|
||
assert set(_states(db, user)) == {"plain000000"}
|
||
assert {v for v, r in _rows(db, user).items() if r.outcome is None} == {
|
||
"search00000",
|
||
"explore0000",
|
||
}
|
||
|
||
# Subscribed to: the catalog keeps the channel's videos of its own, the search one included.
|
||
db.add(Subscription(user_id=user.id, channel_id=CHANNEL))
|
||
db.get(Video, "explore0000").via_explore = False # as subscribing does (catalog.service)
|
||
db.commit()
|
||
assert takeout.apply_waiting(db) == {"takeout_watched": 2}
|
||
assert set(_states(db, user)) == {"plain000000", "search00000", "explore0000"}
|
||
|
||
|
||
def test_the_shorts_job_settles_the_waiting_rows_its_probe_made_ready(db, make_user):
|
||
user = make_user("job-shorts@example.com")
|
||
_video(db, "ready000000") # probed already: the job's probe has nothing left to do
|
||
db.commit()
|
||
service.stage_takeout(db, user, TakeoutKind.WATCHED, {"ready000000": (T0, None)})
|
||
db.commit()
|
||
|
||
scheduler._shorts_job()
|
||
|
||
db.expire_all()
|
||
run = db.execute(select(JobRun).where(JobRun.job == "shorts")).scalar_one()
|
||
assert run.result == {"probed": 0, "shorts": 0, "takeout_watched": 1}
|
||
assert _states(db, user)["ready000000"].status == "watched"
|
||
|
||
|
||
# --- what the settings page reads --------------------------------------------------------------
|
||
|
||
|
||
def test_the_status_is_the_users_own_last_run_and_counts(db, make_user, as_user):
|
||
user, other = make_user("status@example.com"), make_user("other@example.com")
|
||
assert as_user(user).get("/api/takeout").json() == {
|
||
"queued": False,
|
||
"max_bytes": takeout.MAX_BYTES,
|
||
"run": None,
|
||
"watched": {},
|
||
"watch_later": {},
|
||
}
|
||
_video(db, "plain000000")
|
||
_video(db, "short000000", short=True)
|
||
db.commit()
|
||
service.stage_takeout(
|
||
db,
|
||
user,
|
||
TakeoutKind.WATCHED,
|
||
{v: (T0, None) for v in ("plain000000", "short000000", "unknown0000")},
|
||
)
|
||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"plain000000": (T0, None)})
|
||
service.stage_takeout(db, other, TakeoutKind.WATCHED, {"plain000000": (T0, None)})
|
||
service.apply_takeout_watched(db, user)
|
||
db.commit()
|
||
older = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}")
|
||
jobruns.finish(older, RunStatus.OK, summary=None, result={"watched": 0})
|
||
newest = jobruns.start("takeout_import", RunTrigger.AD_HOC, user.id, f"user:{user.id}")
|
||
jobruns.finish(newest, RunStatus.OK, summary=None, result={"watched": 1})
|
||
theirs = jobruns.start("takeout_import", RunTrigger.AD_HOC, other.id, f"user:{other.id}")
|
||
jobruns.finish(theirs, RunStatus.OK, summary=None, result={"watched": 99})
|
||
|
||
body = as_user(user).get("/api/takeout").json()
|
||
|
||
assert body["run"]["status"] == "ok"
|
||
assert body["run"]["result"] == {"watched": 1}
|
||
assert body["run"]["finished_at"] is not None
|
||
assert body["watched"] == {"watched": 1, "short": 1, "waiting": 1}
|
||
assert body["watch_later"] == {"waiting": 1}
|