Files
siftlode/backend/tests/test_watched_channels.py
T
peter 33dd9ff054 fix(reader): the watched-channels list keeps its channels, its shares and its retries
- A channel someone follows: the videos its exploration brings in (the list's
  dialog, the channel page's explore) come in as the catalog's own, not
  flagged via_explore, which subscribing would never clear and which kept
  their watched marks waiting for good.
- The explore cleanup keeps the row of a channel a Takeout history shows
  watched (it still reclaims the videos), so a looked-at channel no longer
  drops off the list.
- The Shorts job takes up only users whose list still has lengths to ask or
  channels to name, by each user's own follows, blocks and dismissals; a
  naming that failed is tried again, and the list says it is not complete.
- The length lookups have a daily share of their own: a large history no
  longer leaves the export's Watch later file waiting a day.
- Only a channel of the user's own history can be put out of the list.
- The dialog's videos have a query root of their own: the list's
  invalidations no longer re-ask (and re-explore) under an open dialog.
2026-10-24 01:10:29 +02:00

501 lines
20 KiB
Python

"""The channels a user's Takeout history shows them watching but not following: which channels make
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
import asks YouTube for it and when, and the channel's watched videos made ready to play."""
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select, update
from app import quota
from app.catalog import service as catalog_service
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
from app.reader import queries, service, takeout
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
from app.routes import feed as feed_routes
from app.sync import explore as sync_explore
from app.sync import videos as sync_videos
from app.youtube.client import YouTubeError
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
LONG = 600
SHORT = 120
def _ch(n: int) -> str:
return f"UC{n:022d}"
def _vid(channel: int, n: int) -> str:
return f"c{channel:02d}v{n:07d}"
def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
"""Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
already asked (None: YouTube no longer has it) — or not asked yet."""
service.stage_takeout(
db,
user,
TakeoutKind.WATCHED,
{_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
)
db.flush()
if asked:
ids = [_vid(channel, i) for i in range(len(lengths))]
service.record_takeout_lengths(
db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
)
db.commit()
def _channel(db, n: int, title: str | None = None):
db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
db.commit()
def _listed(db, user) -> list[tuple[str, int]]:
return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]
# --- which channels make the list --------------------------------------------------------------
def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
user = make_user("list@example.com")
for n in (1, 2, 3):
_channel(db, n)
_watched(db, user, 1, [LONG] * 5 + [SHORT] * 3) # five long, three short: on it with 5
_watched(db, user, 2, [LONG] * 4 + [SHORT] * 6) # ten watched, only four long: off it
_watched(db, user, 3, [299, 300, 300, 300, 300, 300]) # five at exactly five minutes: on it
# A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]
def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
user = make_user("gone@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 4 + [None] * 3)
assert _listed(db, user) == []
def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
user = make_user("order@example.com")
for n in (1, 2, 3):
_channel(db, n)
_watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
_watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
_watched(db, user, 3, [LONG] * 6, at=T0)
assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
assert last[_ch(3)] == T0 # the last watched of the counted ones
def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
user = make_user("off@example.com", yt_channel_id=_ch(4))
for n in (1, 2, 3, 4, 5):
_channel(db, n)
_watched(db, user, n, [LONG] * 5)
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
db.commit()
service.dismiss_watched_channel(db, user, _ch(3))
db.commit()
assert _listed(db, user) == [(_ch(5), 5)]
def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
me, other = make_user("me@example.com"), make_user("other@example.com")
_channel(db, 1)
for user in (me, other):
_watched(db, user, 1, [LONG] * 5)
db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
db.commit()
service.dismiss_watched_channel(db, other, _ch(1))
db.commit()
assert _listed(db, me) == [(_ch(1), 5)]
assert _listed(db, other) == []
def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
user = make_user("pending@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 6, asked=False)
_watched(db, user, 2, [LONG] * 4, asked=False) # four watched: it can never make the list
assert queries.watched_channels_pending(db, user) == 1
assert _listed(db, user) == []
ids = [_vid(1, i) for i in range(6)]
service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
db.commit()
assert queries.watched_channels_pending(db, user) == 0
assert _listed(db, user) == [(_ch(1), 6)]
def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
user = make_user("again@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
_watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
assert _listed(db, user) == [(_ch(1), 5)]
assert queries.takeout_lengths_to_ask(db, user) == []
# --- what the import asks YouTube for ----------------------------------------------------------
class FakeYouTube:
"""Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
`channels.list` with a title per channel, and records what it was asked."""
def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
self.lengths = lengths or {}
self.fail = fail
self.fail_channels = False
self.videos_asked: list[list[str]] = []
self.channels_asked: list[list[str]] = []
self.actions: list[tuple[int | None, str]] = []
def __call__(self, db, user):
return self
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def get_videos(self, ids):
self.videos_asked.append(list(ids))
self.actions.append((quota._actor_id.get(), quota._action.get()))
if self.fail:
raise YouTubeError("refused", reason="quotaExceeded", path="videos")
return [
{"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
for v in ids
if v in self.lengths
]
def get_channels(self, ids):
self.channels_asked.append(list(ids))
self.actions.append((quota._actor_id.get(), quota._action.get()))
if self.fail_channels:
raise YouTubeError("refused", reason="backendError", path="channels")
return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]
@pytest.fixture
def youtube(monkeypatch, db, make_user):
fetcher = make_user("fetcher@example.com")
fake = FakeYouTube()
monkeypatch.setattr(takeout, "YouTubeClient", fake)
monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
return fake
def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
user = make_user("import@example.com")
_watched(db, user, 1, [None] * 7, asked=False) # the most watched channel
_watched(db, user, 2, [None] * 5, asked=False)
_watched(db, user, 3, [None] * 4, asked=False) # too few to ever make it: not asked about
youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}
result = takeout.run_import(user.id)
asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
assert youtube.videos_asked == [asked] # the top channel first
assert result["lengths_asked"] == 12
# Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
assert youtube.channels_asked == [[_ch(1)]]
assert result["channels_described"] == 1
assert {
(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
} == set(youtube.actions)
db.expire_all()
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
assert queries.watched_channels_pending(db, user) == 0
# Asked once: a second run asks nothing again, the gone video included.
youtube.videos_asked.clear()
youtube.channels_asked.clear()
assert "lengths_asked" not in takeout.run_import(user.id)
assert youtube.videos_asked == [] and youtube.channels_asked == []
def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
db, make_user, youtube, monkeypatch
):
user = make_user("share@example.com")
_watched(db, user, 1, [None] * 8, asked=False)
_watched(db, user, 2, [None] * 6, asked=False)
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
takeout.run_import(user.id)
assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
db.expire_all()
assert queries.watched_channels_pending(db, user) == 1 # channel 2 still has some to ask
youtube.videos_asked.clear()
takeout.run_import(user.id)
assert youtube.videos_asked == [] # the share is spent for today
@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
db, make_user, youtube, monkeypatch, why
):
user = make_user(f"{why}@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
if why == "no_fetcher":
monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
else:
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
assert "lengths_asked" not in takeout.run_import(user.id)
assert youtube.videos_asked == []
assert queries.watched_channels_pending(db, user) == 1
def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
db, make_user, youtube
):
user = make_user("fail@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.fail = True
result = takeout.run_import(user.id)
assert "lengths_asked" not in result
assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
db.expire_all()
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
assert all(r.looked_up_at is None for r in rows) # left to ask again
def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
user = make_user("later@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
db.expire_all()
assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]
@pytest.mark.parametrize("why", ["follows", "blocked", "dismissed", "own"])
def test_a_channel_off_the_list_is_not_asked_about_nor_looked_at_again(
db, make_user, youtube, monkeypatch, why
):
user = make_user(f"off-{why}@example.com", yt_channel_id=_ch(1) if why == "own" else None)
_channel(db, 1)
_watched(db, user, 1, [None] * 5, asked=False)
if why == "follows":
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
elif why == "blocked":
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
elif why == "dismissed":
service.dismiss_watched_channel(db, user, _ch(1))
db.commit()
runs = []
monkeypatch.setattr(takeout, "_channel_list", lambda session, u: runs.append(u.id) or {})
assert takeout.apply_waiting(db) == {}
# Its rows are never asked about, so they would stay "to ask" for good: the Shorts job must not
# take the user up again for them on every run.
assert runs == []
assert youtube.videos_asked == []
def test_naming_the_channels_is_tried_again_and_meanwhile_the_list_says_it_is_not_complete(
db, make_user, youtube
):
user = make_user("unnamed@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
youtube.fail_channels = True
takeout.run_import(user.id)
db.expire_all()
# Every length is known, the channel could not be named: not on the list, and not complete.
assert queries.takeout_lengths_to_ask(db, user) == []
assert _listed(db, user) == []
assert queries.watched_channels_pending(db, user) == 1
youtube.fail_channels = False
takeout.apply_waiting(db)
db.expire_all()
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 5)]
assert queries.watched_channels_pending(db, user) == 0
def test_the_lengths_leave_the_watch_later_lookups_their_own_share(
db, make_user, youtube, monkeypatch
):
user = make_user("shares@example.com")
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
_watched(db, user, 1, [None] * 15, asked=False)
takeout.run_import(user.id) # the history: its lengths spend all they may today
youtube.videos_asked.clear()
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"later000001": (T0, None)})
db.commit()
takeout.run_import(user.id) # then the export's Watch later file
assert ["later000001"] in youtube.videos_asked
# --- the channel's watched videos, ready to play ------------------------------------------------
def _fake_ensure(calls):
def ensure(session, yt, ids, *, explore=False):
calls.append((list(ids), explore))
for vid in ids:
if vid.endswith("4"):
continue # gone from YouTube since its length was asked
session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
session.commit()
return ensure
def test_a_channels_watched_videos_come_ready_to_play_and_explored(
db, make_user, as_user, monkeypatch
):
user = make_user("play@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0)) # already in the catalog
db.commit()
calls = []
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")
assert resp.status_code == 200, resp.text
body = resp.json()
# Only the counted ones, newest first; the short one and the gone ones are left out.
assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
assert body["gone"] == 2 # one YouTube had lost before, one since
assert body["ok"] is True
db.expire_all()
assert db.execute(
select(ExploredChannel).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
)
).scalar_one()
assert (
db.get(Channel, _ch(1)).from_explore is True
) # nobody follows it: the cleanup's to reclaim
def test_a_followed_channels_watched_videos_come_in_as_the_catalogs_own(
db, make_user, as_user, monkeypatch
):
user, follower = make_user("viewer@example.com"), make_user("follower@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
db.add(Subscription(user_id=follower.id, channel_id=_ch(1)))
db.commit()
calls = []
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
# Someone follows it, so it is no exploration's to reclaim: a video flagged as one would never
# be cleared again (subscribing clears only an explored channel's) and never marked watched.
assert calls == [([_vid(1, i) for i in range(5)], False)]
db.expire_all()
assert db.get(Channel, _ch(1)).from_explore is False
def test_the_cleanup_keeps_a_channel_a_watch_history_lists_and_reclaims_its_videos(db, make_user):
user = make_user("kept@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
catalog_service.explore(db, user, db.get(Channel, _ch(1)))
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0, via_explore=True))
db.add(Channel(id=_ch(2), from_explore=True)) # explored, in nobody's history
db.commit()
db.execute(update(ExploredChannel).values(created_at=T0 - timedelta(days=365)))
db.commit()
out = sync_explore.purge_unkept_explored(db)
# The unplayed video goes (opening the list's videos brings it back); the channel stays listed.
assert (out["videos_deleted"], out["channels_deleted"]) == (1, 1)
db.expire_all()
assert db.get(Channel, _ch(2)) is None
assert _listed(db, user) == [(_ch(1), 5)]
def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
user = make_user("refused@example.com")
_channel(db, 1)
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
db.commit()
client = as_user(user)
assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404
def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
db.add(Channel(id=_ch(1), title="C"))
db.commit()
class YT:
def get_videos(self, ids):
return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]
sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
sync_videos.ensure_videos(db, YT(), ["organicvid0"])
db.expire_all()
assert db.get(Video, "exploredvid").via_explore is True
assert db.get(Video, "organicvid0").via_explore is False
# --- the routes -----------------------------------------------------------------------------------
def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
db, make_user, as_user
):
user = make_user("route@example.com")
client = as_user(user)
assert client.get("/api/takeout/channels").json() == {
"imported": False,
"pending": 0,
"channels": [],
}
_channel(db, 1, "Some channel")
_watched(db, user, 1, [LONG] * 5)
_watched(db, user, 2, [None] * 5, asked=False)
body = client.get("/api/takeout/channels").json()
assert body["imported"] is True and body["pending"] == 1
assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
(_ch(1), "Some channel", 5)
]
assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")
def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
user = make_user("dismiss@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
client = as_user(user)
url = f"/api/takeout/channels/{_ch(1)}/dismissal"
assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
assert client.put(url).status_code == 200 # again: no error
assert client.get("/api/takeout/channels").json()["channels"] == []
assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
db.expire_all()
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
def test_only_a_channel_of_the_users_own_history_can_be_put_out_of_the_list(db, make_user, as_user):
user, other = make_user("mine@example.com"), make_user("theirs@example.com")
_channel(db, 1)
_watched(db, other, 1, [LONG] * 5)
client = as_user(user)
assert client.put(f"/api/takeout/channels/{_ch(1)}/dismissal").status_code == 404
assert client.put(f"/api/takeout/channels/{_ch(9)}/dismissal").status_code == 404
db.expire_all()
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
db, make_user, as_user
):
user = make_user("long@example.com")
assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422