Files
siftlode/backend/tests/test_watched_channels.py
peter bc8fe2cc6c fix(catalog): a follow from YouTube keeps an explored channel, and the list counts only what can come
A subscription pulled in from YouTube left an explored channel flagged as an exploration's, so its
later videos came in hidden from the Library with their watched marks waiting for good; a subscribe
landing while an exploration's videos came in left those flagged too. One step, settle_explored,
now holds the rule by who follows the channel now: the subscribe here, the subscription import, the
watched-channel videos and the explore page all end with it. Migration 0116 clears what the gap
left behind.

The list's "may still grow" count no longer counts a channel it already shows from a stub row, and
the comments on the cleanup's keep rule and on the in-memory daily shares say what they do.
2026-10-24 11:29:14 +02:00

539 lines
22 KiB
Python

"""The channels a user's Takeout history shows them watching but not following: which channels make
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
import asks YouTube for it and when, and the channel's watched videos made ready to play."""
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import select, update
from app import quota
from app.catalog import service as catalog_service
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
from app.db import SessionLocal
from app.reader import queries, service, takeout
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
from app.routes import feed as feed_routes
from app.sync import explore as sync_explore
from app.sync import videos as sync_videos
from app.youtube.client import YouTubeError
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
LONG = 600
SHORT = 120
def _ch(n: int) -> str:
return f"UC{n:022d}"
def _vid(channel: int, n: int) -> str:
return f"c{channel:02d}v{n:07d}"
def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
"""Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
already asked (None: YouTube no longer has it) — or not asked yet."""
service.stage_takeout(
db,
user,
TakeoutKind.WATCHED,
{_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
)
db.flush()
if asked:
ids = [_vid(channel, i) for i in range(len(lengths))]
service.record_takeout_lengths(
db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
)
db.commit()
def _channel(db, n: int, title: str | None = None):
db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
db.commit()
def _listed(db, user) -> list[tuple[str, int]]:
return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]
# --- which channels make the list --------------------------------------------------------------
def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
user = make_user("list@example.com")
for n in (1, 2, 3):
_channel(db, n)
_watched(db, user, 1, [LONG] * 5 + [SHORT] * 3) # five long, three short: on it with 5
_watched(db, user, 2, [LONG] * 4 + [SHORT] * 6) # ten watched, only four long: off it
_watched(db, user, 3, [299, 300, 300, 300, 300, 300]) # five at exactly five minutes: on it
# A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]
def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
user = make_user("gone@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 4 + [None] * 3)
assert _listed(db, user) == []
def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
user = make_user("order@example.com")
for n in (1, 2, 3):
_channel(db, n)
_watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
_watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
_watched(db, user, 3, [LONG] * 6, at=T0)
assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
assert last[_ch(3)] == T0 # the last watched of the counted ones
def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
user = make_user("off@example.com", yt_channel_id=_ch(4))
for n in (1, 2, 3, 4, 5):
_channel(db, n)
_watched(db, user, n, [LONG] * 5)
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
db.commit()
service.dismiss_watched_channel(db, user, _ch(3))
db.commit()
assert _listed(db, user) == [(_ch(5), 5)]
def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
me, other = make_user("me@example.com"), make_user("other@example.com")
_channel(db, 1)
for user in (me, other):
_watched(db, user, 1, [LONG] * 5)
db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
db.commit()
service.dismiss_watched_channel(db, other, _ch(1))
db.commit()
assert _listed(db, me) == [(_ch(1), 5)]
assert _listed(db, other) == []
def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
user = make_user("pending@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 6, asked=False)
_watched(db, user, 2, [LONG] * 4, asked=False) # four watched: it can never make the list
assert queries.watched_channels_pending(db, user) == 1
assert _listed(db, user) == []
ids = [_vid(1, i) for i in range(6)]
service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
db.commit()
assert queries.watched_channels_pending(db, user) == 0
assert _listed(db, user) == [(_ch(1), 6)]
def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
user = make_user("again@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
_watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
assert _listed(db, user) == [(_ch(1), 5)]
assert queries.takeout_lengths_to_ask(db, user) == []
# --- what the import asks YouTube for ----------------------------------------------------------
class FakeYouTube:
"""Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
`channels.list` with a title per channel, and records what it was asked."""
def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
self.lengths = lengths or {}
self.fail = fail
self.fail_channels = False
self.videos_asked: list[list[str]] = []
self.channels_asked: list[list[str]] = []
self.actions: list[tuple[int | None, str]] = []
def __call__(self, db, user):
return self
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def get_videos(self, ids):
self.videos_asked.append(list(ids))
self.actions.append((quota._actor_id.get(), quota._action.get()))
if self.fail:
raise YouTubeError("refused", reason="quotaExceeded", path="videos")
return [
{"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
for v in ids
if v in self.lengths
]
def get_channels(self, ids):
self.channels_asked.append(list(ids))
self.actions.append((quota._actor_id.get(), quota._action.get()))
if self.fail_channels:
raise YouTubeError("refused", reason="backendError", path="channels")
return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]
@pytest.fixture
def youtube(monkeypatch, db, make_user):
fetcher = make_user("fetcher@example.com")
fake = FakeYouTube()
monkeypatch.setattr(takeout, "YouTubeClient", fake)
monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
return fake
def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
user = make_user("import@example.com")
_watched(db, user, 1, [None] * 7, asked=False) # the most watched channel
_watched(db, user, 2, [None] * 5, asked=False)
_watched(db, user, 3, [None] * 4, asked=False) # too few to ever make it: not asked about
youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}
result = takeout.run_import(user.id)
asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
assert youtube.videos_asked == [asked] # the top channel first
assert result["lengths_asked"] == 12
# Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
assert youtube.channels_asked == [[_ch(1)]]
assert result["channels_described"] == 1
assert {
(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
} == set(youtube.actions)
db.expire_all()
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
assert queries.watched_channels_pending(db, user) == 0
# Asked once: a second run asks nothing again, the gone video included.
youtube.videos_asked.clear()
youtube.channels_asked.clear()
assert "lengths_asked" not in takeout.run_import(user.id)
assert youtube.videos_asked == [] and youtube.channels_asked == []
def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
db, make_user, youtube, monkeypatch
):
user = make_user("share@example.com")
_watched(db, user, 1, [None] * 8, asked=False)
_watched(db, user, 2, [None] * 6, asked=False)
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
takeout.run_import(user.id)
assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
db.expire_all()
assert queries.watched_channels_pending(db, user) == 1 # channel 2 still has some to ask
youtube.videos_asked.clear()
takeout.run_import(user.id)
assert youtube.videos_asked == [] # the share is spent for today
@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
db, make_user, youtube, monkeypatch, why
):
user = make_user(f"{why}@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
if why == "no_fetcher":
monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
else:
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
assert "lengths_asked" not in takeout.run_import(user.id)
assert youtube.videos_asked == []
assert queries.watched_channels_pending(db, user) == 1
def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
db, make_user, youtube
):
user = make_user("fail@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.fail = True
result = takeout.run_import(user.id)
assert "lengths_asked" not in result
assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
db.expire_all()
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
assert all(r.looked_up_at is None for r in rows) # left to ask again
def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
user = make_user("later@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
db.expire_all()
assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]
@pytest.mark.parametrize("why", ["follows", "blocked", "dismissed", "own"])
def test_a_channel_off_the_list_is_not_asked_about_nor_looked_at_again(
db, make_user, youtube, monkeypatch, why
):
user = make_user(f"off-{why}@example.com", yt_channel_id=_ch(1) if why == "own" else None)
_channel(db, 1)
_watched(db, user, 1, [None] * 5, asked=False)
if why == "follows":
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
elif why == "blocked":
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
elif why == "dismissed":
service.dismiss_watched_channel(db, user, _ch(1))
db.commit()
runs = []
monkeypatch.setattr(takeout, "_channel_list", lambda session, u: runs.append(u.id) or {})
assert takeout.apply_waiting(db) == {}
# Its rows are never asked about, so they would stay "to ask" for good: the Shorts job must not
# take the user up again for them on every run.
assert runs == []
assert youtube.videos_asked == []
def test_naming_the_channels_is_tried_again_and_meanwhile_the_list_says_it_is_not_complete(
db, make_user, youtube
):
user = make_user("unnamed@example.com")
_watched(db, user, 1, [None] * 5, asked=False)
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
youtube.fail_channels = True
takeout.run_import(user.id)
db.expire_all()
# Every length is known, the channel could not be named: not on the list, and not complete.
assert queries.takeout_lengths_to_ask(db, user) == []
assert _listed(db, user) == []
assert queries.watched_channels_pending(db, user) == 1
youtube.fail_channels = False
takeout.apply_waiting(db)
db.expire_all()
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 5)]
assert queries.watched_channels_pending(db, user) == 0
def test_a_listed_channel_still_to_be_described_is_not_counted_as_one_to_come(db, make_user):
user = make_user("stub@example.com")
# A stub the catalog made when one of its videos came in: a title, no details yet.
db.add(Channel(id=_ch(1), title="From a video"))
db.commit()
_watched(db, user, 1, [LONG] * 5)
assert queries.watched_channels_to_describe(db, user) == [_ch(1)]
assert _listed(db, user) == [(_ch(1), 5)]
# It is on the list already: "the list may grow" would promise one more that cannot come.
assert queries.watched_channels_pending(db, user) == 0
def test_the_lengths_leave_the_watch_later_lookups_their_own_share(
db, make_user, youtube, monkeypatch
):
user = make_user("shares@example.com")
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
_watched(db, user, 1, [None] * 15, asked=False)
takeout.run_import(user.id) # the history: its lengths spend all they may today
youtube.videos_asked.clear()
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"later000001": (T0, None)})
db.commit()
takeout.run_import(user.id) # then the export's Watch later file
assert ["later000001"] in youtube.videos_asked
# --- the channel's watched videos, ready to play ------------------------------------------------
def _fake_ensure(calls):
def ensure(session, yt, ids, *, explore=False):
calls.append((list(ids), explore))
for vid in ids:
if vid.endswith("4"):
continue # gone from YouTube since its length was asked
session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
session.commit()
return ensure
def test_a_channels_watched_videos_come_ready_to_play_and_explored(
db, make_user, as_user, monkeypatch
):
user = make_user("play@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0)) # already in the catalog
db.commit()
calls = []
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")
assert resp.status_code == 200, resp.text
body = resp.json()
# Only the counted ones, newest first; the short one and the gone ones are left out.
assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
assert body["gone"] == 2 # one YouTube had lost before, one since
assert body["ok"] is True
db.expire_all()
assert db.execute(
select(ExploredChannel).where(
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
)
).scalar_one()
assert (
db.get(Channel, _ch(1)).from_explore is True
) # nobody follows it: the cleanup's to reclaim
def test_a_followed_channels_watched_videos_come_in_as_the_catalogs_own(
db, make_user, as_user, monkeypatch
):
user, follower = make_user("viewer@example.com"), make_user("follower@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
db.add(Subscription(user_id=follower.id, channel_id=_ch(1)))
db.commit()
calls = []
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
# Someone follows it, so it is no exploration's to reclaim: a video flagged as one would never
# be cleared again (subscribing clears only an explored channel's) and never marked watched.
assert calls == [([_vid(1, i) for i in range(5)], False)]
db.expire_all()
assert db.get(Channel, _ch(1)).from_explore is False
def test_a_follow_landing_while_the_videos_come_in_leaves_none_of_them_flagged(
db, make_user, as_user, monkeypatch
):
user, follower = make_user("racer@example.com"), make_user("late@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
fake = _fake_ensure(calls := [])
def ensure(session, yt, ids, *, explore=False):
# Someone subscribes, in a request of their own, while YouTube answers this one.
with SessionLocal() as other:
ch = other.get(Channel, _ch(1))
catalog_service.record_subscription(other, follower, ch, "sub-late")
other.commit()
fake(session, yt, ids, explore=explore)
monkeypatch.setattr(feed_routes, "ensure_videos", ensure)
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
assert calls[0][1] is True # asked as the exploration's
db.expire_all()
flagged = db.scalars(select(Video.id).where(Video.via_explore.is_(True))).all()
assert (db.get(Channel, _ch(1)).from_explore, flagged) == (False, [])
def test_the_cleanup_keeps_a_channel_a_watch_history_lists_and_reclaims_its_videos(db, make_user):
user = make_user("kept@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
catalog_service.explore(db, user, db.get(Channel, _ch(1)))
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0, via_explore=True))
db.add(Channel(id=_ch(2), from_explore=True)) # explored, in nobody's history
db.commit()
db.execute(update(ExploredChannel).values(created_at=T0 - timedelta(days=365)))
db.commit()
out = sync_explore.purge_unkept_explored(db)
# The unplayed video goes (opening the list's videos brings it back); the channel stays listed.
assert (out["videos_deleted"], out["channels_deleted"]) == (1, 1)
db.expire_all()
assert db.get(Channel, _ch(2)) is None
assert _listed(db, user) == [(_ch(1), 5)]
def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
user = make_user("refused@example.com")
_channel(db, 1)
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
db.commit()
client = as_user(user)
assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404
def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
db.add(Channel(id=_ch(1), title="C"))
db.commit()
class YT:
def get_videos(self, ids):
return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]
sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
sync_videos.ensure_videos(db, YT(), ["organicvid0"])
db.expire_all()
assert db.get(Video, "exploredvid").via_explore is True
assert db.get(Video, "organicvid0").via_explore is False
# --- the routes -----------------------------------------------------------------------------------
def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
db, make_user, as_user
):
user = make_user("route@example.com")
client = as_user(user)
assert client.get("/api/takeout/channels").json() == {
"imported": False,
"pending": 0,
"channels": [],
}
_channel(db, 1, "Some channel")
_watched(db, user, 1, [LONG] * 5)
_watched(db, user, 2, [None] * 5, asked=False)
body = client.get("/api/takeout/channels").json()
assert body["imported"] is True and body["pending"] == 1
assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
(_ch(1), "Some channel", 5)
]
assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")
def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
user = make_user("dismiss@example.com")
_channel(db, 1)
_watched(db, user, 1, [LONG] * 5)
client = as_user(user)
url = f"/api/takeout/channels/{_ch(1)}/dismissal"
assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
assert client.put(url).status_code == 200 # again: no error
assert client.get("/api/takeout/channels").json()["channels"] == []
assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
db.expire_all()
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
def test_only_a_channel_of_the_users_own_history_can_be_put_out_of_the_list(db, make_user, as_user):
user, other = make_user("mine@example.com"), make_user("theirs@example.com")
_channel(db, 1)
_watched(db, other, 1, [LONG] * 5)
client = as_user(user)
assert client.put(f"/api/takeout/channels/{_ch(1)}/dismissal").status_code == 404
assert client.put(f"/api/takeout/channels/{_ch(9)}/dismissal").status_code == 404
db.expire_all()
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
db, make_user, as_user
):
user = make_user("long@example.com")
assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422