feat(reader): the channels a Takeout history shows watched but not followed
The import asks YouTube how long the watched videos of the candidate channels are (five or more watched, none the user follows, blocked, put out or owns), keeps the length on the Takeout row and names the channels that make the list: five videos of five minutes or more. The shorts job asks what an import could not. A channel's counted videos come in as an exploration's when its list is opened, ready to play; a channel can be put out of the list and back.
This commit is contained in:
@@ -0,0 +1,399 @@
|
||||
"""The channels a user's Takeout history shows them watching but not following: which channels make
|
||||
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
|
||||
import asks YouTube for it and when, and the channel's watched videos made ready to play."""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import quota
|
||||
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
|
||||
from app.reader import queries, service, takeout
|
||||
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
|
||||
from app.routes import feed as feed_routes
|
||||
from app.sync import videos as sync_videos
|
||||
from app.youtube.client import YouTubeError
|
||||
|
||||
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||||
LONG = 600
|
||||
SHORT = 120
|
||||
|
||||
|
||||
def _ch(n: int) -> str:
|
||||
return f"UC{n:022d}"
|
||||
|
||||
|
||||
def _vid(channel: int, n: int) -> str:
|
||||
return f"c{channel:02d}v{n:07d}"
|
||||
|
||||
|
||||
def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
|
||||
"""Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
|
||||
already asked (None: YouTube no longer has it) — or not asked yet."""
|
||||
service.stage_takeout(
|
||||
db,
|
||||
user,
|
||||
TakeoutKind.WATCHED,
|
||||
{_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
|
||||
)
|
||||
db.flush()
|
||||
if asked:
|
||||
ids = [_vid(channel, i) for i in range(len(lengths))]
|
||||
service.record_takeout_lengths(
|
||||
db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _channel(db, n: int, title: str | None = None):
|
||||
db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _listed(db, user) -> list[tuple[str, int]]:
|
||||
return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]
|
||||
|
||||
|
||||
# --- which channels make the list --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
|
||||
user = make_user("list@example.com")
|
||||
for n in (1, 2, 3):
|
||||
_channel(db, n)
|
||||
_watched(db, user, 1, [LONG] * 5 + [SHORT] * 3) # five long, three short: on it with 5
|
||||
_watched(db, user, 2, [LONG] * 4 + [SHORT] * 6) # ten watched, only four long: off it
|
||||
_watched(db, user, 3, [299, 300, 300, 300, 300, 300]) # five at exactly five minutes: on it
|
||||
# A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
|
||||
assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]
|
||||
|
||||
|
||||
def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
|
||||
user = make_user("gone@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 4 + [None] * 3)
|
||||
assert _listed(db, user) == []
|
||||
|
||||
|
||||
def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
|
||||
user = make_user("order@example.com")
|
||||
for n in (1, 2, 3):
|
||||
_channel(db, n)
|
||||
_watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
|
||||
_watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
|
||||
_watched(db, user, 3, [LONG] * 6, at=T0)
|
||||
assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
|
||||
last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
|
||||
assert last[_ch(3)] == T0 # the last watched of the counted ones
|
||||
|
||||
|
||||
def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
|
||||
user = make_user("off@example.com", yt_channel_id=_ch(4))
|
||||
for n in (1, 2, 3, 4, 5):
|
||||
_channel(db, n)
|
||||
_watched(db, user, n, [LONG] * 5)
|
||||
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
|
||||
db.commit()
|
||||
service.dismiss_watched_channel(db, user, _ch(3))
|
||||
db.commit()
|
||||
assert _listed(db, user) == [(_ch(5), 5)]
|
||||
|
||||
|
||||
def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
|
||||
me, other = make_user("me@example.com"), make_user("other@example.com")
|
||||
_channel(db, 1)
|
||||
for user in (me, other):
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
service.dismiss_watched_channel(db, other, _ch(1))
|
||||
db.commit()
|
||||
assert _listed(db, me) == [(_ch(1), 5)]
|
||||
assert _listed(db, other) == []
|
||||
|
||||
|
||||
def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
|
||||
user = make_user("pending@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 6, asked=False)
|
||||
_watched(db, user, 2, [LONG] * 4, asked=False) # four watched: it can never make the list
|
||||
assert queries.watched_channels_pending(db, user) == 1
|
||||
assert _listed(db, user) == []
|
||||
ids = [_vid(1, i) for i in range(6)]
|
||||
service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
|
||||
db.commit()
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
assert _listed(db, user) == [(_ch(1), 6)]
|
||||
|
||||
|
||||
def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
|
||||
user = make_user("again@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
_watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
|
||||
assert _listed(db, user) == [(_ch(1), 5)]
|
||||
assert queries.takeout_lengths_to_ask(db, user) == []
|
||||
|
||||
|
||||
# --- what the import asks YouTube for ----------------------------------------------------------
|
||||
|
||||
|
||||
class FakeYouTube:
|
||||
"""Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
|
||||
`channels.list` with a title per channel, and records what it was asked."""
|
||||
|
||||
def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
|
||||
self.lengths = lengths or {}
|
||||
self.fail = fail
|
||||
self.videos_asked: list[list[str]] = []
|
||||
self.channels_asked: list[list[str]] = []
|
||||
self.actions: list[tuple[int | None, str]] = []
|
||||
|
||||
def __call__(self, db, user):
|
||||
return self
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def get_videos(self, ids):
|
||||
self.videos_asked.append(list(ids))
|
||||
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||
if self.fail:
|
||||
raise YouTubeError("refused", reason="quotaExceeded", path="videos")
|
||||
return [
|
||||
{"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
|
||||
for v in ids
|
||||
if v in self.lengths
|
||||
]
|
||||
|
||||
def get_channels(self, ids):
|
||||
self.channels_asked.append(list(ids))
|
||||
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||
return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def youtube(monkeypatch, db, make_user):
|
||||
fetcher = make_user("fetcher@example.com")
|
||||
fake = FakeYouTube()
|
||||
monkeypatch.setattr(takeout, "YouTubeClient", fake)
|
||||
monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
|
||||
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
|
||||
return fake
|
||||
|
||||
|
||||
def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
|
||||
user = make_user("import@example.com")
|
||||
_watched(db, user, 1, [None] * 7, asked=False) # the most watched channel
|
||||
_watched(db, user, 2, [None] * 5, asked=False)
|
||||
_watched(db, user, 3, [None] * 4, asked=False) # too few to ever make it: not asked about
|
||||
youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}
|
||||
|
||||
result = takeout.run_import(user.id)
|
||||
|
||||
asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
|
||||
assert youtube.videos_asked == [asked] # the top channel first
|
||||
assert result["lengths_asked"] == 12
|
||||
# Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
|
||||
assert youtube.channels_asked == [[_ch(1)]]
|
||||
assert result["channels_described"] == 1
|
||||
assert {
|
||||
(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
||||
(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
} == set(youtube.actions)
|
||||
db.expire_all()
|
||||
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
# Asked once: a second run asks nothing again, the gone video included.
|
||||
youtube.videos_asked.clear()
|
||||
youtube.channels_asked.clear()
|
||||
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [] and youtube.channels_asked == []
|
||||
|
||||
|
||||
def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
|
||||
db, make_user, youtube, monkeypatch
|
||||
):
|
||||
user = make_user("share@example.com")
|
||||
_watched(db, user, 1, [None] * 8, asked=False)
|
||||
_watched(db, user, 2, [None] * 6, asked=False)
|
||||
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
|
||||
takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
|
||||
db.expire_all()
|
||||
assert queries.watched_channels_pending(db, user) == 1 # channel 2 still has some to ask
|
||||
youtube.videos_asked.clear()
|
||||
takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [] # the share is spent for today
|
||||
|
||||
|
||||
@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
|
||||
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
|
||||
db, make_user, youtube, monkeypatch, why
|
||||
):
|
||||
user = make_user(f"{why}@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
if why == "no_fetcher":
|
||||
monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
|
||||
else:
|
||||
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
|
||||
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == []
|
||||
assert queries.watched_channels_pending(db, user) == 1
|
||||
|
||||
|
||||
def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
|
||||
db, make_user, youtube
|
||||
):
|
||||
user = make_user("fail@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
youtube.fail = True
|
||||
result = takeout.run_import(user.id)
|
||||
assert "lengths_asked" not in result
|
||||
assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
|
||||
db.expire_all()
|
||||
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||||
assert all(r.looked_up_at is None for r in rows) # left to ask again
|
||||
|
||||
|
||||
def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
|
||||
user = make_user("later@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
|
||||
assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
|
||||
db.expire_all()
|
||||
assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]
|
||||
|
||||
|
||||
def test_a_channel_the_user_follows_is_not_asked_about(db, make_user, youtube):
|
||||
user = make_user("follows@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
assert takeout.apply_waiting(db) == {}
|
||||
assert youtube.videos_asked == []
|
||||
|
||||
|
||||
# --- the channel's watched videos, ready to play ------------------------------------------------
|
||||
|
||||
|
||||
def _fake_ensure(calls):
|
||||
def ensure(session, yt, ids, *, explore=False):
|
||||
calls.append((list(ids), explore))
|
||||
for vid in ids:
|
||||
if vid.endswith("4"):
|
||||
continue # gone from YouTube since its length was asked
|
||||
session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
|
||||
session.commit()
|
||||
|
||||
return ensure
|
||||
|
||||
|
||||
def test_a_channels_watched_videos_come_ready_to_play_and_explored(
|
||||
db, make_user, as_user, monkeypatch
|
||||
):
|
||||
user = make_user("play@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
|
||||
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0)) # already in the catalog
|
||||
db.commit()
|
||||
calls = []
|
||||
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
|
||||
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
|
||||
|
||||
resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
# Only the counted ones, newest first; the short one and the gone ones are left out.
|
||||
assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
|
||||
assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
|
||||
assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
|
||||
assert body["gone"] == 2 # one YouTube had lost before, one since
|
||||
assert body["ok"] is True
|
||||
db.expire_all()
|
||||
assert db.execute(
|
||||
select(ExploredChannel).where(
|
||||
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
|
||||
)
|
||||
).scalar_one()
|
||||
assert (
|
||||
db.get(Channel, _ch(1)).from_explore is True
|
||||
) # nobody follows it: the cleanup's to reclaim
|
||||
|
||||
|
||||
def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
|
||||
user = make_user("refused@example.com")
|
||||
_channel(db, 1)
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
client = as_user(user)
|
||||
assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
|
||||
assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404
|
||||
|
||||
|
||||
def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
|
||||
db.add(Channel(id=_ch(1), title="C"))
|
||||
db.commit()
|
||||
|
||||
class YT:
|
||||
def get_videos(self, ids):
|
||||
return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]
|
||||
|
||||
sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
|
||||
sync_videos.ensure_videos(db, YT(), ["organicvid0"])
|
||||
db.expire_all()
|
||||
assert db.get(Video, "exploredvid").via_explore is True
|
||||
assert db.get(Video, "organicvid0").via_explore is False
|
||||
|
||||
|
||||
# --- the routes -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
|
||||
db, make_user, as_user
|
||||
):
|
||||
user = make_user("route@example.com")
|
||||
client = as_user(user)
|
||||
assert client.get("/api/takeout/channels").json() == {
|
||||
"imported": False,
|
||||
"pending": 0,
|
||||
"channels": [],
|
||||
}
|
||||
_channel(db, 1, "Some channel")
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
_watched(db, user, 2, [None] * 5, asked=False)
|
||||
body = client.get("/api/takeout/channels").json()
|
||||
assert body["imported"] is True and body["pending"] == 1
|
||||
assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
|
||||
(_ch(1), "Some channel", 5)
|
||||
]
|
||||
assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")
|
||||
|
||||
|
||||
def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
|
||||
user = make_user("dismiss@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
client = as_user(user)
|
||||
url = f"/api/takeout/channels/{_ch(1)}/dismissal"
|
||||
assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||
assert client.put(url).status_code == 200 # again: no error
|
||||
assert client.get("/api/takeout/channels").json()["channels"] == []
|
||||
assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||
assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
|
||||
db.expire_all()
|
||||
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
|
||||
|
||||
|
||||
def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
|
||||
db, make_user, as_user
|
||||
):
|
||||
user = make_user("long@example.com")
|
||||
assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422
|
||||
Reference in New Issue
Block a user