siftlode/backend/tests/test_watched_channels.py

"""The channels a user's Takeout history shows them watching but not following: which channels make
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
import asks YouTube for it and when, and the channel's watched videos made ready to play."""

from datetime import datetime, timedelta, timezone

import pytest
from sqlalchemy import select, update

from app import quota
from app.catalog import service as catalog_service
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
from app.db import SessionLocal
from app.reader import queries, service, takeout
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
from app.routes import feed as feed_routes
from app.sync import explore as sync_explore
from app.sync import videos as sync_videos
from app.youtube.client import YouTubeError

T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
LONG = 600
SHORT = 120


def _ch(n: int) -> str:
    return f"UC{n:022d}"


def _vid(channel: int, n: int) -> str:
    return f"c{channel:02d}v{n:07d}"


def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
    """Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
    already asked (None: YouTube no longer has it) — or not asked yet."""
    service.stage_takeout(
        db,
        user,
        TakeoutKind.WATCHED,
        {_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
    )
    db.flush()
    if asked:
        ids = [_vid(channel, i) for i in range(len(lengths))]
        service.record_takeout_lengths(
            db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
        )
    db.commit()


def _channel(db, n: int, title: str | None = None):
    db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
    db.commit()


def _listed(db, user) -> list[tuple[str, int]]:
    return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]


# --- which channels make the list --------------------------------------------------------------


def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
    user = make_user("list@example.com")
    for n in (1, 2, 3):
        _channel(db, n)
    _watched(db, user, 1, [LONG] * 5 + [SHORT] * 3)  # five long, three short: on it with 5
    _watched(db, user, 2, [LONG] * 4 + [SHORT] * 6)  # ten watched, only four long: off it
    _watched(db, user, 3, [299, 300, 300, 300, 300, 300])  # five at exactly five minutes: on it
    # A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
    assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]


def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
    user = make_user("gone@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 4 + [None] * 3)
    assert _listed(db, user) == []


def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
    user = make_user("order@example.com")
    for n in (1, 2, 3):
        _channel(db, n)
    _watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
    _watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
    _watched(db, user, 3, [LONG] * 6, at=T0)
    assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
    last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
    assert last[_ch(3)] == T0  # the last watched of the counted ones


def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
    user = make_user("off@example.com", yt_channel_id=_ch(4))
    for n in (1, 2, 3, 4, 5):
        _channel(db, n)
        _watched(db, user, n, [LONG] * 5)
    db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
    db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
    db.commit()
    service.dismiss_watched_channel(db, user, _ch(3))
    db.commit()
    assert _listed(db, user) == [(_ch(5), 5)]


def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
    me, other = make_user("me@example.com"), make_user("other@example.com")
    _channel(db, 1)
    for user in (me, other):
        _watched(db, user, 1, [LONG] * 5)
    db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
    db.commit()
    service.dismiss_watched_channel(db, other, _ch(1))
    db.commit()
    assert _listed(db, me) == [(_ch(1), 5)]
    assert _listed(db, other) == []


def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
    user = make_user("pending@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 6, asked=False)
    _watched(db, user, 2, [LONG] * 4, asked=False)  # four watched: it can never make the list
    assert queries.watched_channels_pending(db, user) == 1
    assert _listed(db, user) == []
    ids = [_vid(1, i) for i in range(6)]
    service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
    db.commit()
    assert queries.watched_channels_pending(db, user) == 0
    assert _listed(db, user) == [(_ch(1), 6)]


def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
    user = make_user("again@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 5)
    _watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
    assert _listed(db, user) == [(_ch(1), 5)]
    assert queries.takeout_lengths_to_ask(db, user) == []


# --- what the import asks YouTube for ----------------------------------------------------------


class FakeYouTube:
    """Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
    `channels.list` with a title per channel, and records what it was asked."""

    def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
        self.lengths = lengths or {}
        self.fail = fail
        self.fail_channels = False
        self.videos_asked: list[list[str]] = []
        self.channels_asked: list[list[str]] = []
        self.actions: list[tuple[int | None, str]] = []

    def __call__(self, db, user):
        return self

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

    def get_videos(self, ids):
        self.videos_asked.append(list(ids))
        self.actions.append((quota._actor_id.get(), quota._action.get()))
        if self.fail:
            raise YouTubeError("refused", reason="quotaExceeded", path="videos")
        return [
            {"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
            for v in ids
            if v in self.lengths
        ]

    def get_channels(self, ids):
        self.channels_asked.append(list(ids))
        self.actions.append((quota._actor_id.get(), quota._action.get()))
        if self.fail_channels:
            raise YouTubeError("refused", reason="backendError", path="channels")
        return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]


@pytest.fixture
def youtube(monkeypatch, db, make_user):
    fetcher = make_user("fetcher@example.com")
    fake = FakeYouTube()
    monkeypatch.setattr(takeout, "YouTubeClient", fake)
    monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
    monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
    return fake


def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
    user = make_user("import@example.com")
    _watched(db, user, 1, [None] * 7, asked=False)  # the most watched channel
    _watched(db, user, 2, [None] * 5, asked=False)
    _watched(db, user, 3, [None] * 4, asked=False)  # too few to ever make it: not asked about
    youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}

    result = takeout.run_import(user.id)

    asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
    assert youtube.videos_asked == [asked]  # the top channel first
    assert result["lengths_asked"] == 12
    # Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
    assert youtube.channels_asked == [[_ch(1)]]
    assert result["channels_described"] == 1
    assert {
        (user.id, quota.QuotaAction.VIDEOS_LOOKUP),
        (user.id, quota.QuotaAction.CHANNELS_DISCOVER),
    } == set(youtube.actions)
    db.expire_all()
    assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
    assert queries.watched_channels_pending(db, user) == 0
    # Asked once: a second run asks nothing again, the gone video included.
    youtube.videos_asked.clear()
    youtube.channels_asked.clear()
    assert "lengths_asked" not in takeout.run_import(user.id)
    assert youtube.videos_asked == [] and youtube.channels_asked == []


def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
    db, make_user, youtube, monkeypatch
):
    user = make_user("share@example.com")
    _watched(db, user, 1, [None] * 8, asked=False)
    _watched(db, user, 2, [None] * 6, asked=False)
    monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
    takeout.run_import(user.id)
    assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
    db.expire_all()
    assert queries.watched_channels_pending(db, user) == 1  # channel 2 still has some to ask
    youtube.videos_asked.clear()
    takeout.run_import(user.id)
    assert youtube.videos_asked == []  # the share is spent for today


@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
    db, make_user, youtube, monkeypatch, why
):
    user = make_user(f"{why}@example.com")
    _watched(db, user, 1, [None] * 5, asked=False)
    if why == "no_fetcher":
        monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
    else:
        monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
    assert "lengths_asked" not in takeout.run_import(user.id)
    assert youtube.videos_asked == []
    assert queries.watched_channels_pending(db, user) == 1


def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
    db, make_user, youtube
):
    user = make_user("fail@example.com")
    _watched(db, user, 1, [None] * 5, asked=False)
    youtube.fail = True
    result = takeout.run_import(user.id)
    assert "lengths_asked" not in result
    assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
    db.expire_all()
    rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
    assert all(r.looked_up_at is None for r in rows)  # left to ask again


def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
    user = make_user("later@example.com")
    _watched(db, user, 1, [None] * 5, asked=False)
    youtube.lengths = {_vid(1, i): LONG for i in range(5)}
    assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
    db.expire_all()
    assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]


@pytest.mark.parametrize("why", ["follows", "blocked", "dismissed", "own"])
def test_a_channel_off_the_list_is_not_asked_about_nor_looked_at_again(
    db, make_user, youtube, monkeypatch, why
):
    user = make_user(f"off-{why}@example.com", yt_channel_id=_ch(1) if why == "own" else None)
    _channel(db, 1)
    _watched(db, user, 1, [None] * 5, asked=False)
    if why == "follows":
        db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
    elif why == "blocked":
        db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
    elif why == "dismissed":
        service.dismiss_watched_channel(db, user, _ch(1))
    db.commit()
    runs = []
    monkeypatch.setattr(takeout, "_channel_list", lambda session, u: runs.append(u.id) or {})
    assert takeout.apply_waiting(db) == {}
    # Its rows are never asked about, so they would stay "to ask" for good: the Shorts job must not
    # take the user up again for them on every run.
    assert runs == []
    assert youtube.videos_asked == []


def test_naming_the_channels_is_tried_again_and_meanwhile_the_list_says_it_is_not_complete(
    db, make_user, youtube
):
    user = make_user("unnamed@example.com")
    _watched(db, user, 1, [None] * 5, asked=False)
    youtube.lengths = {_vid(1, i): LONG for i in range(5)}
    youtube.fail_channels = True
    takeout.run_import(user.id)
    db.expire_all()
    # Every length is known, the channel could not be named: not on the list, and not complete.
    assert queries.takeout_lengths_to_ask(db, user) == []
    assert _listed(db, user) == []
    assert queries.watched_channels_pending(db, user) == 1
    youtube.fail_channels = False
    takeout.apply_waiting(db)
    db.expire_all()
    assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 5)]
    assert queries.watched_channels_pending(db, user) == 0


def test_a_listed_channel_still_to_be_described_is_not_counted_as_one_to_come(db, make_user):
    user = make_user("stub@example.com")
    # A stub the catalog made when one of its videos came in: a title, no details yet.
    db.add(Channel(id=_ch(1), title="From a video"))
    db.commit()
    _watched(db, user, 1, [LONG] * 5)
    assert queries.watched_channels_to_describe(db, user) == [_ch(1)]
    assert _listed(db, user) == [(_ch(1), 5)]
    # It is on the list already: "the list may grow" would promise one more that cannot come.
    assert queries.watched_channels_pending(db, user) == 0


def test_the_lengths_leave_the_watch_later_lookups_their_own_share(
    db, make_user, youtube, monkeypatch
):
    user = make_user("shares@example.com")
    monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
    monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
    _watched(db, user, 1, [None] * 15, asked=False)
    takeout.run_import(user.id)  # the history: its lengths spend all they may today
    youtube.videos_asked.clear()
    service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"later000001": (T0, None)})
    db.commit()
    takeout.run_import(user.id)  # then the export's Watch later file
    assert ["later000001"] in youtube.videos_asked


# --- the channel's watched videos, ready to play ------------------------------------------------


def _fake_ensure(calls):
    def ensure(session, yt, ids, *, explore=False):
        calls.append((list(ids), explore))
        for vid in ids:
            if vid.endswith("4"):
                continue  # gone from YouTube since its length was asked
            session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
        session.commit()

    return ensure


def test_a_channels_watched_videos_come_ready_to_play_and_explored(
    db, make_user, as_user, monkeypatch
):
    user = make_user("play@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
    db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0))  # already in the catalog
    db.commit()
    calls = []
    monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
    monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)

    resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")

    assert resp.status_code == 200, resp.text
    body = resp.json()
    # Only the counted ones, newest first; the short one and the gone ones are left out.
    assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
    assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
    assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
    assert body["gone"] == 2  # one YouTube had lost before, one since
    assert body["ok"] is True
    db.expire_all()
    assert db.execute(
        select(ExploredChannel).where(
            ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
        )
    ).scalar_one()
    assert (
        db.get(Channel, _ch(1)).from_explore is True
    )  # nobody follows it: the cleanup's to reclaim


def test_a_followed_channels_watched_videos_come_in_as_the_catalogs_own(
    db, make_user, as_user, monkeypatch
):
    user, follower = make_user("viewer@example.com"), make_user("follower@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 5)
    db.add(Subscription(user_id=follower.id, channel_id=_ch(1)))
    db.commit()
    calls = []
    monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
    monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
    assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
    # Someone follows it, so it is no exploration's to reclaim: a video flagged as one would never
    # be cleared again (subscribing clears only an explored channel's) and never marked watched.
    assert calls == [([_vid(1, i) for i in range(5)], False)]
    db.expire_all()
    assert db.get(Channel, _ch(1)).from_explore is False


def test_a_follow_landing_while_the_videos_come_in_leaves_none_of_them_flagged(
    db, make_user, as_user, monkeypatch
):
    user, follower = make_user("racer@example.com"), make_user("late@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 5)
    fake = _fake_ensure(calls := [])

    def ensure(session, yt, ids, *, explore=False):
        # Someone subscribes, in a request of their own, while YouTube answers this one.
        with SessionLocal() as other:
            ch = other.get(Channel, _ch(1))
            catalog_service.record_subscription(other, follower, ch, "sub-late")
            other.commit()
        fake(session, yt, ids, explore=explore)

    monkeypatch.setattr(feed_routes, "ensure_videos", ensure)
    monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
    assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
    assert calls[0][1] is True  # asked as the exploration's
    db.expire_all()
    flagged = db.scalars(select(Video.id).where(Video.via_explore.is_(True))).all()
    assert (db.get(Channel, _ch(1)).from_explore, flagged) == (False, [])


def test_the_cleanup_keeps_a_channel_a_watch_history_lists_and_reclaims_its_videos(db, make_user):
    user = make_user("kept@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 5)
    catalog_service.explore(db, user, db.get(Channel, _ch(1)))
    db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0, via_explore=True))
    db.add(Channel(id=_ch(2), from_explore=True))  # explored, in nobody's history
    db.commit()
    db.execute(update(ExploredChannel).values(created_at=T0 - timedelta(days=365)))
    db.commit()
    out = sync_explore.purge_unkept_explored(db)
    # The unplayed video goes (opening the list's videos brings it back); the channel stays listed.
    assert (out["videos_deleted"], out["channels_deleted"]) == (1, 1)
    db.expire_all()
    assert db.get(Channel, _ch(2)) is None
    assert _listed(db, user) == [(_ch(1), 5)]


def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
    user = make_user("refused@example.com")
    _channel(db, 1)
    db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
    db.commit()
    client = as_user(user)
    assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
    assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404


def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
    db.add(Channel(id=_ch(1), title="C"))
    db.commit()

    class YT:
        def get_videos(self, ids):
            return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]

    sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
    sync_videos.ensure_videos(db, YT(), ["organicvid0"])
    db.expire_all()
    assert db.get(Video, "exploredvid").via_explore is True
    assert db.get(Video, "organicvid0").via_explore is False


# --- the routes -----------------------------------------------------------------------------------


def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
    db, make_user, as_user
):
    user = make_user("route@example.com")
    client = as_user(user)
    assert client.get("/api/takeout/channels").json() == {
        "imported": False,
        "pending": 0,
        "channels": [],
    }
    _channel(db, 1, "Some channel")
    _watched(db, user, 1, [LONG] * 5)
    _watched(db, user, 2, [None] * 5, asked=False)
    body = client.get("/api/takeout/channels").json()
    assert body["imported"] is True and body["pending"] == 1
    assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
        (_ch(1), "Some channel", 5)
    ]
    assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")


def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
    user = make_user("dismiss@example.com")
    _channel(db, 1)
    _watched(db, user, 1, [LONG] * 5)
    client = as_user(user)
    url = f"/api/takeout/channels/{_ch(1)}/dismissal"
    assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
    assert client.put(url).status_code == 200  # again: no error
    assert client.get("/api/takeout/channels").json()["channels"] == []
    assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
    assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
    db.expire_all()
    assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []


def test_only_a_channel_of_the_users_own_history_can_be_put_out_of_the_list(db, make_user, as_user):
    user, other = make_user("mine@example.com"), make_user("theirs@example.com")
    _channel(db, 1)
    _watched(db, other, 1, [LONG] * 5)
    client = as_user(user)
    assert client.put(f"/api/takeout/channels/{_ch(1)}/dismissal").status_code == 404
    assert client.put(f"/api/takeout/channels/{_ch(9)}/dismissal").status_code == 404
    db.expire_all()
    assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []


def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
    db, make_user, as_user
):
    user = make_user("long@example.com")
    assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422