siftlode/backend/tests/test_feed_fallback.py

"""The RSS poll's API fallback (`sync/runner.run_feed_fallback`), DB-lane: which channels are read
through the Data API when their feed stays dark, and how often."""

from datetime import datetime, timedelta, timezone

import pytest

from app import sysconfig
from app.catalog.models import Channel, Video
from app.sync import runner
from app.youtube.client import YouTubeError

NOW = datetime.now(timezone.utc)
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)


class _FakeYt:
    # playlist id -> the reason its read fails with (the tests put entries here)
    failing: dict[str, str] = {}

    def __init__(self, calls):
        self.calls = calls

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

    def get_playlist_items(self, playlist_id, page_token=None):
        self.calls.append(playlist_id)
        if playlist_id in self.failing:
            reason = self.failing[playlist_id]
            # A request that never got through has no endpoint that answered.
            path = "" if reason == "transport" else "playlistItems"
            raise YouTubeError("refused", reason=reason, path=path)
        cid = "UC" + playlist_id[2:]
        return {
            "items": [
                {
                    "contentDetails": {"videoId": f"{cid}-v1", "videoPublishedAt": NOW.isoformat()},
                    "snippet": {"title": "new upload"},
                }
            ]
        }


@pytest.fixture
def calls(monkeypatch):
    made: list[str] = []
    monkeypatch.setattr(runner, "get_service_user", lambda db: object())
    monkeypatch.setattr(runner, "YouTubeClient", lambda db, user: _FakeYt(made))
    monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
    monkeypatch.setattr(_FakeYt, "failing", {})
    return made


def _channel(db, cid, *, rss=None, fallback=None, uploads=True):
    ch = Channel(
        id=cid,
        title=cid,
        published_at=_PUB,
        uploads_playlist_id=("UU" + cid[2:]) if uploads else None,
        last_rss_at=rss,
        feed_fallback_at=fallback,
    )
    db.add(ch)
    db.commit()
    return ch


def test_a_feed_dark_past_the_window_is_read_through_the_api(db, calls):
    _channel(db, "UCdark", rss=NOW - timedelta(hours=3))

    out = runner.run_feed_fallback(db, ["UCdark"])

    assert out == {"read": 1, "new": 1, "failed": 0}
    assert calls == ["UUdark"]
    assert db.get(Video, "UCdark-v1") is not None
    assert db.get(Channel, "UCdark").feed_fallback_at is not None


@pytest.mark.parametrize(
    ("rss", "fallback"),
    [
        (NOW - timedelta(minutes=30), None),  # the feed answered recently: a blip, not an outage
        (NOW - timedelta(hours=5), NOW - timedelta(minutes=30)),  # already read this window
    ],
)
def test_a_channel_read_within_the_window_is_left_alone(db, calls, rss, fallback):
    _channel(db, "UCrecent", rss=rss, fallback=fallback)

    assert runner.run_feed_fallback(db, ["UCrecent"])["read"] == 0
    assert calls == []


def test_the_window_is_the_admins_and_zero_turns_it_off(db, calls):
    _channel(db, "UCdark", rss=NOW - timedelta(hours=3))

    sysconfig.set_value(db, "rss_fallback_hours", 4)
    db.commit()
    assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0

    sysconfig.set_value(db, "rss_fallback_hours", 0)
    db.commit()
    _channel(db, "UCnever")
    assert runner.run_feed_fallback(db, ["UCdark", "UCnever"])["read"] == 0
    assert calls == []


def test_only_the_named_channels_and_only_those_with_an_uploads_playlist(db, calls):
    _channel(db, "UCdark")
    _channel(db, "UCother")
    _channel(db, "UCnolist", uploads=False)

    assert runner.run_feed_fallback(db, ["UCdark", "UCnolist"])["read"] == 1
    assert calls == ["UUdark"]


def test_it_yields_to_the_backfill_reserve(db, calls, monkeypatch):
    _channel(db, "UCdark")
    monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: True)

    assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0
    assert calls == []


def test_a_read_that_fails_waits_the_window_out_too(db, calls):
    # A terminated channel: its feed 404s and so does its uploads playlist, for good. Without a
    # stamp it was charged on every 20-minute poll instead of once per window.
    _channel(db, "UCgone", rss=NOW - timedelta(hours=3))
    _FakeYt.failing["UUgone"] = "playlistNotFound"

    assert runner.run_feed_fallback(db, ["UCgone"]) == {"read": 0, "new": 0, "failed": 1}
    assert db.get(Channel, "UCgone").feed_fallback_at is not None
    assert runner.run_feed_fallback(db, ["UCgone"])["failed"] == 0
    assert calls == ["UUgone"]


@pytest.mark.parametrize("reason", ["quotaExceeded", "transport"])
def test_a_failure_not_about_the_channel_ends_the_round_and_costs_no_window(db, calls, reason):
    # Out of quota, or no connection: every further channel would fail the same way, and a channel
    # that was never answered about keeps its turn for the next poll.
    _channel(db, "UCa")
    _channel(db, "UCb")
    _FakeYt.failing["UUa"] = reason

    assert runner.run_feed_fallback(db, ["UCa", "UCb"])["failed"] == 1
    assert calls == ["UUa"]
    assert db.get(Channel, "UCa").feed_fallback_at is None