peter
1fa6a0bea8
A channel whose uploads playlist also failed was never stamped, so a terminated channel was charged again on every 20-minute poll instead of once per rss_fallback_hours. An upstream quotaExceeded now stops the round instead of charging every remaining channel the same refusal.
144 lines
4.6 KiB
Python
144 lines
4.6 KiB
Python
"""The RSS poll's API fallback (`sync/runner.run_feed_fallback`), DB-lane: which channels are read
|
|
through the Data API when their feed stays dark, and how often."""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from app import sysconfig
|
|
from app.models import Channel, Video
|
|
from app.sync import runner
|
|
from app.youtube.client import YouTubeError
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
class _FakeYt:
|
|
# playlist id -> the reason its read fails with (the tests put entries here)
|
|
failing: dict[str, str] = {}
|
|
|
|
def __init__(self, calls):
|
|
self.calls = calls
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def get_playlist_items(self, playlist_id, page_token=None):
|
|
self.calls.append(playlist_id)
|
|
if playlist_id in self.failing:
|
|
raise YouTubeError("refused", reason=self.failing[playlist_id])
|
|
cid = "UC" + playlist_id[2:]
|
|
return {
|
|
"items": [
|
|
{
|
|
"contentDetails": {"videoId": f"{cid}-v1", "videoPublishedAt": NOW.isoformat()},
|
|
"snippet": {"title": "new upload"},
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def calls(monkeypatch):
|
|
made: list[str] = []
|
|
monkeypatch.setattr(runner, "get_service_user", lambda db: object())
|
|
monkeypatch.setattr(runner, "YouTubeClient", lambda db, user: _FakeYt(made))
|
|
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
|
|
monkeypatch.setattr(_FakeYt, "failing", {})
|
|
return made
|
|
|
|
|
|
def _channel(db, cid, *, rss=None, fallback=None, uploads=True):
|
|
ch = Channel(
|
|
id=cid,
|
|
title=cid,
|
|
published_at=_PUB,
|
|
uploads_playlist_id=("UU" + cid[2:]) if uploads else None,
|
|
last_rss_at=rss,
|
|
feed_fallback_at=fallback,
|
|
)
|
|
db.add(ch)
|
|
db.commit()
|
|
return ch
|
|
|
|
|
|
def test_a_feed_dark_past_the_window_is_read_through_the_api(db, calls):
|
|
_channel(db, "UCdark", rss=NOW - timedelta(hours=3))
|
|
|
|
out = runner.run_feed_fallback(db, ["UCdark"])
|
|
|
|
assert out == {"read": 1, "new": 1, "failed": 0}
|
|
assert calls == ["UUdark"]
|
|
assert db.get(Video, "UCdark-v1") is not None
|
|
assert db.get(Channel, "UCdark").feed_fallback_at is not None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("rss", "fallback"),
|
|
[
|
|
(NOW - timedelta(minutes=30), None), # the feed answered recently: a blip, not an outage
|
|
(NOW - timedelta(hours=5), NOW - timedelta(minutes=30)), # already read this window
|
|
],
|
|
)
|
|
def test_a_channel_read_within_the_window_is_left_alone(db, calls, rss, fallback):
|
|
_channel(db, "UCrecent", rss=rss, fallback=fallback)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCrecent"])["read"] == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_the_window_is_the_admins_and_zero_turns_it_off(db, calls):
|
|
_channel(db, "UCdark", rss=NOW - timedelta(hours=3))
|
|
|
|
sysconfig.set_value(db, "rss_fallback_hours", 4)
|
|
db.commit()
|
|
assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0
|
|
|
|
sysconfig.set_value(db, "rss_fallback_hours", 0)
|
|
db.commit()
|
|
_channel(db, "UCnever")
|
|
assert runner.run_feed_fallback(db, ["UCdark", "UCnever"])["read"] == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_only_the_named_channels_and_only_those_with_an_uploads_playlist(db, calls):
|
|
_channel(db, "UCdark")
|
|
_channel(db, "UCother")
|
|
_channel(db, "UCnolist", uploads=False)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCdark", "UCnolist"])["read"] == 1
|
|
assert calls == ["UUdark"]
|
|
|
|
|
|
def test_it_yields_to_the_backfill_reserve(db, calls, monkeypatch):
|
|
_channel(db, "UCdark")
|
|
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: True)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_a_read_that_fails_waits_the_window_out_too(db, calls):
|
|
# A terminated channel: its feed 404s and so does its uploads playlist, for good. Without a
|
|
# stamp it was charged on every 20-minute poll instead of once per window.
|
|
_channel(db, "UCgone", rss=NOW - timedelta(hours=3))
|
|
_FakeYt.failing["UUgone"] = "playlistNotFound"
|
|
|
|
assert runner.run_feed_fallback(db, ["UCgone"]) == {"read": 0, "new": 0, "failed": 1}
|
|
assert db.get(Channel, "UCgone").feed_fallback_at is not None
|
|
assert runner.run_feed_fallback(db, ["UCgone"])["failed"] == 0
|
|
assert calls == ["UUgone"]
|
|
|
|
|
|
def test_out_of_quota_upstream_stops_the_round(db, calls):
|
|
_channel(db, "UCa")
|
|
_channel(db, "UCb")
|
|
_FakeYt.failing["UUa"] = "quotaExceeded"
|
|
|
|
assert runner.run_feed_fallback(db, ["UCa", "UCb"])["failed"] == 1
|
|
assert calls == ["UUa"]
|