peter
29094f5f51
YouTube's RSS goes dark per channel for hours (prod 2026-09-18/19: 393 of 396 feeds answered 404 for four hours, sticky per channel, so a retry does not help), and nothing else read a followed channel's new uploads. A channel whose feed has not answered for `rss_fallback_hours` (admin knob, default 2, 0 = off) now gets one page of its uploads playlist through the Data API (1 unit), at most once per window, yielding to the backfill reserve. Spend is attributed to its own quota action.
116 lines
3.4 KiB
Python
116 lines
3.4 KiB
Python
"""The RSS poll's API fallback (`sync/runner.run_feed_fallback`), DB-lane: which channels are read
|
|
through the Data API when their feed stays dark, and how often."""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
from app import sysconfig
|
|
from app.models import Channel, Video
|
|
from app.sync import runner
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
class _FakeYt:
|
|
def __init__(self, calls):
|
|
self.calls = calls
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def get_playlist_items(self, playlist_id, page_token=None):
|
|
self.calls.append(playlist_id)
|
|
cid = "UC" + playlist_id[2:]
|
|
return {
|
|
"items": [
|
|
{
|
|
"contentDetails": {"videoId": f"{cid}-v1", "videoPublishedAt": NOW.isoformat()},
|
|
"snippet": {"title": "new upload"},
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def calls(monkeypatch):
|
|
made: list[str] = []
|
|
monkeypatch.setattr(runner, "get_service_user", lambda db: object())
|
|
monkeypatch.setattr(runner, "YouTubeClient", lambda db, user: _FakeYt(made))
|
|
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
|
|
return made
|
|
|
|
|
|
def _channel(db, cid, *, rss=None, fallback=None, uploads=True):
|
|
ch = Channel(
|
|
id=cid,
|
|
title=cid,
|
|
published_at=_PUB,
|
|
uploads_playlist_id=("UU" + cid[2:]) if uploads else None,
|
|
last_rss_at=rss,
|
|
feed_fallback_at=fallback,
|
|
)
|
|
db.add(ch)
|
|
db.commit()
|
|
return ch
|
|
|
|
|
|
def test_a_feed_dark_past_the_window_is_read_through_the_api(db, calls):
|
|
_channel(db, "UCdark", rss=NOW - timedelta(hours=3))
|
|
|
|
out = runner.run_feed_fallback(db, ["UCdark"])
|
|
|
|
assert out == {"read": 1, "new": 1, "failed": 0}
|
|
assert calls == ["UUdark"]
|
|
assert db.get(Video, "UCdark-v1") is not None
|
|
assert db.get(Channel, "UCdark").feed_fallback_at is not None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("rss", "fallback"),
|
|
[
|
|
(NOW - timedelta(minutes=30), None), # the feed answered recently: a blip, not an outage
|
|
(NOW - timedelta(hours=5), NOW - timedelta(minutes=30)), # already read this window
|
|
],
|
|
)
|
|
def test_a_channel_read_within_the_window_is_left_alone(db, calls, rss, fallback):
|
|
_channel(db, "UCrecent", rss=rss, fallback=fallback)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCrecent"])["read"] == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_the_window_is_the_admins_and_zero_turns_it_off(db, calls):
|
|
_channel(db, "UCdark", rss=NOW - timedelta(hours=3))
|
|
|
|
sysconfig.set_value(db, "rss_fallback_hours", 4)
|
|
db.commit()
|
|
assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0
|
|
|
|
sysconfig.set_value(db, "rss_fallback_hours", 0)
|
|
db.commit()
|
|
_channel(db, "UCnever")
|
|
assert runner.run_feed_fallback(db, ["UCdark", "UCnever"])["read"] == 0
|
|
assert calls == []
|
|
|
|
|
|
def test_only_the_named_channels_and_only_those_with_an_uploads_playlist(db, calls):
|
|
_channel(db, "UCdark")
|
|
_channel(db, "UCother")
|
|
_channel(db, "UCnolist", uploads=False)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCdark", "UCnolist"])["read"] == 1
|
|
assert calls == ["UUdark"]
|
|
|
|
|
|
def test_it_yields_to_the_backfill_reserve(db, calls, monkeypatch):
|
|
_channel(db, "UCdark")
|
|
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: True)
|
|
|
|
assert runner.run_feed_fallback(db, ["UCdark"])["read"] == 0
|
|
assert calls == []
|