fix(sync): only a refusal about the channel spends its fallback window
A failed connection or token step answered nothing about the channel, yet stamped it: one proxy blip during an RSS blackout delayed every due channel by a whole window. YouTubeError now names the endpoint that answered; a failure no endpoint gave, or a spent quota, ends the round and stamps nothing.
This commit is contained in:
@@ -197,9 +197,10 @@ def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]:
|
|||||||
uploads playlist is 1 unit, so a whole day of total outage costs 12 units per channel at the
|
uploads playlist is 1 unit, so a whole day of total outage costs 12 units per channel at the
|
||||||
default 2 hours; it yields to the backfill reserve like every other background read. The clock is
|
default 2 hours; it yields to the backfill reserve like every other background read. The clock is
|
||||||
`last_rss_at` or `feed_fallback_at`, whichever is later, so a channel is read at most once per
|
`last_rss_at` or `feed_fallback_at`, whichever is later, so a channel is read at most once per
|
||||||
window whether its feed came back in between or not. A FAILED read stamps the clock too: a
|
window whether its feed came back in between or not. A read the API REFUSED stamps the clock
|
||||||
terminated channel's uploads playlist answers 404 for good, and without the stamp it would be
|
too: a terminated channel's uploads playlist answers 404 for good, and without the stamp it
|
||||||
charged again on every 20-minute poll instead of once per window."""
|
would be charged again on every 20-minute poll instead of once per window. A failure that is not
|
||||||
|
about the channel (no connection, no token, the quota spent) stamps nothing and ends the round."""
|
||||||
hours = sysconfig.get_int(db, "rss_fallback_hours")
|
hours = sysconfig.get_int(db, "rss_fallback_hours")
|
||||||
if hours <= 0 or not channel_ids:
|
if hours <= 0 or not channel_ids:
|
||||||
return {"read": 0, "new": 0, "failed": 0}
|
return {"read": 0, "new": 0, "failed": 0}
|
||||||
@@ -235,10 +236,11 @@ def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]:
|
|||||||
# already logged the call's own failure (`youtube.api.error`).
|
# already logged the call's own failure (`youtube.api.error`).
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
failed += 1
|
failed += 1
|
||||||
_stamp_fallback_attempt(db, channel.id)
|
# Not an answer about THIS channel — no connection, no token, no quota: every
|
||||||
# Out of quota upstream: every further channel would be charged the same refusal.
|
# further channel would fail the same way, and none of them has used its window.
|
||||||
if is_quota_refusal(exc):
|
if exc.path != "playlistItems" or is_quota_refusal(exc):
|
||||||
break
|
break
|
||||||
|
_stamp_fallback_attempt(db, channel.id)
|
||||||
continue
|
continue
|
||||||
rows = [
|
rows = [
|
||||||
s
|
s
|
||||||
|
|||||||
@@ -26,11 +26,15 @@ class YouTubeError(Exception):
|
|||||||
|
|
||||||
`reason` is the API's own error reason code (`quotaExceeded`, `subscriptionDuplicate`,
|
`reason` is the API's own error reason code (`quotaExceeded`, `subscriptionDuplicate`,
|
||||||
`videoNotFound`, …) when the body carried one. Branch on THAT, never on the message: the
|
`videoNotFound`, …) when the body carried one. Branch on THAT, never on the message: the
|
||||||
message is prose, it changes under us, and it is deliberately the poorer of the two here."""
|
message is prose, it changes under us, and it is deliberately the poorer of the two here.
|
||||||
|
|
||||||
def __init__(self, message: str, *, reason: str = ""):
|
`path` is the API endpoint that ANSWERED with the failure (`playlistItems`); empty when no
|
||||||
|
endpoint did — the request never got through (`transport`) or the token step refused first."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, *, reason: str = "", path: str = ""):
|
||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
self.reason = reason
|
self.reason = reason
|
||||||
|
self.path = path
|
||||||
|
|
||||||
|
|
||||||
def _error_reason(resp: httpx.Response) -> str:
|
def _error_reason(resp: httpx.Response) -> str:
|
||||||
@@ -87,7 +91,9 @@ def _upstream_failure(
|
|||||||
# log-pii-ok: only under this user's own Teljes consent, which is what it is for
|
# log-pii-ok: only under this user's own Teljes consent, which is what it is for
|
||||||
extra=logs.event("youtube.api.body", body=resp.text[:500]),
|
extra=logs.event("youtube.api.body", body=resp.text[:500]),
|
||||||
)
|
)
|
||||||
return YouTubeError(f"{method} {path} -> {resp.status_code}: {reason}", reason=reason)
|
return YouTubeError(
|
||||||
|
f"{method} {path} -> {resp.status_code}: {reason}", reason=reason, path=path
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# The `mine` reads' answer for a Google account that has no YouTube channel.
|
# The `mine` reads' answer for a Google account that has no YouTube channel.
|
||||||
|
|||||||
@@ -30,7 +30,10 @@ class _FakeYt:
|
|||||||
def get_playlist_items(self, playlist_id, page_token=None):
|
def get_playlist_items(self, playlist_id, page_token=None):
|
||||||
self.calls.append(playlist_id)
|
self.calls.append(playlist_id)
|
||||||
if playlist_id in self.failing:
|
if playlist_id in self.failing:
|
||||||
raise YouTubeError("refused", reason=self.failing[playlist_id])
|
reason = self.failing[playlist_id]
|
||||||
|
# A request that never got through has no endpoint that answered.
|
||||||
|
path = "" if reason == "transport" else "playlistItems"
|
||||||
|
raise YouTubeError("refused", reason=reason, path=path)
|
||||||
cid = "UC" + playlist_id[2:]
|
cid = "UC" + playlist_id[2:]
|
||||||
return {
|
return {
|
||||||
"items": [
|
"items": [
|
||||||
@@ -134,10 +137,14 @@ def test_a_read_that_fails_waits_the_window_out_too(db, calls):
|
|||||||
assert calls == ["UUgone"]
|
assert calls == ["UUgone"]
|
||||||
|
|
||||||
|
|
||||||
def test_out_of_quota_upstream_stops_the_round(db, calls):
|
@pytest.mark.parametrize("reason", ["quotaExceeded", "transport"])
|
||||||
|
def test_a_failure_not_about_the_channel_ends_the_round_and_costs_no_window(db, calls, reason):
|
||||||
|
# Out of quota, or no connection: every further channel would fail the same way, and a channel
|
||||||
|
# that was never answered about keeps its turn for the next poll.
|
||||||
_channel(db, "UCa")
|
_channel(db, "UCa")
|
||||||
_channel(db, "UCb")
|
_channel(db, "UCb")
|
||||||
_FakeYt.failing["UUa"] = "quotaExceeded"
|
_FakeYt.failing["UUa"] = reason
|
||||||
|
|
||||||
assert runner.run_feed_fallback(db, ["UCa", "UCb"])["failed"] == 1
|
assert runner.run_feed_fallback(db, ["UCa", "UCb"])["failed"] == 1
|
||||||
assert calls == ["UUa"]
|
assert calls == ["UUa"]
|
||||||
|
assert db.get(Channel, "UCa").feed_fallback_at is None
|
||||||
|
|||||||
Reference in New Issue
Block a user