fix(scheduler,i18n): a failure a job names reads in the reader's language
The audit log showed a failed RSS poll as the server's English sentence under a Hungarian UI. A job can now raise progress.JobFailed(reason, message, **params): the scheduler stores the reason and params as the run's data (dashboard, manual-run notice, audit row) beside the English line, and the client renders scheduler.failure.<reason>. The RSS outage names its first channel and the reason code (HTTP 404 vs unreachable). An upstream's own error text stays as raised (diagnostic).
This commit is contained in:
@@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
|
||||
sink(current, total, phase)
|
||||
|
||||
|
||||
class JobFailed(RuntimeError):
|
||||
"""A job's own failure, said so the reader's language can name it (E31): the client reads
|
||||
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
|
||||
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
|
||||
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
|
||||
being diagnostic."""
|
||||
|
||||
def __init__(self, reason: str, message: str, **params: object):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
self.params = params
|
||||
|
||||
|
||||
class bind:
|
||||
"""Context manager that binds a progress sink for the duration of a job run."""
|
||||
|
||||
|
||||
@@ -231,17 +231,22 @@ def _job(name: str, fn) -> None:
|
||||
db.rollback()
|
||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
# A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
|
||||
# client says it in the reader's language; `err` stays the English line.
|
||||
failure = (
|
||||
{"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
|
||||
)
|
||||
_record(
|
||||
name,
|
||||
status="error",
|
||||
last_finished=_now_iso(),
|
||||
last_error=err,
|
||||
result=None,
|
||||
result=failure,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "error", err)
|
||||
_audit_run(db, actor_id, name, "error", err, None)
|
||||
_notify_done(db, actor_id, name, "error", err, failure)
|
||||
_audit_run(db, actor_id, name, "error", err, failure)
|
||||
finally:
|
||||
# ALWAYS release the claim, even if the body raised before its own status update — so a
|
||||
# failure can never wedge the job as permanently "running".
|
||||
|
||||
@@ -83,6 +83,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# outage from one permanently dead feed id — which is the whole difference on an instance with
|
||||
# a single subscription, where BOTH render as 100% failed.
|
||||
first_error: str | None = None
|
||||
# The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key).
|
||||
first_failure: tuple[str, str] | None = None
|
||||
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
|
||||
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
|
||||
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
|
||||
@@ -92,18 +94,19 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# hand the same rows to the same broken write) — the feed fallback's candidates.
|
||||
unread: list[str] = []
|
||||
|
||||
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
|
||||
def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None:
|
||||
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
|
||||
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
|
||||
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
|
||||
tell nobody anything the first one didn't. Call it from inside the `except` block."""
|
||||
nonlocal failed, first_error
|
||||
nonlocal failed, first_error, first_failure
|
||||
if unexpected is not None:
|
||||
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
|
||||
log.log(level, unexpected, exc_info=True)
|
||||
failed += 1
|
||||
reasons[reason] += 1
|
||||
first_error = first_error or detail
|
||||
first_failure = first_failure or (channel_id, reason)
|
||||
if len(examples) < RSS_FAILURE_EXAMPLES:
|
||||
examples.append(detail)
|
||||
|
||||
@@ -125,13 +128,16 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# bug on our side and earns a traceback (the first of its kind per poll).
|
||||
except RssError as exc:
|
||||
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
|
||||
fail(exc.reason, f"{channel.id}: {exc}")
|
||||
fail(exc.reason, channel.id, f"{channel.id}: {exc}")
|
||||
unread.append(channel.id)
|
||||
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
|
||||
# traceback, once per reason per poll, which is the flood control this loop needs.
|
||||
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
|
||||
fail(
|
||||
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
|
||||
"error",
|
||||
channel.id,
|
||||
f"{channel.id}: {exc}",
|
||||
unexpected=f"RSS fetch failed for {channel.id}",
|
||||
)
|
||||
else:
|
||||
try:
|
||||
@@ -144,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
db.rollback()
|
||||
fail(
|
||||
"apply",
|
||||
channel.id,
|
||||
f"{channel.id} (apply): {exc}",
|
||||
unexpected=f"RSS apply failed for {channel.id}",
|
||||
)
|
||||
@@ -180,9 +187,17 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# "fix this subscription"; `Connection refused` says "the egress is down".
|
||||
# The fallback's count rides along: a red card during an RSS outage should still say
|
||||
# whether the uploads are arriving another way.
|
||||
raise RuntimeError(
|
||||
# As data too (E31): the audit log and the inbox say it in the reader's language, the first
|
||||
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
|
||||
first_channel, first_reason = first_failure or ("", "error")
|
||||
raise progress.JobFailed(
|
||||
"rss_all_failed",
|
||||
f"RSS poll failed for all {total} channels — first: {first_error}"
|
||||
f" (read through the API instead: {fallback['read']})"
|
||||
f" (read through the API instead: {fallback['read']})",
|
||||
total=total,
|
||||
channel=first_channel,
|
||||
cause=first_reason,
|
||||
read=fallback["read"],
|
||||
)
|
||||
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
|
||||
|
||||
|
||||
@@ -86,3 +86,31 @@ def test_a_job_runs_audit_row_carries_its_result_as_data(db):
|
||||
{"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}},
|
||||
{"status": "error", "summary": "boom"},
|
||||
]
|
||||
|
||||
|
||||
def test_a_failure_the_job_names_rides_along_as_data(db, make_user):
|
||||
"""E31: a failed RSS poll reached the Hungarian audit log as the server's English sentence. A
|
||||
failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the
|
||||
dashboard, the manual-run notice and the audit row — so the client says it in the reader's
|
||||
language. Any other failure keeps only its own text (the test above)."""
|
||||
from app import progress
|
||||
from app.models import AuditLog
|
||||
|
||||
def outage(_db):
|
||||
raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport")
|
||||
|
||||
admin = make_user("failure@example.com", role="admin")
|
||||
token = scheduler._manual_actor.set(admin.id)
|
||||
try:
|
||||
scheduler._job("rss_poll", outage)
|
||||
finally:
|
||||
scheduler._manual_actor.reset(token)
|
||||
|
||||
named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"}
|
||||
rss = _snapshot_job("rss_poll")
|
||||
assert rss["last_error"] == "RSS poll failed"
|
||||
assert rss["result"] == named
|
||||
notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one()
|
||||
assert notice.data["result"] == named
|
||||
row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first()
|
||||
assert row is not None and row.after["result"] == named
|
||||
|
||||
@@ -6,6 +6,7 @@ import logging
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import progress
|
||||
from app.sync import runner
|
||||
from app.sync.playlists import FAILED, _Breaker
|
||||
from app.sync.subscriptions import should_prune
|
||||
@@ -224,8 +225,15 @@ class TestRssPollResult:
|
||||
def dead(cid):
|
||||
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404")
|
||||
|
||||
with pytest.raises(RuntimeError, match="HTTP 404"):
|
||||
with pytest.raises(RuntimeError, match="HTTP 404") as err:
|
||||
self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",))
|
||||
# E31: and as data, for the reader's language — whose feed, and the reason code that tells a
|
||||
# dead feed (`http_404`) from a dead egress (`transport`).
|
||||
assert isinstance(err.value, progress.JobFailed)
|
||||
assert (err.value.reason, err.value.params) == (
|
||||
"rss_all_failed",
|
||||
{"total": 1, "channel": "UC_dead", "cause": "http_404", "read": 0},
|
||||
)
|
||||
|
||||
def test_a_read_only_database_is_a_total_outage_too(self, monkeypatch):
|
||||
# Every feed reads fine, nothing can be stored: just as un-polled, just as red.
|
||||
|
||||
Reference in New Issue
Block a user