fix(scheduler,i18n): a failure a job names reads in the reader's language

The audit log showed a failed RSS poll as the server's English sentence
under a Hungarian UI. A job can now raise progress.JobFailed(reason,
message, **params): the scheduler stores the reason and params as the
run's data (dashboard, manual-run notice, audit row) beside the English
line, and the client renders scheduler.failure.<reason>. The RSS outage
names its first channel and the reason code (HTTP 404 vs unreachable).
An upstream's own error text stays as raised (diagnostic).
This commit is contained in:
2026-10-01 22:51:37 +02:00
parent ec22eefd31
commit ab146211fa
10 changed files with 160 additions and 17 deletions
+13
View File
@@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
sink(current, total, phase)
class JobFailed(RuntimeError):
"""A job's own failure, said so the reader's language can name it (E31): the client reads
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
being diagnostic."""
def __init__(self, reason: str, message: str, **params: object):
super().__init__(message)
self.reason = reason
self.params = params
class bind:
"""Context manager that binds a progress sink for the duration of a job run."""
+8 -3
View File
@@ -231,17 +231,22 @@ def _job(name: str, fn) -> None:
db.rollback()
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
err = str(exc) or exc.__class__.__name__
# A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
# client says it in the reader's language; `err` stays the English line.
failure = (
{"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
)
_record(
name,
status="error",
last_finished=_now_iso(),
last_error=err,
result=None,
result=failure,
progress=None,
)
if actor_id is not None:
_notify_done(db, actor_id, name, "error", err)
_audit_run(db, actor_id, name, "error", err, None)
_notify_done(db, actor_id, name, "error", err, failure)
_audit_run(db, actor_id, name, "error", err, failure)
finally:
# ALWAYS release the claim, even if the body raised before its own status update — so a
# failure can never wedge the job as permanently "running".
+21 -6
View File
@@ -83,6 +83,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# outage from one permanently dead feed id — which is the whole difference on an instance with
# a single subscription, where BOTH render as 100% failed.
first_error: str | None = None
# The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key).
first_failure: tuple[str, str] | None = None
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
@@ -92,18 +94,19 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# hand the same rows to the same broken write) — the feed fallback's candidates.
unread: list[str] = []
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None:
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
tell nobody anything the first one didn't. Call it from inside the `except` block."""
nonlocal failed, first_error
nonlocal failed, first_error, first_failure
if unexpected is not None:
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
log.log(level, unexpected, exc_info=True)
failed += 1
reasons[reason] += 1
first_error = first_error or detail
first_failure = first_failure or (channel_id, reason)
if len(examples) < RSS_FAILURE_EXAMPLES:
examples.append(detail)
@@ -125,13 +128,16 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# bug on our side and earns a traceback (the first of its kind per poll).
except RssError as exc:
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
fail(exc.reason, f"{channel.id}: {exc}")
fail(exc.reason, channel.id, f"{channel.id}: {exc}")
unread.append(channel.id)
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
# traceback, once per reason per poll, which is the flood control this loop needs.
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
fail(
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
"error",
channel.id,
f"{channel.id}: {exc}",
unexpected=f"RSS fetch failed for {channel.id}",
)
else:
try:
@@ -144,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
db.rollback()
fail(
"apply",
channel.id,
f"{channel.id} (apply): {exc}",
unexpected=f"RSS apply failed for {channel.id}",
)
@@ -180,9 +187,17 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# "fix this subscription"; `Connection refused` says "the egress is down".
# The fallback's count rides along: a red card during an RSS outage should still say
# whether the uploads are arriving another way.
raise RuntimeError(
# As data too (E31): the audit log and the inbox say it in the reader's language, the first
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
first_channel, first_reason = first_failure or ("", "error")
raise progress.JobFailed(
"rss_all_failed",
f"RSS poll failed for all {total} channels — first: {first_error}"
f" (read through the API instead: {fallback['read']})"
f" (read through the API instead: {fallback['read']})",
total=total,
channel=first_channel,
cause=first_reason,
read=fallback["read"],
)
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
+28
View File
@@ -86,3 +86,31 @@ def test_a_job_runs_audit_row_carries_its_result_as_data(db):
{"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}},
{"status": "error", "summary": "boom"},
]
def test_a_failure_the_job_names_rides_along_as_data(db, make_user):
"""E31: a failed RSS poll reached the Hungarian audit log as the server's English sentence. A
failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the
dashboard, the manual-run notice and the audit row — so the client says it in the reader's
language. Any other failure keeps only its own text (the test above)."""
from app import progress
from app.models import AuditLog
def outage(_db):
raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport")
admin = make_user("failure@example.com", role="admin")
token = scheduler._manual_actor.set(admin.id)
try:
scheduler._job("rss_poll", outage)
finally:
scheduler._manual_actor.reset(token)
named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"}
rss = _snapshot_job("rss_poll")
assert rss["last_error"] == "RSS poll failed"
assert rss["result"] == named
notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one()
assert notice.data["result"] == named
row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first()
assert row is not None and row.after["result"] == named
+9 -1
View File
@@ -6,6 +6,7 @@ import logging
import httpx
import pytest
from app import progress
from app.sync import runner
from app.sync.playlists import FAILED, _Breaker
from app.sync.subscriptions import should_prune
@@ -224,8 +225,15 @@ class TestRssPollResult:
def dead(cid):
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404")
with pytest.raises(RuntimeError, match="HTTP 404"):
with pytest.raises(RuntimeError, match="HTTP 404") as err:
self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",))
# E31: and as data, for the reader's language — whose feed, and the reason code that tells a
# dead feed (`http_404`) from a dead egress (`transport`).
assert isinstance(err.value, progress.JobFailed)
assert (err.value.reason, err.value.params) == (
"rss_all_failed",
{"total": 1, "channel": "UC_dead", "cause": "http_404", "read": 0},
)
def test_a_read_only_database_is_a_total_outage_too(self, monkeypatch):
# Every feed reads fine, nothing can be stored: just as un-polled, just as red.