fix(scheduler,i18n): a failure a job names reads in the reader's language

The audit log showed a failed RSS poll as the server's English sentence
under a Hungarian UI. A job can now raise progress.JobFailed(reason,
message, **params): the scheduler stores the reason and params as the
run's data (dashboard, manual-run notice, audit row) beside the English
line, and the client renders scheduler.failure.<reason>. The RSS outage
names its first channel and the reason code (HTTP 404 vs unreachable).
An upstream's own error text stays as raised (diagnostic).
This commit is contained in:
2026-10-01 22:51:37 +02:00
parent ec22eefd31
commit ab146211fa
10 changed files with 160 additions and 17 deletions
+13
View File
@@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
sink(current, total, phase)
class JobFailed(RuntimeError):
"""A job's own failure, said so the reader's language can name it (E31): the client reads
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
being diagnostic."""
def __init__(self, reason: str, message: str, **params: object):
super().__init__(message)
self.reason = reason
self.params = params
class bind:
"""Context manager that binds a progress sink for the duration of a job run."""
+8 -3
View File
@@ -231,17 +231,22 @@ def _job(name: str, fn) -> None:
db.rollback()
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
err = str(exc) or exc.__class__.__name__
# A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
# client says it in the reader's language; `err` stays the English line.
failure = (
{"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
)
_record(
name,
status="error",
last_finished=_now_iso(),
last_error=err,
result=None,
result=failure,
progress=None,
)
if actor_id is not None:
_notify_done(db, actor_id, name, "error", err)
_audit_run(db, actor_id, name, "error", err, None)
_notify_done(db, actor_id, name, "error", err, failure)
_audit_run(db, actor_id, name, "error", err, failure)
finally:
# ALWAYS release the claim, even if the body raised before its own status update — so a
# failure can never wedge the job as permanently "running".
+21 -6
View File
@@ -83,6 +83,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# outage from one permanently dead feed id — which is the whole difference on an instance with
# a single subscription, where BOTH render as 100% failed.
first_error: str | None = None
# The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key).
first_failure: tuple[str, str] | None = None
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
@@ -92,18 +94,19 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# hand the same rows to the same broken write) — the feed fallback's candidates.
unread: list[str] = []
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None:
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
tell nobody anything the first one didn't. Call it from inside the `except` block."""
nonlocal failed, first_error
nonlocal failed, first_error, first_failure
if unexpected is not None:
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
log.log(level, unexpected, exc_info=True)
failed += 1
reasons[reason] += 1
first_error = first_error or detail
first_failure = first_failure or (channel_id, reason)
if len(examples) < RSS_FAILURE_EXAMPLES:
examples.append(detail)
@@ -125,13 +128,16 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# bug on our side and earns a traceback (the first of its kind per poll).
except RssError as exc:
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
fail(exc.reason, f"{channel.id}: {exc}")
fail(exc.reason, channel.id, f"{channel.id}: {exc}")
unread.append(channel.id)
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
# traceback, once per reason per poll, which is the flood control this loop needs.
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
fail(
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
"error",
channel.id,
f"{channel.id}: {exc}",
unexpected=f"RSS fetch failed for {channel.id}",
)
else:
try:
@@ -144,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
db.rollback()
fail(
"apply",
channel.id,
f"{channel.id} (apply): {exc}",
unexpected=f"RSS apply failed for {channel.id}",
)
@@ -180,9 +187,17 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
# "fix this subscription"; `Connection refused` says "the egress is down".
# The fallback's count rides along: a red card during an RSS outage should still say
# whether the uploads are arriving another way.
raise RuntimeError(
# As data too (E31): the audit log and the inbox say it in the reader's language, the first
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
first_channel, first_reason = first_failure or ("", "error")
raise progress.JobFailed(
"rss_all_failed",
f"RSS poll failed for all {total} channels — first: {first_error}"
f" (read through the API instead: {fallback['read']})"
f" (read through the API instead: {fallback['read']})",
total=total,
channel=first_channel,
cause=first_reason,
read=fallback["read"],
)
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
+28
View File
@@ -86,3 +86,31 @@ def test_a_job_runs_audit_row_carries_its_result_as_data(db):
{"status": "skipped", "summary": "sync paused", "result": {"skipped": "sync paused"}},
{"status": "error", "summary": "boom"},
]
def test_a_failure_the_job_names_rides_along_as_data(db, make_user):
"""E31: a failed RSS poll reached the Hungarian audit log as the server's English sentence. A
failure the job names (`progress.JobFailed`) is stored as data beside that sentence — on the
dashboard, the manual-run notice and the audit row — so the client says it in the reader's
language. Any other failure keeps only its own text (the test above)."""
from app import progress
from app.models import AuditLog
def outage(_db):
raise progress.JobFailed("rss_all_failed", "RSS poll failed", total=2, cause="transport")
admin = make_user("failure@example.com", role="admin")
token = scheduler._manual_actor.set(admin.id)
try:
scheduler._job("rss_poll", outage)
finally:
scheduler._manual_actor.reset(token)
named = {"failure": "rss_all_failed", "total": 2, "cause": "transport"}
rss = _snapshot_job("rss_poll")
assert rss["last_error"] == "RSS poll failed"
assert rss["result"] == named
notice = db.execute(select(Notification).where(Notification.user_id == admin.id)).scalar_one()
assert notice.data["result"] == named
row = db.execute(select(AuditLog).order_by(AuditLog.id.desc())).scalars().first()
assert row is not None and row.after["result"] == named
+9 -1
View File
@@ -6,6 +6,7 @@ import logging
import httpx
import pytest
from app import progress
from app.sync import runner
from app.sync.playlists import FAILED, _Breaker
from app.sync.subscriptions import should_prune
@@ -224,8 +225,15 @@ class TestRssPollResult:
def dead(cid):
raise rss.RssError(f"RSS fetch for {cid} returned HTTP 404", "http_404")
with pytest.raises(RuntimeError, match="HTTP 404"):
with pytest.raises(RuntimeError, match="HTTP 404") as err:
self._run(monkeypatch, fetch=dead, apply_=lambda db, ch, e: 1, channels=("UC_dead",))
# E31: and as data, for the reader's language — whose feed, and the reason code that tells a
# dead feed (`http_404`) from a dead egress (`transport`).
assert isinstance(err.value, progress.JobFailed)
assert (err.value.reason, err.value.params) == (
"rss_all_failed",
{"total": 1, "channel": "UC_dead", "cause": "http_404", "read": 0},
)
def test_a_read_only_database_is_a_total_outage_too(self, monkeypatch):
# Every feed reads fine, nothing can be stored: just as un-polled, just as red.
+5 -2
View File
@@ -35,7 +35,7 @@ import { useContextMenu } from "./ui/useContextMenu";
import { ContextMenu } from "./ui/ContextMenu";
import { MoreButton } from "./ui/MoreButton";
import { type ActionMenuItem } from "./ui/ActionMenu";
import { jobResultText } from "../lib/jobResult";
import { jobFailureText, jobResultText } from "../lib/jobResult";
const POLL_MS = 4000;
// While any job is running, poll faster so short scheduled runs aren't missed between ticks.
@@ -257,7 +257,10 @@ function JobRow({
? t("scheduler.lastRun", { time: relativeTime(job.last_finished, lang) })
: t("scheduler.neverRun")}
{job.last_error ? (
<span className="text-danger"> · {job.last_error}</span>
<span className="text-danger">
{" · "}
{jobFailureText(t, job.result, lang) ?? job.last_error}
</span>
) : job.result ? (
<span> · {jobResultText(t, job.result, lang)}</span>
) : job.last_result ? (
@@ -186,5 +186,14 @@
"lowquota": "low quota",
"noreadscope": "no YouTube read access",
"busy": "another sync is running"
},
"failure": {
"rss_all_failed": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}"
},
"rssCause": {
"http": "HTTP {{status}}",
"transport": "unreachable",
"apply": "could not be saved",
"error": "unexpected error"
}
}
@@ -186,5 +186,14 @@
"lowquota": "kevés a kvóta",
"noreadscope": "nincs YouTube-olvasási jogosultság",
"busy": "egy másik szinkron fut"
},
"failure": {
"rss_all_failed": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}"
},
"rssCause": {
"http": "HTTP {{status}}",
"transport": "nem érhető el",
"apply": "nem sikerült menteni",
"error": "váratlan hiba"
}
}
+32 -1
View File
@@ -4,7 +4,7 @@
// one-liner was "recent={'channels_processed': 0, …}, deep={…}" — English keys and a Python repr.
import { afterEach, describe, expect, it } from "vitest";
import i18n, { setLanguage } from "../i18n";
import { jobResultText, parseSummary } from "./jobResult";
import { jobFailureText, jobResultText, jobRunText, parseSummary } from "./jobResult";
afterEach(() => setLanguage("en"));
@@ -65,3 +65,34 @@ describe("an older notice's one-liner", () => {
expect(parseSummary(null)).toBeNull();
});
});
// E31: the audit log showed a failed RSS poll as the server's English sentence ("RSS poll failed for
// all 399 channels — first: …") under a Hungarian UI. A failure the job names rides along as data.
describe("a failed run", () => {
const run = {
job_id: "rss_poll",
status: "error",
summary:
"RSS poll failed for all 399 channels — first: UCx: RSS fetch for UCx returned HTTP 404",
result: { failure: "rss_all_failed", total: 1399, channel: "UCx", cause: "http_404", read: 0 },
};
it("says the failure the job named in the reader's language", () => {
const t = i18n.getFixedT("hu");
expect(jobRunText(t, run, "hu").body).toBe(
"Az RSS-hírcsatorna egyik csatornánál sem válaszolt (1399) — az első: UCx, HTTP 404. " +
"Az API-n át beolvasva helyette: 0",
);
expect(jobFailureText(t, { ...run.result, cause: "transport" }, "hu")).toContain(
"UCx, nem érhető el.",
);
});
it("keeps any other failure's own text", () => {
const t = i18n.getFixedT("hu");
expect(jobRunText(t, { ...run, summary: "Plex returned 500", result: null }, "hu").body).toBe(
"Plex returned 500",
);
expect(jobFailureText(t, { failure: "moon_is_full" }, "hu")).toBeNull();
});
});
+26 -4
View File
@@ -38,10 +38,30 @@ export function jobResultText(t: TFunction, result: JobResult, lang: string): st
.join(" · ");
}
/** A failure the job named (the server's `progress.JobFailed`: `{failure: <reason>, …params}`) in the
* reader's language (E31); null for any other failure, which keeps the upstream's own text. */
export function jobFailureText(t: TFunction, result: unknown, lang: string): string | null {
if (!result || typeof result !== "object") return null;
const { failure, ...params } = result as Record<string, unknown>;
if (typeof failure !== "string") return null;
const values: Record<string, string> = {};
for (const [k, v] of Object.entries(params))
values[k] = typeof v === "number" ? formatNumber(v, lang) : String(v);
if (typeof params.cause === "string") values.cause = rssCause(t, params.cause);
return t(`scheduler.failure.${failure}`, { ...values, defaultValue: "" }) || null;
}
/** Why a feed did not answer, from the poll's reason code (`http_404`, `transport`, …). */
function rssCause(t: TFunction, cause: string): string {
const http = /^http_(\d+)$/.exec(cause);
if (http) return t("scheduler.rssCause.http", { status: http[1] });
return t(`scheduler.rssCause.${cause}`, { defaultValue: cause });
}
/** What a job run said, as the inbox notice and the audit row both show it (E30): a title naming the job
* and how the run ended, and its result in the reader's language — from the data when it rode along,
* else read back from the English one-liner. A skipped run names only why; a failed run keeps the
* upstream's own error text (it is diagnostic). */
* else read back from the English one-liner. A skipped run names only why; a failed run says the
* failure the job named (`jobFailureText`, E31), else keeps the upstream's own text (diagnostic). */
export function jobRunText(
t: TFunction,
run: { job_id: string; status?: unknown; summary?: unknown; result?: unknown },
@@ -65,10 +85,12 @@ export function jobRunText(
}
// A skipped run's title already says "skipped": its body is only the reason, not "skipped: <reason>".
const reason = run.status === "skipped" ? onlySkipReason(result) : null;
const failure = run.status === "error" ? jobFailureText(t, run.result, lang) : null;
return {
title: t(titleKey, { job }),
body:
reason !== null
body: failure
? failure
: reason !== null
? t(`scheduler.skip.${vocabKey(reason)}`, { defaultValue: reason })
: result
? jobResultText(t, result, lang)