feat(scheduler): a channel's "get full history" backfill is a recorded run, and a row knows its last three runs

The one-channel deep backfill ran as a FastAPI BackgroundTask: what it fetched, what it spent and
how it failed reached only the log. It is now a one-shot run on the scheduler (`channel_backfill`,
trigger ad_hoc, actor = the user, target = the channel) with its quota tagged to the run. Its
per-channel exclusion with the scheduled backfill and the two-run cap are unchanged, and a run
they turn away says why in its result instead of returning nothing. A failure is no longer
swallowed: the run records it.

The scheduler snapshot carries each job's last three runs (one windowed query), so the page can
show an interrupted or skipped run instead of the latest outcome alone.
This commit is contained in:
2026-10-18 01:05:24 +02:00
parent 911527a548
commit 27c2fcbd2e
8 changed files with 256 additions and 55 deletions
+51 -7
View File
@@ -107,22 +107,32 @@ def _setup_channel_and_sub(db, *, email):
def _patch_deep_requested(db, monkeypatch, override_dep, user):
submitted: list[tuple] = []
calls: list[tuple] = []
# background.add_task runs after the response in TestClient; record what would be scheduled.
# Record the run the route submits, and what its work would call, instead of running it.
monkeypatch.setattr(
channels_route.scheduler,
"run_ad_hoc",
lambda job, fn, **kw: submitted.append((job, kw)) or fn(),
)
monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a))
monkeypatch.setattr(state, "_configured_cache", True)
override_dep(get_db, db.info["request"])
override_dep(current_user, lambda: user)
resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True})
return resp, calls
return resp, submitted, calls
def test_deep_request_schedules_immediate_backfill(db, monkeypatch, override_dep):
def test_deep_request_starts_the_backfill_as_the_users_run_of_that_channel(
db, monkeypatch, override_dep
):
"""It was a BackgroundTask: what it did, spent or failed with reached only the log."""
ch, me = _setup_channel_and_sub(db, email="wire@example.com")
resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
resp, submitted, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
assert resp.status_code == 200, resp.text
assert resp.json()["deep_requested"] is True
# scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
assert submitted == [("channel_backfill", {"actor_id": me.id, "target": ch.id})]
# its work runs exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
assert calls == [(ch.id, me.id)]
@@ -148,15 +158,49 @@ class TestImmediateBackfillGuard:
def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch):
runner._backfill_in_progress.add("UC_busy")
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs
# returns at once — _boom_session never runs — and the run says why it did nothing.
assert runner.backfill_channel_now("UC_busy", 1) == {"skipped": "channel busy"}
assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched
def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch):
runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it
# over the cap → no-op, the scheduler will handle it
assert runner.backfill_channel_now("UC_new", 1) == {"skipped": "at capacity"}
assert "UC_new" not in runner._backfill_in_progress
@staticmethod
def _one_channel(db, monkeypatch, cid):
ch = Channel(id=cid, title=cid, uploads_playlist_id="UU" + cid, backfill_done=False)
me = User(email=f"{cid}@example.com")
db.add_all([ch, me])
db.commit()
monkeypatch.setattr(runner, "get_service_user", lambda _db: me)
monkeypatch.setattr(runner.quota, "backfill_reserve_reached", lambda: False)
monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt())
return ch, me
def test_a_failure_reaches_its_run_and_still_frees_the_channel(self, db, monkeypatch):
"""The failure was logged and swallowed, so the run that now wraps it would read "ok"."""
ch, me = self._one_channel(db, monkeypatch, "UC_fail")
def boom(*_a):
raise RuntimeError("playlistItems 500")
monkeypatch.setattr(runner, "backfill_channel_deep", boom)
with pytest.raises(RuntimeError, match="playlistItems 500"):
runner.backfill_channel_now(ch.id, me.id)
assert runner._backfill_in_progress == set() and runner._immediate_running == 0
def test_a_finished_backfill_says_what_it_added(self, db, monkeypatch):
ch, me = self._one_channel(db, monkeypatch, "UC_ok")
monkeypatch.setattr(runner, "backfill_channel_deep", lambda *_a: 7)
monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 1)
assert runner.backfill_channel_now(ch.id, me.id) == {
"videos_added": 7,
"reconciled_full": 1,
}
def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch):
# A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)…
ch = Channel(