fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+31 -5
View File
@@ -342,6 +342,29 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure
assert (failed.status, failed.error) == ("error", "quota exceeded")
def _ad_hoc_row(db, job: str) -> dict:
rows = scheduler.scheduler_snapshot(db)["ad_hoc"]
assert [r["id"] for r in rows] == list(scheduler.AD_HOC_JOBS) # every one has its row
return next(r for r in rows if r["id"] == job)
def test_a_takeout_import_has_its_row_named_by_the_importing_user(db, make_user):
user = make_user("importer@example.com")
user.display_name = "Ida Importer"
db.commit()
assert _ad_hoc_row(db, "takeout_import")["recent"] == []
scheduler._ad_hoc("takeout_import", lambda: {"watched": 2}, user.id, f"user:{user.id}")
row = _ad_hoc_row(db, "takeout_import")
assert (row["running"], row["target"], row["target_title"], row["status"]) == (
False,
f"user:{user.id}",
"Ida Importer",
"ok",
)
assert _ad_hoc_row(db, "channel_backfill")["recent"] == []
def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user):
user = make_user("shown@example.com")
db.add(Channel(id="UC_shown", title="BBC Earth"))
@@ -349,7 +372,7 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none")
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown")
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
row = _ad_hoc_row(db, "channel_backfill")
assert (row["id"], row["running"], row["target"], row["target_title"]) == (
"channel_backfill",
False,
@@ -367,12 +390,12 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
seen = []
scheduler._ad_hoc(
"channel_backfill",
lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {},
lambda: seen.append(_ad_hoc_row(db, "channel_backfill")["running"]) or {},
user.id,
"UC_shown",
)
assert seen == [True]
assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False
assert _ad_hoc_row(db, "channel_backfill")["running"] is False
def _finished(db, job) -> list[JobRun]:
@@ -397,6 +420,9 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run(
"channel_backfill@UC_x",
"channel_backfill@UC_y",
]
# Queued, so its target's newest run row is not this run's yet: a page watching it can tell.
assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_x")
assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_z")
live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next
@@ -475,7 +501,7 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip")
assert calls == ["ok"]
# Its row is still `running`, but nothing here is: the page does not call it going.
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
row = _ad_hoc_row(db, "channel_backfill")
assert (row["running"], row["status"], row["recent"][0]["status"]) == (
False,
"interrupted",
@@ -529,7 +555,7 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use
return {}
scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long")
(row,) = rows
(row,) = [r for r in rows if r["id"] == "channel_backfill"]
assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one")
# The outcome beside it is the long one's (none yet), never the short one's skip.
assert (row["status"], row["result"], row["last_error"]) == (None, None, None)