fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
@@ -342,6 +342,29 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure
|
||||
assert (failed.status, failed.error) == ("error", "quota exceeded")
|
||||
|
||||
|
||||
def _ad_hoc_row(db, job: str) -> dict:
|
||||
rows = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
assert [r["id"] for r in rows] == list(scheduler.AD_HOC_JOBS) # every one has its row
|
||||
return next(r for r in rows if r["id"] == job)
|
||||
|
||||
|
||||
def test_a_takeout_import_has_its_row_named_by_the_importing_user(db, make_user):
|
||||
user = make_user("importer@example.com")
|
||||
user.display_name = "Ida Importer"
|
||||
db.commit()
|
||||
assert _ad_hoc_row(db, "takeout_import")["recent"] == []
|
||||
scheduler._ad_hoc("takeout_import", lambda: {"watched": 2}, user.id, f"user:{user.id}")
|
||||
|
||||
row = _ad_hoc_row(db, "takeout_import")
|
||||
assert (row["running"], row["target"], row["target_title"], row["status"]) == (
|
||||
False,
|
||||
f"user:{user.id}",
|
||||
"Ida Importer",
|
||||
"ok",
|
||||
)
|
||||
assert _ad_hoc_row(db, "channel_backfill")["recent"] == []
|
||||
|
||||
|
||||
def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user):
|
||||
user = make_user("shown@example.com")
|
||||
db.add(Channel(id="UC_shown", title="BBC Earth"))
|
||||
@@ -349,7 +372,7 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none")
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown")
|
||||
|
||||
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
row = _ad_hoc_row(db, "channel_backfill")
|
||||
assert (row["id"], row["running"], row["target"], row["target_title"]) == (
|
||||
"channel_backfill",
|
||||
False,
|
||||
@@ -367,12 +390,12 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
|
||||
seen = []
|
||||
scheduler._ad_hoc(
|
||||
"channel_backfill",
|
||||
lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {},
|
||||
lambda: seen.append(_ad_hoc_row(db, "channel_backfill")["running"]) or {},
|
||||
user.id,
|
||||
"UC_shown",
|
||||
)
|
||||
assert seen == [True]
|
||||
assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False
|
||||
assert _ad_hoc_row(db, "channel_backfill")["running"] is False
|
||||
|
||||
|
||||
def _finished(db, job) -> list[JobRun]:
|
||||
@@ -397,6 +420,9 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run(
|
||||
"channel_backfill@UC_x",
|
||||
"channel_backfill@UC_y",
|
||||
]
|
||||
# Queued, so its target's newest run row is not this run's yet: a page watching it can tell.
|
||||
assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_x")
|
||||
assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_z")
|
||||
live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next
|
||||
|
||||
|
||||
@@ -475,7 +501,7 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip")
|
||||
assert calls == ["ok"]
|
||||
# Its row is still `running`, but nothing here is: the page does not call it going.
|
||||
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
row = _ad_hoc_row(db, "channel_backfill")
|
||||
assert (row["running"], row["status"], row["recent"][0]["status"]) == (
|
||||
False,
|
||||
"interrupted",
|
||||
@@ -529,7 +555,7 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use
|
||||
return {}
|
||||
|
||||
scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long")
|
||||
(row,) = rows
|
||||
(row,) = [r for r in rows if r["id"] == "channel_backfill"]
|
||||
assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one")
|
||||
# The outcome beside it is the long one's (none yet), never the short one's skip.
|
||||
assert (row["status"], row["result"], row["last_error"]) == (None, None, None)
|
||||
|
||||
Reference in New Issue
Block a user