fix(reader): the Takeout import holds up under abuse, races and a run it cannot see

- Uploads: one at a time per user and ten an hour, each refused only after its body arrived;
  the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000,
  a user's kept videos at 200k; any unreadable shape is a 422, not a 500.
- Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is
  not asked about waits, known videos still go in, and the shorts job retries the rest.
- Staging and settling share one per-user advisory lock (no deadlock); a run settles only the
  rows it read, so a row staged meanwhile is never called gone.
- A failed run's notice names a translated failure, never the upstream's text.
- The status says when an import is queued; the page watches it to its end even when left, then
  reads the feed, its counts and the playlists again.
- Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back,
  a missed drop kept from the browser, an oversize file refused before it is sent.
- Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
2026-10-23 20:48:23 +02:00
parent e0b02a0c14
commit 9f580fd8c8
29 changed files with 1235 additions and 213 deletions
+68 -27
View File
@@ -274,7 +274,6 @@ def scheduler_snapshot(db) -> dict:
open_runs = _open_run_ids()
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
open_runs |= _open_run_ids()
going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None)
jobs = []
for job_id, default_interval in JOB_INTERVALS.items():
a = acts.get(job_id, {})
@@ -290,30 +289,11 @@ def scheduler_snapshot(db) -> dict:
)
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
# target's own (sync/runner), so "running" is the runs this process has open: two channels can
# backfill at once, so a run still going counts even when a newer one already ended. That run then
# leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the
# newer run's outcome under the older one's channel.
backfills = recent.get(CHANNEL_BACKFILL, [])
going = db.get(JobRun, going_id) if going_id is not None else None
if going is not None and going.status != RunStatus.RUNNING:
going = None # its outcome is written; only the bookkeeping has not caught up
if going is not None:
backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS]
shown = backfills[0] if backfills else None
ad_hoc = [
{
"id": CHANNEL_BACKFILL,
"running": going is not None,
"target": shown.target if shown else None,
"target_title": (
catalog_queries.channel_title(db, shown.target)
if shown is not None and shown.target
else None
),
**_last_runs(backfills, open_runs),
}
]
# target's own (sync/runner, the import's per-user lock), so "running" is the runs this process
# has open: two targets can run at once, so a run still going counts even when a newer one already
# ended. That run then leads the row, so its target, its "no outcome yet" and its dot are one
# run's, never the newer run's outcome under the older one's target.
ad_hoc = [_ad_hoc_row(db, job, recent.get(job, []), open_runs) for job in AD_HOC_JOBS]
return {
"running_here": running_here,
"enabled": settings.scheduler_enabled,
@@ -335,6 +315,27 @@ def _seen(run, open_runs: dict[int, str]) -> RunStatus:
return RunStatus(run.status)
def _ad_hoc_row(db, job: str, runs: list[JobRun], open_runs: dict[int, str]) -> dict:
going_id = max((r for r, j in open_runs.items() if j == job), default=None)
going = db.get(JobRun, going_id) if going_id is not None else None
if going is not None and going.status != RunStatus.RUNNING:
going = None # its outcome is written; only the bookkeeping has not caught up
if going is not None:
runs = [going, *(r for r in runs if r.id != going.id)][:RECENT_RUNS]
shown = runs[0] if runs else None
return {
"id": job,
"running": going is not None,
"target": shown.target if shown else None,
"target_title": (
ad_hoc_target_title(db, job, shown.target)
if shown is not None and shown.target
else None
),
**_last_runs(runs, open_runs),
}
def _last_runs(runs: list, open_runs: dict[int, str]) -> dict:
"""A row's run fields from its newest runs (newest first): how the last one went, and the few
before it for the history dots."""
@@ -619,6 +620,29 @@ CHANNEL_BACKFILL = "channel_backfill"
TAKEOUT_IMPORT = "takeout_import"
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT)
# What each one's target is called on the admin's pages: a channel's title, an importing user's name.
_TARGET_TITLES: dict[str, Callable[..., str | None]] = {
CHANNEL_BACKFILL: catalog_queries.channel_title,
TAKEOUT_IMPORT: takeout.target_title,
}
def ad_hoc_target_title(db, job: str, target: str) -> str | None:
"""The name of an ad-hoc run's target (None for a job with none, or a target since gone)."""
title = _TARGET_TITLES.get(job)
return title(db, target) if title is not None else None
# The ad-hoc runs added and not yet recorded, `job@target`: between the two the target's newest
# run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too.
_queued: set[str] = set()
_queued_guard = threading.Lock()
def ad_hoc_queued(job: str, target: str) -> bool:
"""Whether a run of `job` on `target` is queued and its run row not written yet."""
with _queued_guard:
return f"{job}@{target}" in _queued
def run_ad_hoc(
@@ -634,17 +658,24 @@ def run_ad_hoc(
if sched is None:
# Only between shutdown and exit: start_scheduler runs at every API startup.
raise RuntimeError("the scheduler is not running")
key = f"{job}@{target}"
with _queued_guard:
_queued.add(key)
try:
sched.add_job(
_ad_hoc,
"date",
args=(job, fn, actor_id, target, notify),
id=f"{job}@{target}",
id=key,
misfire_grace_time=None,
)
# silent-ok: not a failure — the run for this target is already queued, and it does this work.
except ConflictingIdError:
log.info("ad-hoc %s for %s already queued", job, target)
except Exception:
with _queued_guard:
_queued.discard(key)
raise
def _ad_hoc(
@@ -656,6 +687,10 @@ def _ad_hoc(
# No row to tell it in: the database is what failed. The log is the record.
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
return
finally:
# Recorded (or never will be): the run row now speaks for it.
with _queued_guard:
_queued.discard(f"{job}@{target}")
_opened(run_id, job)
try:
_ad_hoc_work(job, fn, run_id, actor_id if notify else None)
@@ -672,7 +707,11 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int |
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
err, failure = _failure(exc)
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
_notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure)
# The user's notice names the failure, never the upstream's own text: that is an English
# diagnostic, for the admin's run row and the log, not a user's inbox.
_notify_ad_hoc(
notify_id, job, RunStatus.ERROR, None, failure or {"failure": "unexpected"}
)
return
summary, data = _finish_ok(job, run_id, result)
_notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data)
@@ -725,3 +764,5 @@ def shutdown_scheduler() -> None:
if _scheduler is not None:
_scheduler.shutdown(wait=False)
_scheduler = None
with _queued_guard:
_queued.clear() # what it held will not run