fix(reader): the Takeout import holds up under abuse, races and a run it cannot see
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
This commit is contained in:
+68
-27
@@ -274,7 +274,6 @@ def scheduler_snapshot(db) -> dict:
|
||||
open_runs = _open_run_ids()
|
||||
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
|
||||
open_runs |= _open_run_ids()
|
||||
going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None)
|
||||
jobs = []
|
||||
for job_id, default_interval in JOB_INTERVALS.items():
|
||||
a = acts.get(job_id, {})
|
||||
@@ -290,30 +289,11 @@ def scheduler_snapshot(db) -> dict:
|
||||
)
|
||||
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
|
||||
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
|
||||
# target's own (sync/runner), so "running" is the runs this process has open: two channels can
|
||||
# backfill at once, so a run still going counts even when a newer one already ended. That run then
|
||||
# leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the
|
||||
# newer run's outcome under the older one's channel.
|
||||
backfills = recent.get(CHANNEL_BACKFILL, [])
|
||||
going = db.get(JobRun, going_id) if going_id is not None else None
|
||||
if going is not None and going.status != RunStatus.RUNNING:
|
||||
going = None # its outcome is written; only the bookkeeping has not caught up
|
||||
if going is not None:
|
||||
backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS]
|
||||
shown = backfills[0] if backfills else None
|
||||
ad_hoc = [
|
||||
{
|
||||
"id": CHANNEL_BACKFILL,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
catalog_queries.channel_title(db, shown.target)
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(backfills, open_runs),
|
||||
}
|
||||
]
|
||||
# target's own (sync/runner, the import's per-user lock), so "running" is the runs this process
|
||||
# has open: two targets can run at once, so a run still going counts even when a newer one already
|
||||
# ended. That run then leads the row, so its target, its "no outcome yet" and its dot are one
|
||||
# run's, never the newer run's outcome under the older one's target.
|
||||
ad_hoc = [_ad_hoc_row(db, job, recent.get(job, []), open_runs) for job in AD_HOC_JOBS]
|
||||
return {
|
||||
"running_here": running_here,
|
||||
"enabled": settings.scheduler_enabled,
|
||||
@@ -335,6 +315,27 @@ def _seen(run, open_runs: dict[int, str]) -> RunStatus:
|
||||
return RunStatus(run.status)
|
||||
|
||||
|
||||
def _ad_hoc_row(db, job: str, runs: list[JobRun], open_runs: dict[int, str]) -> dict:
|
||||
going_id = max((r for r, j in open_runs.items() if j == job), default=None)
|
||||
going = db.get(JobRun, going_id) if going_id is not None else None
|
||||
if going is not None and going.status != RunStatus.RUNNING:
|
||||
going = None # its outcome is written; only the bookkeeping has not caught up
|
||||
if going is not None:
|
||||
runs = [going, *(r for r in runs if r.id != going.id)][:RECENT_RUNS]
|
||||
shown = runs[0] if runs else None
|
||||
return {
|
||||
"id": job,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
ad_hoc_target_title(db, job, shown.target)
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(runs, open_runs),
|
||||
}
|
||||
|
||||
|
||||
def _last_runs(runs: list, open_runs: dict[int, str]) -> dict:
|
||||
"""A row's run fields from its newest runs (newest first): how the last one went, and the few
|
||||
before it for the history dots."""
|
||||
@@ -619,6 +620,29 @@ CHANNEL_BACKFILL = "channel_backfill"
|
||||
TAKEOUT_IMPORT = "takeout_import"
|
||||
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
|
||||
AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT)
|
||||
# What each one's target is called on the admin's pages: a channel's title, an importing user's name.
|
||||
_TARGET_TITLES: dict[str, Callable[..., str | None]] = {
|
||||
CHANNEL_BACKFILL: catalog_queries.channel_title,
|
||||
TAKEOUT_IMPORT: takeout.target_title,
|
||||
}
|
||||
|
||||
|
||||
def ad_hoc_target_title(db, job: str, target: str) -> str | None:
|
||||
"""The name of an ad-hoc run's target (None for a job with none, or a target since gone)."""
|
||||
title = _TARGET_TITLES.get(job)
|
||||
return title(db, target) if title is not None else None
|
||||
|
||||
|
||||
# The ad-hoc runs added and not yet recorded, `job@target`: between the two the target's newest
|
||||
# run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too.
|
||||
_queued: set[str] = set()
|
||||
_queued_guard = threading.Lock()
|
||||
|
||||
|
||||
def ad_hoc_queued(job: str, target: str) -> bool:
|
||||
"""Whether a run of `job` on `target` is queued and its run row not written yet."""
|
||||
with _queued_guard:
|
||||
return f"{job}@{target}" in _queued
|
||||
|
||||
|
||||
def run_ad_hoc(
|
||||
@@ -634,17 +658,24 @@ def run_ad_hoc(
|
||||
if sched is None:
|
||||
# Only between shutdown and exit: start_scheduler runs at every API startup.
|
||||
raise RuntimeError("the scheduler is not running")
|
||||
key = f"{job}@{target}"
|
||||
with _queued_guard:
|
||||
_queued.add(key)
|
||||
try:
|
||||
sched.add_job(
|
||||
_ad_hoc,
|
||||
"date",
|
||||
args=(job, fn, actor_id, target, notify),
|
||||
id=f"{job}@{target}",
|
||||
id=key,
|
||||
misfire_grace_time=None,
|
||||
)
|
||||
# silent-ok: not a failure — the run for this target is already queued, and it does this work.
|
||||
except ConflictingIdError:
|
||||
log.info("ad-hoc %s for %s already queued", job, target)
|
||||
except Exception:
|
||||
with _queued_guard:
|
||||
_queued.discard(key)
|
||||
raise
|
||||
|
||||
|
||||
def _ad_hoc(
|
||||
@@ -656,6 +687,10 @@ def _ad_hoc(
|
||||
# No row to tell it in: the database is what failed. The log is the record.
|
||||
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
|
||||
return
|
||||
finally:
|
||||
# Recorded (or never will be): the run row now speaks for it.
|
||||
with _queued_guard:
|
||||
_queued.discard(f"{job}@{target}")
|
||||
_opened(run_id, job)
|
||||
try:
|
||||
_ad_hoc_work(job, fn, run_id, actor_id if notify else None)
|
||||
@@ -672,7 +707,11 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int |
|
||||
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
_notify_ad_hoc(notify_id, job, RunStatus.ERROR, err, failure)
|
||||
# The user's notice names the failure, never the upstream's own text: that is an English
|
||||
# diagnostic, for the admin's run row and the log, not a user's inbox.
|
||||
_notify_ad_hoc(
|
||||
notify_id, job, RunStatus.ERROR, None, failure or {"failure": "unexpected"}
|
||||
)
|
||||
return
|
||||
summary, data = _finish_ok(job, run_id, result)
|
||||
_notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data)
|
||||
@@ -725,3 +764,5 @@ def shutdown_scheduler() -> None:
|
||||
if _scheduler is not None:
|
||||
_scheduler.shutdown(wait=False)
|
||||
_scheduler = None
|
||||
with _queued_guard:
|
||||
_queued.clear() # what it held will not run
|
||||
|
||||
Reference in New Issue
Block a user