fix(downloads): an upload's conversion is guarded, counted, named and cleaned up
- Every ffmpeg/ffprobe over a library file names the demuxers it may use
(edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
kept after a failed conversion) — the footprint, the GC's cap and the admin
view read one ON_DISK predicate — and a conversion that outgrows the cap is
refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
uploads shared a file); a converted file placed but not committed is removed,
and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
convert_failed / upload_missing, reports the "converting" phase, and notifies
as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
@@ -59,6 +59,10 @@ _DB_WRITE_INTERVAL_S = 1.0
|
||||
_CANCEL_POLL_INTERVAL_S = 1.0
|
||||
|
||||
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
||||
# The job phase while a file is brought into the phone-safe shape (`_ensure_phone_safe`) for a
|
||||
# download or an upload. Its own word, not yt-dlp's "processing": it reports ffmpeg's real progress,
|
||||
# so the queue row shows a % for it, where yt-dlp's post-steps only get an indeterminate pulse.
|
||||
CONVERTING = "converting"
|
||||
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
||||
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
||||
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
||||
@@ -76,7 +80,7 @@ def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *,
|
||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||
scheduler._notify_done). `kind` ("download"/"edit") lets the inbox pick accurate copy; the
|
||||
scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the
|
||||
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
||||
caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row."""
|
||||
if user_id is None:
|
||||
@@ -529,7 +533,7 @@ def _download(
|
||||
job_id,
|
||||
produced,
|
||||
staging,
|
||||
"processing",
|
||||
CONVERTING,
|
||||
int(info["duration"]) if info.get("duration") else None,
|
||||
)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
@@ -751,7 +755,9 @@ def _fail_edit(
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_code=dlerrors.classify_conversion(msg)
|
||||
if what == "upload"
|
||||
else dlerrors.classify(msg),
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
@@ -833,6 +839,8 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
@@ -844,7 +852,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.size_bytes = size
|
||||
asset.container = out_ext
|
||||
asset.duration_s = total_s or asset.duration_s
|
||||
if crop:
|
||||
if out_w and out_h:
|
||||
asset.width, asset.height = out_w, out_h
|
||||
elif crop:
|
||||
asset.width = edit_spec["crop"]["w"]
|
||||
asset.height = edit_spec["crop"]["h"]
|
||||
else:
|
||||
@@ -886,29 +896,57 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
||||
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
||||
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
||||
for the owner to delete or keep."""
|
||||
if owner_id is None:
|
||||
return
|
||||
with SessionLocal() as db:
|
||||
if not quota.fits_after_conversion(db, owner_id, old_size, new_size):
|
||||
raise StatedFailure(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
|
||||
|
||||
def _unplace(placed: Path | None) -> None:
|
||||
"""Remove a converted upload that was placed at its new rel but never committed: the row still
|
||||
names the original, so nothing else would ever find this file."""
|
||||
if placed is not None:
|
||||
placed.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
|
||||
|
||||
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
|
||||
under the new extension, and only then is the original removed — so a failed or stopped
|
||||
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`."""
|
||||
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
|
||||
A converted file placed but never committed is removed again on the way out; a stop in the
|
||||
instant between the commit and the original's unlink is left to the GC's `.uploads` sweep."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
old_rel = asset.rel_path if asset else None
|
||||
old_ext = asset.container if asset else None
|
||||
old_size = (asset.size_bytes or 0) if asset else 0
|
||||
duration = asset.duration_s if asset else None
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
|
||||
if old_rel is None or src is None or not src.exists():
|
||||
_fail_edit(job_id, asset_id, "The uploaded file is missing.", what="upload")
|
||||
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||
return
|
||||
staging = _staging_dir(asset_id)
|
||||
placed: Path | None = None # the converted file while it is at its new rel, uncommitted
|
||||
try:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
safe = _ensure_phone_safe(job_id, src, staging, "processing", duration, keep_source=True)
|
||||
safe = _ensure_phone_safe(job_id, src, staging, CONVERTING, duration, keep_source=True)
|
||||
ext = safe.suffix.lstrip(".").lower()
|
||||
new_rel = Path(old_rel).with_suffix(f".{ext}").as_posix()
|
||||
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||
if safe != src:
|
||||
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||
storage.place_file(safe, settings.download_root, new_rel)
|
||||
if new_rel != old_rel:
|
||||
placed = storage.abs_path(settings.download_root, new_rel)
|
||||
final = storage.abs_path(settings.download_root, new_rel)
|
||||
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||
with SessionLocal() as db:
|
||||
@@ -923,15 +961,14 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
asset.duration_s = dur or asset.duration_s
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
title = asset.title if asset else None
|
||||
finalized = service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||
)
|
||||
db.commit()
|
||||
placed = None
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, title, job_id, kind="download")
|
||||
_notify_download_done(db, owner_id, title, job_id, kind="upload")
|
||||
if new_rel != old_rel:
|
||||
src.unlink(missing_ok=True)
|
||||
if finalized:
|
||||
@@ -943,6 +980,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
)
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: the original is untouched; free the asset for a later retry.
|
||||
_unplace(placed)
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"upload job %s conversion aborted (pause/cancel)",
|
||||
@@ -951,6 +989,8 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_unplace(placed)
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="upload")
|
||||
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
Reference in New Issue
Block a user