fix(downloads): an upload's conversion is guarded, counted, named and cleaned up
- Every ffmpeg/ffprobe over a library file names the demuxers it may use
(edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
kept after a failed conversion) — the footprint, the GC's cap and the admin
view read one ON_DISK predicate — and a conversion that outgrows the cap is
refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
uploads shared a file); a converted file placed but not committed is removed,
and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
convert_failed / upload_missing, reports the "converting" phase, and notifies
as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
+29
-21
@@ -6,12 +6,13 @@ Five passes each run:
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets until back under it.
|
||||
ready assets (never an upload) until back under it.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
|
||||
anything newer than an hour so an in-flight download is never touched.
|
||||
re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file).
|
||||
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||
so an in-flight download is never touched.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
@@ -20,19 +21,19 @@ import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.downloads import quota, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
# The one dot-prefixed top-level tree the orphan sweep enters. The others are not part of the asset
|
||||
# layout (user clips, Plex HLS/img caches, in-progress staging, editor storyboards) and are skipped.
|
||||
_UPLOAD_TREE = ".uploads"
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
@@ -67,16 +68,20 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
|
||||
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
|
||||
is never deleted, and only files NOT owned by a surviving asset."""
|
||||
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||
NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
# An upload's file is on disk from ingest on, whatever its state (a queued conversion's original,
|
||||
# a failed one kept for Retry), so every upload row keeps its file — not only a ready one.
|
||||
for rel in db.execute(
|
||||
select(MediaAsset.rel_path).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
or_(MediaAsset.status == "ready", MediaAsset.source_kind == "upload"),
|
||||
MediaAsset.rel_path.is_not(None),
|
||||
)
|
||||
).scalars():
|
||||
media = (root / rel).resolve()
|
||||
@@ -99,7 +104,10 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
# .edits / .plex-* / .staging / .storyboards + any dotfile — but `.uploads` is swept: a
|
||||
# conversion stopped between placing the new file and the commit that points at it (or the
|
||||
# unlink of the original after it) leaves a file no row names, and nothing else finds it.
|
||||
if entry.name.startswith(".") and entry.name != _UPLOAD_TREE:
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
@@ -188,17 +196,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
if cap > 0:
|
||||
total = (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
# An upload counts toward the total (it is disk like any other) but is never evicted:
|
||||
# it has no source to fetch again, and it lives until its owner deletes it.
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
.where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
@@ -232,10 +240,10 @@ def run_download_gc(db: Session) -> dict:
|
||||
if orphans:
|
||||
db.commit()
|
||||
|
||||
# 4b. Reap unheld uploads. An upload carries no expires_at and is never re-enqueued, so the passes
|
||||
# above never free one that lost its last holder outside `release_asset` — a conversion that
|
||||
# finished after its job was cancelled, or one cancelled mid-run (the asset was `downloading`
|
||||
# then). Its file, ready or still the unconverted original, sits at `rel_path`.
|
||||
# 4b. Reap unheld uploads. An upload carries no expires_at, is never evicted and never re-enqueued,
|
||||
# so the passes above never free one that lost its last holder outside `release_asset` — a
|
||||
# conversion that finished after its job was cancelled, or one cancelled mid-run (the asset
|
||||
# was `downloading` then). Its file, ready or still the unconverted original, is at `rel_path`.
|
||||
unheld = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
|
||||
Reference in New Issue
Block a user