fix(downloads): shell-safe names, refresh display_name on re-download, GC orphan sweep

Follow-up to user prod testing of 0.51.0:
- re-download now clears job.display_name so the card title + Content-Disposition filename are
  re-derived from the freshly-cleaned title (previously an auto-name from an older un-cleaned
  title — still carrying a "… views · … reactions |" prefix — survived the rebuild).
- on-disk AND served names are now a strict shell-safe slug: only [A-Za-z0-9._-] survive; every
  other char (the "!" the user saw, plus ( ) [ ] & $ ' " ; | ? * …) collapses to "_". The id no
  longer uses [brackets] (glob metacharacters). Shared _slug helper for sanitize + display_filename.
- GC gains a 5th pass: sweep on-disk files/dirs no ready asset owns (old accented channel dirs, a
  poster from a superseded naming scheme left after a re-download), skipping the dot system trees
  and anything newer than an hour so an in-flight download is never touched.
- Tests: slug strips !/$/()[]; rel_path is bracket-free; orphan-sweep keeps owned/system/fresh files.
This commit is contained in:
2026-07-22 03:18:14 +02:00
parent e21c6f6b92
commit fa35925cc6
6 changed files with 196 additions and 60 deletions
+65 -2
View File
@@ -1,6 +1,6 @@
"""Download retention garbage collector (a scheduler job).
Four passes each run:
Five passes each run:
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
once (gc_notified flag) so a download isn't silently deleted.
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
@@ -9,11 +9,16 @@ Four passes each run:
ready assets until back under it.
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
passes 1-3 skip them; benign row-count hygiene).
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
anything newer than an hour so an in-flight download is never touched.
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import func, select
from sqlalchemy.orm import Session
@@ -26,6 +31,11 @@ from app.notifications import create_notification
log = logging.getLogger("siftlode.downloads.gc")
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
@@ -57,6 +67,53 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
def _sweep_orphan_files(db: Session, download_root: str) -> int:
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
is never deleted, and only files NOT owned by a surviving asset."""
root = Path(download_root)
if not root.is_dir():
return 0
keep: set[Path] = set()
for rel in db.execute(
select(MediaAsset.rel_path).where(
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
)
).scalars():
media = (root / rel).resolve()
keep |= {media, media.with_suffix(".nfo"), media.with_suffix(".jpg")}
cutoff = time.time() - _ORPHAN_MIN_AGE_S
removed = 0
for entry in root.iterdir():
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
continue
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
for f in files:
try:
if f.resolve() in keep or f.stat().st_mtime > cutoff:
continue
f.unlink()
removed += 1
except OSError:
pass
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
for d in sorted(
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
):
try:
d.rmdir()
except OSError:
pass
try:
entry.rmdir()
except OSError:
pass
return removed
def run_download_gc(db: Session) -> dict:
now = datetime.now(timezone.utc)
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
@@ -132,6 +189,12 @@ def run_download_gc(db: Session) -> dict:
if orphans:
db.commit()
result = {"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped}
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
swept = _sweep_orphan_files(db, settings.download_root)
result = {
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
}
log.info("download_gc %s", result)
return result