fix(downloads): shell-safe names, refresh display_name on re-download, GC orphan sweep
Follow-up to user prod testing of 0.51.0: - re-download now clears job.display_name so the card title + Content-Disposition filename are re-derived from the freshly-cleaned title (previously an auto-name from an older un-cleaned title — still carrying a "… views · … reactions |" prefix — survived the rebuild). - on-disk AND served names are now a strict shell-safe slug: only [A-Za-z0-9._-] survive; every other char (the "!" the user saw, plus ( ) [ ] & $ ' " ; | ? * …) collapses to "_". The id no longer uses [brackets] (glob metacharacters). Shared _slug helper for sanitize + display_filename. - GC gains a 5th pass: sweep on-disk files/dirs no ready asset owns (old accented channel dirs, a poster from a superseded naming scheme left after a re-download), skipping the dot system trees and anything newer than an hour so an in-flight download is never touched. - Tests: slug strips !/$/()[]; rel_path is bracket-free; orphan-sweep keeps owned/system/fresh files.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
"""Download retention garbage collector (a scheduler job).
|
||||
|
||||
Four passes each run:
|
||||
Five passes each run:
|
||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||
once (gc_notified flag) so a download isn't silently deleted.
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
@@ -9,11 +9,16 @@ Four passes each run:
|
||||
ready assets until back under it.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
|
||||
anything newer than an hour so an in-flight download is never touched.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -26,6 +31,11 @@ from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
@@ -57,6 +67,53 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
||||
|
||||
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
|
||||
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
|
||||
is never deleted, and only files NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
for rel in db.execute(
|
||||
select(MediaAsset.rel_path).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
)
|
||||
).scalars():
|
||||
media = (root / rel).resolve()
|
||||
keep |= {media, media.with_suffix(".nfo"), media.with_suffix(".jpg")}
|
||||
|
||||
cutoff = time.time() - _ORPHAN_MIN_AGE_S
|
||||
removed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
try:
|
||||
if f.resolve() in keep or f.stat().st_mtime > cutoff:
|
||||
continue
|
||||
f.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||
for d in sorted(
|
||||
(p for p in entry.rglob("*") if p.is_dir()), key=lambda p: len(p.parts), reverse=True
|
||||
):
|
||||
try:
|
||||
d.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
entry.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
return removed
|
||||
|
||||
|
||||
def run_download_gc(db: Session) -> dict:
|
||||
now = datetime.now(timezone.utc)
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
@@ -132,6 +189,12 @@ def run_download_gc(db: Session) -> dict:
|
||||
if orphans:
|
||||
db.commit()
|
||||
|
||||
result = {"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped}
|
||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
|
||||
result = {
|
||||
"warned": warned, "expired": expired, "evicted": evicted, "reaped": reaped, "swept": swept
|
||||
}
|
||||
log.info("download_gc %s", result)
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user