feat(files): every file sits in a library folder, and keeps its own expiry

The Files library starts here. A user's every download, clip and upload now
lands in a source folder made per origin (YouTube, Plex clips, uploads, one per
other site), decided once in downloads/folders.py at every place a job is made.
How long a file stays moves from the shared asset to the user's job: stamped
from its folder when it is done, never for a file the user keeps. The GC warns
each user once, then takes the expired file out of their library; the cache cap
never evicts a kept file, and an evicted one leaves every holder's library.

The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume
siftlode_files); docs/self-hosting.md says how to carry an old volume over.
Migration 0102 places every existing job and carries each done file's expiry
over from its asset.

safe_abs_path takes a missing path and answers None, which drops one baselined
type error.
This commit is contained in:
2026-10-11 03:30:34 +02:00
parent 07e0990ecf
commit bf99444ded
49 changed files with 984 additions and 226 deletions
+77 -58
View File
@@ -1,12 +1,13 @@
"""Download retention garbage collector (a scheduler job).
Five passes each run:
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
once (gc_notified flag) so a download isn't silently deleted.
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
ready assets (never an upload) until back under it.
1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
released, so the shared file goes with its last holder.
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
assets (never an upload, never one a user kept) until back under it; each holder's file leaves
their library, with a notice.
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
passes 1-3 skip them; benign row-count hygiene).
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
@@ -14,7 +15,7 @@ Five passes each run:
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
so an in-flight download is never touched.
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
"""
import time
@@ -26,7 +27,8 @@ from sqlalchemy.orm import Session
from app import logs, sysconfig
from app.config import settings
from app.downloads import quota, storage
from app.downloads import quota, service, storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
@@ -37,42 +39,39 @@ _UPLOAD_TREE = ".uploads"
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
def _holders(db: Session, asset_id: int) -> list[int]:
return list(
db.execute(
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
).scalars()
def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
title = job.display_name or (asset.title if asset else None) or "download"
create_notification(
db,
user_id=job.user_id,
type=ntype,
title=title, # English fallback; the client renders a translated message from data
data={
"kind": ntype,
"title": title,
"video_id": asset.source_ref if asset else None,
"job_id": job.id,
},
commit=False,
)
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
title = asset.title or "download"
for uid in _holders(db, asset.id):
create_notification(
db,
user_id=uid,
type=ntype,
title=title, # English fallback; the client renders a translated message from data
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
commit=False,
)
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
file goes with the last holder), delete the row (its shares and links cascade)."""
_notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
service.release_asset(db, job)
db.delete(job)
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
_notify_holders(db, asset, ntype)
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
def _sweep_orphan_files(db: Session, download_root: str) -> int:
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
def _sweep_orphan_files(db: Session, files_root: str) -> int:
"""Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
row names at all. Skips the other dot-prefixed system trees and only removes files older than
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
NOT owned by a surviving asset."""
root = Path(download_root)
root = Path(files_root)
if not root.is_dir():
return 0
keep: set[Path] = set()
@@ -151,47 +150,52 @@ def run_download_gc(db: Session) -> dict:
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = reaped = 0
# 1. Pre-expiry warnings.
# 1. Pre-expiry warnings — one per file, the first run inside the grace window.
warn_before = now + timedelta(days=grace_days)
soon = (
db.execute(
select(MediaAsset).where(
MediaAsset.status == "ready",
MediaAsset.gc_notified.is_(False),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at > now,
MediaAsset.expires_at <= warn_before,
select(DownloadJob).where(
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
DownloadJob.expiry_warned.is_(False),
DownloadJob.expires_at.is_not(None),
DownloadJob.expires_at > now,
DownloadJob.expires_at <= warn_before,
)
)
.scalars()
.all()
)
for asset in soon:
_notify_holders(db, asset, "download_expiring")
asset.gc_notified = True
for job in soon:
_notify(
db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
)
job.expiry_warned = True
warned += 1
if soon:
db.commit()
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
dead = (
db.execute(
select(MediaAsset).where(
MediaAsset.status.in_(("ready", "error")),
MediaAsset.expires_at.is_not(None),
MediaAsset.expires_at <= now,
select(DownloadJob).where(
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
DownloadJob.expires_at.is_not(None),
DownloadJob.expires_at <= now,
)
)
.scalars()
.all()
)
for asset in dead:
_delete_asset(db, asset, "download_expired")
for job in dead:
_drop_job(db, job, "download_expired")
expired += 1
if dead:
db.commit()
# 3. LRU eviction over the total cache cap.
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = (
@@ -201,12 +205,17 @@ def run_download_gc(db: Session) -> dict:
or 0
)
if total > cap:
# An upload counts toward the total (it is disk like any other) but is never evicted:
# it has no source to fetch again, and it lives until its owner deletes it.
kept_assets = select(DownloadJob.asset_id).where(
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
)
candidates = (
db.execute(
select(MediaAsset)
.where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
.where(
MediaAsset.status == "ready",
MediaAsset.source_kind != "upload",
MediaAsset.id.not_in(kept_assets),
)
.order_by(MediaAsset.last_access_at.asc().nulls_first())
)
.scalars()
@@ -216,7 +225,17 @@ def run_download_gc(db: Session) -> dict:
if total <= cap:
break
total -= asset.size_bytes or 0
_delete_asset(db, asset, "download_evicted")
holders = (
db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
.scalars()
.all()
)
for job in holders:
_notify(db, job, asset, "download_evicted")
db.delete(job)
if asset.rel_path:
storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
evicted += 1
db.commit()
@@ -257,7 +276,7 @@ def run_download_gc(db: Session) -> dict:
)
for asset in unheld:
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
storage.delete_asset_files(settings.download_root, asset.rel_path)
storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
reaped += 1
if unheld:
@@ -265,7 +284,7 @@ def run_download_gc(db: Session) -> dict:
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
swept = _sweep_orphan_files(db, settings.download_root)
swept = _sweep_orphan_files(db, settings.files_root)
result = {
"warned": warned,