feat(files): every file sits in a library folder, and keeps its own expiry
The Files library starts here. A user's every download, clip and upload now lands in a source folder made per origin (YouTube, Plex clips, uploads, one per other site), decided once in downloads/folders.py at every place a job is made. How long a file stays moves from the shared asset to the user's job: stamped from its folder when it is done, never for a file the user keeps. The GC warns each user once, then takes the expired file out of their library; the cache cap never evicts a kept file, and an evicted one leaves every holder's library. The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume siftlode_files); docs/self-hosting.md says how to carry an old volume over. Migration 0102 places every existing job and carries each done file's expiry over from its asset. safe_abs_path takes a missing path and answers None, which drops one baselined type error.
This commit is contained in:
+77
-58
@@ -1,12 +1,13 @@
|
||||
"""Download retention garbage collector (a scheduler job).
|
||||
|
||||
Five passes each run:
|
||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||
once (gc_notified flag) so a download isn't silently deleted.
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets (never an upload) until back under it.
|
||||
1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
|
||||
within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
|
||||
2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
|
||||
released, so the shared file goes with its last holder.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
|
||||
assets (never an upload, never one a user kept) until back under it; each holder's file leaves
|
||||
their library, with a notice.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
@@ -14,7 +15,7 @@ Five passes each run:
|
||||
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||
so an in-flight download is never touched.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -26,7 +27,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import quota, storage
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
@@ -37,42 +39,39 @@ _UPLOAD_TREE = ".uploads"
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||
).scalars()
|
||||
def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
|
||||
title = job.display_name or (asset.title if asset else None) or "download"
|
||||
create_notification(
|
||||
db,
|
||||
user_id=job.user_id,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={
|
||||
"kind": ntype,
|
||||
"title": title,
|
||||
"video_id": asset.source_ref if asset else None,
|
||||
"job_id": job.id,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
|
||||
|
||||
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
title = asset.title or "download"
|
||||
for uid in _holders(db, asset.id):
|
||||
create_notification(
|
||||
db,
|
||||
user_id=uid,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
||||
commit=False,
|
||||
)
|
||||
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||
_notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
|
||||
service.release_asset(db, job)
|
||||
db.delete(job)
|
||||
|
||||
|
||||
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
_notify_holders(db, asset, ntype)
|
||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
||||
|
||||
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
def _sweep_orphan_files(db: Session, files_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||
NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
@@ -151,47 +150,52 @@ def run_download_gc(db: Session) -> dict:
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
warned = expired = evicted = reaped = 0
|
||||
|
||||
# 1. Pre-expiry warnings.
|
||||
# 1. Pre-expiry warnings — one per file, the first run inside the grace window.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at > now,
|
||||
DownloadJob.expires_at <= warn_before,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in soon:
|
||||
_notify_holders(db, asset, "download_expiring")
|
||||
asset.gc_notified = True
|
||||
for job in soon:
|
||||
_notify(
|
||||
db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
|
||||
)
|
||||
job.expiry_warned = True
|
||||
warned += 1
|
||||
if soon:
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||
dead = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at <= now,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in dead:
|
||||
_delete_asset(db, asset, "download_expired")
|
||||
for job in dead:
|
||||
_drop_job(db, job, "download_expired")
|
||||
expired += 1
|
||||
if dead:
|
||||
db.commit()
|
||||
|
||||
# 3. LRU eviction over the total cache cap.
|
||||
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
|
||||
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = (
|
||||
@@ -201,12 +205,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
# An upload counts toward the total (it is disk like any other) but is never evicted:
|
||||
# it has no source to fetch again, and it lives until its owner deletes it.
|
||||
kept_assets = select(DownloadJob.asset_id).where(
|
||||
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
|
||||
)
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
|
||||
.where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.source_kind != "upload",
|
||||
MediaAsset.id.not_in(kept_assets),
|
||||
)
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
@@ -216,7 +225,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
if total <= cap:
|
||||
break
|
||||
total -= asset.size_bytes or 0
|
||||
_delete_asset(db, asset, "download_evicted")
|
||||
holders = (
|
||||
db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in holders:
|
||||
_notify(db, job, asset, "download_evicted")
|
||||
db.delete(job)
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
evicted += 1
|
||||
db.commit()
|
||||
|
||||
@@ -257,7 +276,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
)
|
||||
for asset in unheld:
|
||||
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
reaped += 1
|
||||
if unheld:
|
||||
@@ -265,7 +284,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
|
||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
swept = _sweep_orphan_files(db, settings.files_root)
|
||||
|
||||
result = {
|
||||
"warned": warned,
|
||||
|
||||
Reference in New Issue
Block a user