feat(files): every file sits in a library folder, and keeps its own expiry

The Files library starts here. A user's every download, clip and upload now
lands in a source folder made per origin (YouTube, Plex clips, uploads, one per
other site), decided once in downloads/folders.py at every place a job is made.
How long a file stays moves from the shared asset to the user's job: stamped
from its folder when it is done, never for a file the user keeps. The GC warns
each user once, then takes the expired file out of their library; the cache cap
never evicts a kept file, and an evicted one leaves every holder's library.

The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume
siftlode_files); docs/self-hosting.md says how to carry an old volume over.
Migration 0102 places every existing job and carries each done file's expiry
over from its asset.

safe_abs_path takes a missing path and answers None, which drops one baselined
type error.
This commit is contained in:
2026-10-11 03:30:34 +02:00
parent 07e0990ecf
commit bf99444ded
49 changed files with 984 additions and 226 deletions
+22 -31
View File
@@ -3,7 +3,7 @@
import re
import shutil
import time
from datetime import date, datetime, timedelta, timezone
from datetime import date, datetime, timezone
from pathlib import Path
import yt_dlp
@@ -434,14 +434,12 @@ def _extract_with_fallback(
raise last_exc # unreachable (loop either returns or raises)
def _download_settings() -> tuple[str, int]:
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
def _download_layout() -> str:
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
with SessionLocal() as db:
return (
sysconfig.get_str(db, "download_layout"),
sysconfig.get_int(db, "download_retention_days"),
)
return sysconfig.get_str(db, "download_layout")
def _ensure_phone_safe(
@@ -588,19 +586,18 @@ def _download(
thumbnail_url=info.get("thumbnail"),
)
layout, retention_days = _download_settings()
layout = _download_layout()
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
thumb = _find_thumbnail(staging, produced)
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
storage.place_file(produced, settings.download_root, rel)
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
storage.place_file(produced, settings.files_root, rel)
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
final = storage.abs_path(settings.download_root, rel)
final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -626,12 +623,11 @@ def _download(
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.source_webpage_url = info.get("webpage_url")
asset.nfo_written = nfo_ok
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
owner_id = job.user_id if job else None
@@ -840,7 +836,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
if not src_ready or not src_rel:
_fail_edit(job_id, asset_id, "The source download is no longer available.")
return
src_path = storage.abs_path(settings.download_root, src_rel)
src_path = storage.abs_path(settings.files_root, src_rel)
if not src_path.exists():
_fail_edit(job_id, asset_id, "The source file is missing.")
return
@@ -887,13 +883,11 @@ def _process_edit(job_id: int, asset_id: int) -> None:
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
storage.place_file(dest_staging, settings.download_root, rel)
final = storage.abs_path(settings.download_root, rel)
storage.place_file(dest_staging, settings.files_root, rel)
final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
_, _, out_w, out_h, _ = probe.probe_media(final)
_, retention_days = _download_settings()
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -912,7 +906,6 @@ def _process_edit(job_id: int, asset_id: int) -> None:
asset.width, asset.height = src_w, src_h
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
owner_id = job.user_id if job else None
@@ -1086,10 +1079,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
ext = out.suffix.lstrip(".").lower() or "mp4"
rel = storage.edit_rel_path(user_id, asset_id, ext)
storage.place_file(out, settings.download_root, rel)
final = storage.abs_path(settings.download_root, rel)
storage.place_file(out, settings.files_root, rel)
final = storage.abs_path(settings.files_root, rel)
vcodec, acodec, width, height, _ = probe.probe_media(final)
_, retention_days = _download_settings()
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -1108,10 +1100,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
asset.media_kind = mediakind.derive(vcodec, acodec)
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
# item has no public thumbnail at all.
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
placeholder = service.clip_title(None, None)
@@ -1230,7 +1221,7 @@ def _free_unheld_upload(asset_id: int) -> bool:
if asset is None or asset.ref_count != 0:
return False
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
db.commit()
return True
@@ -1253,7 +1244,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
duration = asset.duration_s if asset else None
job = db.get(DownloadJob, job_id)
owner_id = job.user_id if job else None
src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
if old_rel is None or src is None or not src.exists():
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
return
@@ -1267,10 +1258,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
new_rel = storage.with_ext(old_rel, old_ext, ext)
if safe != src:
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
storage.place_file(safe, settings.download_root, new_rel)
storage.place_file(safe, settings.files_root, new_rel)
if new_rel != old_rel:
placed = storage.abs_path(settings.download_root, new_rel)
final = storage.abs_path(settings.download_root, new_rel)
placed = storage.abs_path(settings.files_root, new_rel)
final = storage.abs_path(settings.files_root, new_rel)
vcodec, acodec, width, height, dur = probe.probe_media(final)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)