feat(files): every file sits in a library folder, and keeps its own expiry
The Files library starts here. A user's every download, clip and upload now lands in a source folder made per origin (YouTube, Plex clips, uploads, one per other site), decided once in downloads/folders.py at every place a job is made. How long a file stays moves from the shared asset to the user's job: stamped from its folder when it is done, never for a file the user keeps. The GC warns each user once, then takes the expired file out of their library; the cache cap never evicts a kept file, and an evicted one leaves every holder's library. The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume siftlode_files); docs/self-hosting.md says how to carry an old volume over. Migration 0102 places every existing job and carries each done file's expiry over from its asset. safe_abs_path takes a missing path and answers None, which drops one baselined type error.
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
@@ -434,14 +434,12 @@ def _extract_with_fallback(
|
||||
raise last_exc # unreachable (loop either returns or raises)
|
||||
|
||||
|
||||
def _download_settings() -> tuple[str, int]:
|
||||
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
|
||||
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
|
||||
def _download_layout() -> str:
|
||||
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
|
||||
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
|
||||
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
|
||||
with SessionLocal() as db:
|
||||
return (
|
||||
sysconfig.get_str(db, "download_layout"),
|
||||
sysconfig.get_int(db, "download_retention_days"),
|
||||
)
|
||||
return sysconfig.get_str(db, "download_layout")
|
||||
|
||||
|
||||
def _ensure_phone_safe(
|
||||
@@ -588,19 +586,18 @@ def _download(
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
|
||||
layout, retention_days = _download_settings()
|
||||
layout = _download_layout()
|
||||
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
||||
thumb = _find_thumbnail(staging, produced)
|
||||
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
||||
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
||||
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
||||
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
||||
storage.place_file(produced, settings.download_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
|
||||
storage.place_file(produced, settings.files_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
|
||||
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -626,12 +623,11 @@ def _download(
|
||||
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
||||
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
||||
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.source_webpage_url = info.get("webpage_url")
|
||||
asset.nfo_written = nfo_ok
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -840,7 +836,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
if not src_ready or not src_rel:
|
||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||
return
|
||||
src_path = storage.abs_path(settings.download_root, src_rel)
|
||||
src_path = storage.abs_path(settings.files_root, src_rel)
|
||||
if not src_path.exists():
|
||||
_fail_edit(job_id, asset_id, "The source file is missing.")
|
||||
return
|
||||
@@ -887,13 +883,11 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
storage.place_file(dest_staging, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -912,7 +906,6 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.width, asset.height = src_w, src_h
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -1086,10 +1079,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
ext = out.suffix.lstrip(".").lower() or "mp4"
|
||||
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
||||
storage.place_file(out, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
storage.place_file(out, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -1108,10 +1100,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
asset.media_kind = mediakind.derive(vcodec, acodec)
|
||||
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
||||
# item has no public thumbnail at all.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
|
||||
placeholder = service.clip_title(None, None)
|
||||
@@ -1230,7 +1221,7 @@ def _free_unheld_upload(asset_id: int) -> bool:
|
||||
if asset is None or asset.ref_count != 0:
|
||||
return False
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
db.commit()
|
||||
return True
|
||||
@@ -1253,7 +1244,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
duration = asset.duration_s if asset else None
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
|
||||
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
|
||||
if old_rel is None or src is None or not src.exists():
|
||||
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||
return
|
||||
@@ -1267,10 +1258,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||
if safe != src:
|
||||
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||
storage.place_file(safe, settings.download_root, new_rel)
|
||||
storage.place_file(safe, settings.files_root, new_rel)
|
||||
if new_rel != old_rel:
|
||||
placed = storage.abs_path(settings.download_root, new_rel)
|
||||
final = storage.abs_path(settings.download_root, new_rel)
|
||||
placed = storage.abs_path(settings.files_root, new_rel)
|
||||
final = storage.abs_path(settings.files_root, new_rel)
|
||||
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
|
||||
Reference in New Issue
Block a user