feat(files): every file sits in a library folder, and keeps its own expiry
The Files library starts here. A user's every download, clip and upload now lands in a source folder made per origin (YouTube, Plex clips, uploads, one per other site), decided once in downloads/folders.py at every place a job is made. How long a file stays moves from the shared asset to the user's job: stamped from its folder when it is done, never for a file the user keeps. The GC warns each user once, then takes the expired file out of their library; the cache cap never evicts a kept file, and an evicted one leaves every holder's library. The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume siftlode_files); docs/self-hosting.md says how to carry an old volume over. Migration 0102 places every existing job and carries each done file's expiry over from its asset. safe_abs_path takes a missing path and answers None, which drops one baselined type error.
This commit is contained in:
@@ -173,7 +173,7 @@ class Settings(BaseSettings):
|
||||
# --- Download center (yt-dlp) ---
|
||||
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
||||
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
||||
download_root: str = "/downloads"
|
||||
files_root: str = "/files"
|
||||
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
||||
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
||||
worker_enabled: bool = False
|
||||
@@ -345,8 +345,8 @@ class Settings(BaseSettings):
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
# dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
|
||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
|
||||
# dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point
|
||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast
|
||||
# local disk, so the default is fine there.
|
||||
plex_hls_dir: str = ""
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ def run() -> dict:
|
||||
if a.poster_path is None:
|
||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||
rel = editmod.ensure_poster(a, settings.download_root)
|
||||
rel = editmod.ensure_poster(a, settings.files_root)
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
|
||||
@@ -732,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
return dest if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None:
|
||||
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
|
||||
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
|
||||
`<base>.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
|
||||
@@ -740,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
short clip) dodges black intro frames."""
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
src = root / asset.rel_path
|
||||
if not src.exists():
|
||||
return None
|
||||
@@ -756,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
return rel if dest.exists() else None
|
||||
|
||||
|
||||
def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
|
||||
def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None:
|
||||
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
|
||||
|
||||
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
|
||||
@@ -768,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
if not asset.rel_path:
|
||||
return None
|
||||
geom = storyboard_geometry(asset.duration_s)
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
rel = f".storyboards/asset-{asset.id}.jpg"
|
||||
dest = root / rel
|
||||
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
"""The folders of the Files library — where each of a user's files sits.
|
||||
|
||||
A folder is a row, never a directory: on disk a file stays where the download layout put it (one
|
||||
shared asset can be held by several users), so a rename or a move is a column update. Two kinds:
|
||||
|
||||
- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
|
||||
uploads, one per other site). Its `source_key` names the origin; the client renders its name
|
||||
from the key, in the reader's language.
|
||||
- an OWN folder is the user's, with the name they gave it.
|
||||
|
||||
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
|
||||
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
|
||||
|
||||
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
|
||||
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
|
||||
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
|
||||
kept it.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
|
||||
class FolderKind(StrEnum):
|
||||
SOURCE = "source"
|
||||
OWN = "own"
|
||||
|
||||
|
||||
class SourceKey(StrEnum):
|
||||
"""The fixed origins. Any other site is `site:<host>` (see `site_key`)."""
|
||||
|
||||
YOUTUBE = "youtube"
|
||||
PLEX = "plex"
|
||||
UPLOAD = "upload"
|
||||
|
||||
|
||||
SITE_PREFIX = "site:"
|
||||
|
||||
# A host's leading label that only says "the same site, another front door".
|
||||
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||
|
||||
|
||||
def site_key(url: str) -> str:
|
||||
"""The source key of a page on some site: `site:<host>` without its front door (www., m.),
|
||||
or the YouTube key for a YouTube address that arrived as a plain URL."""
|
||||
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||
for door in _FRONT_DOORS:
|
||||
if host.startswith(door) and host.count(".") > 1:
|
||||
host = host[len(door) :]
|
||||
break
|
||||
if host in _YOUTUBE_HOSTS:
|
||||
return SourceKey.YOUTUBE
|
||||
return f"{SITE_PREFIX}{host or 'unknown'}"
|
||||
|
||||
|
||||
def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
"""The origin a job's file came from — the source folder it belongs in.
|
||||
|
||||
youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL →
|
||||
its site; an edit → wherever its parent's file came from (its parent job, or, once that is
|
||||
deleted, the parent's asset)."""
|
||||
kind = job.source_kind
|
||||
if kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if kind == "clip":
|
||||
return (
|
||||
SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
|
||||
)
|
||||
if kind == "url":
|
||||
return site_key(job.source_ref)
|
||||
if kind == "edit":
|
||||
parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
|
||||
if parent is not None and parent.id != job.id:
|
||||
return source_key_for(db, parent)
|
||||
asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
|
||||
if asset is not None:
|
||||
if asset.source_kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if asset.source_kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if asset.source_kind == "url":
|
||||
return site_key(asset.source_ref)
|
||||
return SourceKey.UPLOAD
|
||||
|
||||
|
||||
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
retention = (
|
||||
None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
|
||||
)
|
||||
folder = FileFolder(
|
||||
user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
|
||||
)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race the savepoint exists for — another request made
|
||||
# the same folder first; re-selecting it is the whole handling.
|
||||
except IntegrityError:
|
||||
folder = db.execute(stmt).scalar_one()
|
||||
return folder
|
||||
|
||||
|
||||
def place(db: Session, job: DownloadJob) -> None:
|
||||
"""Put a new job's file in its source folder. Called wherever a job is created; the job must
|
||||
carry its `user_id`, `source_kind` and source refs already."""
|
||||
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||
|
||||
|
||||
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
|
||||
"""When a file that finishes now goes: its folder's retention from now, or never (kept, or a
|
||||
folder that keeps files)."""
|
||||
if kept or folder_id is None:
|
||||
return None
|
||||
days = db.execute(select(FileFolder.retention_days).where(FileFolder.id == folder_id)).scalar()
|
||||
return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
|
||||
|
||||
|
||||
def stamp_expiry(db: Session, job: DownloadJob) -> None:
|
||||
"""Stamp a job that just became done (in this session) with its expiry."""
|
||||
job.expires_at = done_expiry(db, job.folder_id, job.kept)
|
||||
job.expiry_warned = False
|
||||
+77
-58
@@ -1,12 +1,13 @@
|
||||
"""Download retention garbage collector (a scheduler job).
|
||||
|
||||
Five passes each run:
|
||||
1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
|
||||
once (gc_notified flag) so a download isn't silently deleted.
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets (never an upload) until back under it.
|
||||
1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
|
||||
within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
|
||||
2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
|
||||
released, so the shared file goes with its last holder.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
|
||||
assets (never an upload, never one a user kept) until back under it; each holder's file leaves
|
||||
their library, with a notice.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
@@ -14,7 +15,7 @@ Five passes each run:
|
||||
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||
so an in-flight download is never touched.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -26,7 +27,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import quota, storage
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
@@ -37,42 +39,39 @@ _UPLOAD_TREE = ".uploads"
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
def _holders(db: Session, asset_id: int) -> list[int]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
|
||||
).scalars()
|
||||
def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
|
||||
title = job.display_name or (asset.title if asset else None) or "download"
|
||||
create_notification(
|
||||
db,
|
||||
user_id=job.user_id,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={
|
||||
"kind": ntype,
|
||||
"title": title,
|
||||
"video_id": asset.source_ref if asset else None,
|
||||
"job_id": job.id,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
|
||||
|
||||
def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
title = asset.title or "download"
|
||||
for uid in _holders(db, asset.id):
|
||||
create_notification(
|
||||
db,
|
||||
user_id=uid,
|
||||
type=ntype,
|
||||
title=title, # English fallback; the client renders a translated message from data
|
||||
data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
|
||||
commit=False,
|
||||
)
|
||||
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||
_notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
|
||||
service.release_asset(db, job)
|
||||
db.delete(job)
|
||||
|
||||
|
||||
def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
_notify_holders(db, asset, ntype)
|
||||
db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
|
||||
|
||||
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
def _sweep_orphan_files(db: Session, files_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||
NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
root = Path(files_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
@@ -151,47 +150,52 @@ def run_download_gc(db: Session) -> dict:
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
warned = expired = evicted = reaped = 0
|
||||
|
||||
# 1. Pre-expiry warnings.
|
||||
# 1. Pre-expiry warnings — one per file, the first run inside the grace window.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.gc_notified.is_(False),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at > now,
|
||||
MediaAsset.expires_at <= warn_before,
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at > now,
|
||||
DownloadJob.expires_at <= warn_before,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in soon:
|
||||
_notify_holders(db, asset, "download_expiring")
|
||||
asset.gc_notified = True
|
||||
for job in soon:
|
||||
_notify(
|
||||
db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
|
||||
)
|
||||
job.expiry_warned = True
|
||||
warned += 1
|
||||
if soon:
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
|
||||
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||
dead = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.status.in_(("ready", "error")),
|
||||
MediaAsset.expires_at.is_not(None),
|
||||
MediaAsset.expires_at <= now,
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at <= now,
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in dead:
|
||||
_delete_asset(db, asset, "download_expired")
|
||||
for job in dead:
|
||||
_drop_job(db, job, "download_expired")
|
||||
expired += 1
|
||||
if dead:
|
||||
db.commit()
|
||||
|
||||
# 3. LRU eviction over the total cache cap.
|
||||
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
|
||||
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = (
|
||||
@@ -201,12 +205,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
# An upload counts toward the total (it is disk like any other) but is never evicted:
|
||||
# it has no source to fetch again, and it lives until its owner deletes it.
|
||||
kept_assets = select(DownloadJob.asset_id).where(
|
||||
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
|
||||
)
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
|
||||
.where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.source_kind != "upload",
|
||||
MediaAsset.id.not_in(kept_assets),
|
||||
)
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
@@ -216,7 +225,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
if total <= cap:
|
||||
break
|
||||
total -= asset.size_bytes or 0
|
||||
_delete_asset(db, asset, "download_evicted")
|
||||
holders = (
|
||||
db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in holders:
|
||||
_notify(db, job, asset, "download_evicted")
|
||||
db.delete(job)
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
evicted += 1
|
||||
db.commit()
|
||||
|
||||
@@ -257,7 +276,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
)
|
||||
for asset in unheld:
|
||||
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
reaped += 1
|
||||
if unheld:
|
||||
@@ -265,7 +284,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
|
||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
swept = _sweep_orphan_files(db, settings.files_root)
|
||||
|
||||
result = {
|
||||
"warned": warned,
|
||||
|
||||
@@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None:
|
||||
|
||||
|
||||
def _staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
|
||||
return Path(settings.files_root) / ".staging" / f"asset-{asset_id}"
|
||||
|
||||
|
||||
def _edit_staging_dir(asset_id: int) -> Path:
|
||||
return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
|
||||
return Path(settings.files_root) / ".staging" / f"edit-{asset_id}"
|
||||
|
||||
@@ -135,7 +135,7 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
||||
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
|
||||
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
|
||||
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
|
||||
poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
image_dims = _jpeg_size(poster_abs) if poster_abs else None
|
||||
if image_dims:
|
||||
tags.append(_tag("og:image:width", str(image_dims[0])))
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
@@ -434,14 +434,12 @@ def _extract_with_fallback(
|
||||
raise last_exc # unreachable (loop either returns or raises)
|
||||
|
||||
|
||||
def _download_settings() -> tuple[str, int]:
|
||||
"""Layout + retention days from the DB config (admin-overridable on the Configuration page),
|
||||
falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
|
||||
def _download_layout() -> str:
|
||||
"""The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
|
||||
back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
|
||||
a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
|
||||
with SessionLocal() as db:
|
||||
return (
|
||||
sysconfig.get_str(db, "download_layout"),
|
||||
sysconfig.get_int(db, "download_retention_days"),
|
||||
)
|
||||
return sysconfig.get_str(db, "download_layout")
|
||||
|
||||
|
||||
def _ensure_phone_safe(
|
||||
@@ -588,19 +586,18 @@ def _download(
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
|
||||
layout, retention_days = _download_settings()
|
||||
layout = _download_layout()
|
||||
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
|
||||
thumb = _find_thumbnail(staging, produced)
|
||||
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
|
||||
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
|
||||
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
|
||||
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
|
||||
storage.place_file(produced, settings.download_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
|
||||
storage.place_file(produced, settings.files_root, rel)
|
||||
nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
|
||||
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -626,12 +623,11 @@ def _download(
|
||||
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
|
||||
# and can't be relied on cross-origin. write_sidecars already wrote <base>.jpg from
|
||||
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.source_webpage_url = info.get("webpage_url")
|
||||
asset.nfo_written = nfo_ok
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -840,7 +836,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
if not src_ready or not src_rel:
|
||||
_fail_edit(job_id, asset_id, "The source download is no longer available.")
|
||||
return
|
||||
src_path = storage.abs_path(settings.download_root, src_rel)
|
||||
src_path = storage.abs_path(settings.files_root, src_rel)
|
||||
if not src_path.exists():
|
||||
_fail_edit(job_id, asset_id, "The source file is missing.")
|
||||
return
|
||||
@@ -887,13 +883,11 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
storage.place_file(dest_staging, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -912,7 +906,6 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.width, asset.height = src_w, src_h
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||
owner_id = job.user_id if job else None
|
||||
@@ -1086,10 +1079,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
|
||||
ext = out.suffix.lstrip(".").lower() or "mp4"
|
||||
rel = storage.edit_rel_path(user_id, asset_id, ext)
|
||||
storage.place_file(out, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
storage.place_file(out, settings.files_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
vcodec, acodec, width, height, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
@@ -1108,10 +1100,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
asset.media_kind = mediakind.derive(vcodec, acodec)
|
||||
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
|
||||
# item has no public thumbnail at all.
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
|
||||
placeholder = service.clip_title(None, None)
|
||||
@@ -1230,7 +1221,7 @@ def _free_unheld_upload(asset_id: int) -> bool:
|
||||
if asset is None or asset.ref_count != 0:
|
||||
return False
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
db.commit()
|
||||
return True
|
||||
@@ -1253,7 +1244,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
duration = asset.duration_s if asset else None
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
|
||||
src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
|
||||
if old_rel is None or src is None or not src.exists():
|
||||
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||
return
|
||||
@@ -1267,10 +1258,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||
if safe != src:
|
||||
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||
storage.place_file(safe, settings.download_root, new_rel)
|
||||
storage.place_file(safe, settings.files_root, new_rel)
|
||||
if new_rel != old_rel:
|
||||
placed = storage.abs_path(settings.download_root, new_rel)
|
||||
final = storage.abs_path(settings.download_root, new_rel)
|
||||
placed = storage.abs_path(settings.files_root, new_rel)
|
||||
final = storage.abs_path(settings.files_root, new_rel)
|
||||
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
|
||||
@@ -19,7 +19,7 @@ from app.downloads import adapters
|
||||
from app.downloads import clip as clipmod
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads import folders, formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import (
|
||||
DISPLAY_TEXT_MAX,
|
||||
@@ -198,12 +198,14 @@ def enqueue(
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
# Cache hit: the file already exists — no worker round-trip needed.
|
||||
if asset.status == "ready":
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_expiry(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
@@ -277,11 +279,13 @@ def enqueue_edit(
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
if asset.status == "ready": # identical clip already produced for this user → instant
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_expiry(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
@@ -364,11 +368,13 @@ def enqueue_clip(
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
incr_ref_count(db, asset.id)
|
||||
|
||||
if asset.status == "ready": # the same range was already cut for this user → instant
|
||||
job.status = JobStatus.DONE
|
||||
job.progress = 100
|
||||
folders.stamp_expiry(db, job)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
|
||||
db.commit()
|
||||
@@ -392,6 +398,17 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
||||
if not fields:
|
||||
return False
|
||||
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
|
||||
if fields.get("status") == JobStatus.DONE:
|
||||
# Done is when a file's retention starts: its folder's, unless the user kept it.
|
||||
held = db.execute(
|
||||
select(DownloadJob.folder_id, DownloadJob.kept).where(DownloadJob.id == job_id)
|
||||
).one_or_none()
|
||||
if held is not None:
|
||||
fields = {
|
||||
**fields,
|
||||
"expires_at": folders.done_expiry(db, held.folder_id, held.kept),
|
||||
"expiry_warned": False,
|
||||
}
|
||||
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
|
||||
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
|
||||
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
|
||||
@@ -436,7 +453,7 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
)
|
||||
if new_count == 0 and freeable:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
|
||||
|
||||
@@ -565,4 +582,6 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
# name would survive the rebuild).
|
||||
job.display_name = None
|
||||
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
|
||||
job.expires_at = None # stamped again when the rebuild is done
|
||||
job.expiry_warned = False
|
||||
db.commit()
|
||||
|
||||
@@ -122,7 +122,7 @@ def display_filename(name: str) -> str:
|
||||
|
||||
|
||||
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
|
||||
"""Path of the media file relative to DOWNLOAD_ROOT (forward slashes).
|
||||
"""Path of the media file relative to FILES_ROOT (forward slashes).
|
||||
|
||||
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
|
||||
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
|
||||
@@ -144,7 +144,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No
|
||||
|
||||
|
||||
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative.
|
||||
"""On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative.
|
||||
|
||||
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
|
||||
@@ -161,7 +161,7 @@ def _clean_upload_ext(filename: str) -> str:
|
||||
|
||||
|
||||
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
||||
"""On-disk path (relative to FILES_ROOT) for a locally-uploaded file.
|
||||
|
||||
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||
@@ -250,24 +250,27 @@ def download_filename(display_name: str, container: str | None, path: Path) -> s
|
||||
return f"{base}.{ext}"
|
||||
|
||||
|
||||
def abs_path(download_root: str, rel: str) -> Path:
|
||||
return Path(download_root) / rel
|
||||
def abs_path(files_root: str, rel: str) -> Path:
|
||||
return Path(files_root) / rel
|
||||
|
||||
|
||||
def safe_abs_path(download_root: str, rel: str) -> Path | None:
|
||||
"""The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on
|
||||
disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint
|
||||
shares, so the containment check lives in exactly one place."""
|
||||
root = Path(download_root).resolve()
|
||||
path = abs_path(download_root, rel).resolve()
|
||||
def safe_abs_path(files_root: str, rel: str | None) -> Path | None:
|
||||
"""The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on
|
||||
disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve).
|
||||
Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the
|
||||
containment check lives in exactly one place."""
|
||||
if not rel:
|
||||
return None
|
||||
root = Path(files_root).resolve()
|
||||
path = abs_path(files_root, rel).resolve()
|
||||
if root not in path.parents or not path.exists():
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def place_file(staging_file: Path, download_root: str, rel: str) -> None:
|
||||
"""Move a completed staging file into its final location under DOWNLOAD_ROOT."""
|
||||
dest = abs_path(download_root, rel)
|
||||
def place_file(staging_file: Path, files_root: str, rel: str) -> None:
|
||||
"""Move a completed staging file into its final location under FILES_ROOT."""
|
||||
dest = abs_path(files_root, rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(staging_file), str(dest))
|
||||
|
||||
@@ -301,9 +304,9 @@ def _sidecar(media: Path, ext: str) -> Path:
|
||||
return media.with_suffix(ext)
|
||||
|
||||
|
||||
def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||
def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
media = abs_path(files_root, rel)
|
||||
try:
|
||||
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
if thumb_src and thumb_src.exists():
|
||||
@@ -319,9 +322,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
||||
return False
|
||||
|
||||
|
||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
def delete_asset_files(files_root: str, rel: str) -> None:
|
||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||
media = abs_path(download_root, rel)
|
||||
media = abs_path(files_root, rel)
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
@@ -335,7 +338,7 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||
)
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
root = Path(files_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
|
||||
@@ -33,7 +33,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads import folders, mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobKind, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
@@ -81,14 +81,14 @@ def ingest_upload(
|
||||
db.flush() # need asset.id for the on-disk path
|
||||
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
storage.place_file(src_path, settings.files_root, rel)
|
||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
|
||||
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
|
||||
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
|
||||
asset_id = asset.id
|
||||
try:
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
final = storage.abs_path(settings.files_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "pending" if convert else "ready"
|
||||
@@ -108,7 +108,7 @@ def ingest_upload(
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
||||
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
||||
# server side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
@@ -123,6 +123,9 @@ def ingest_upload(
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
folders.place(db, job)
|
||||
if not convert:
|
||||
folders.stamp_expiry(db, job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
except Exception:
|
||||
@@ -139,7 +142,7 @@ def ingest_upload(
|
||||
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
||||
# when none was generated (audio/file, or a video whose poster never started).
|
||||
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
||||
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
|
||||
storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True)
|
||||
raise
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
+51
-8
@@ -18,6 +18,7 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
@@ -826,12 +827,13 @@ class MediaAsset(Base, TimestampMixin):
|
||||
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
|
||||
shared; per-user edited derivatives (phase 2) are never cached here.
|
||||
|
||||
`rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
|
||||
`rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
|
||||
system-decided and bound to the source id — never renamed (the user's custom name is display
|
||||
metadata on `download_job`, applied only when downloading to their own machine). The naming
|
||||
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
|
||||
plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
|
||||
uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
|
||||
the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
|
||||
stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
|
||||
|
||||
__tablename__ = "media_assets"
|
||||
__table_args__ = (
|
||||
@@ -847,10 +849,10 @@ class MediaAsset(Base, TimestampMixin):
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), default="pending", server_default="pending", index=True
|
||||
) # "pending" | "ready" | "error"
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
|
||||
rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
|
||||
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
|
||||
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
|
||||
# card / watch page / link preview can still show an image. Relative to download_root.
|
||||
# card / watch page / link preview can still show an image. Relative to files_root.
|
||||
poster_path: Mapped[str | None] = mapped_column(Text)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
duration_s: Mapped[int | None] = mapped_column(Integer)
|
||||
@@ -866,8 +868,6 @@ class MediaAsset(Base, TimestampMixin):
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
||||
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
@@ -883,7 +883,6 @@ class MediaAsset(Base, TimestampMixin):
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class DownloadProfile(Base, TimestampMixin):
|
||||
@@ -907,6 +906,40 @@ class DownloadProfile(Base, TimestampMixin):
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class FileFolder(Base, TimestampMixin):
|
||||
"""A folder of the Files library (`app.downloads.folders`): a row, never a directory.
|
||||
|
||||
A SOURCE folder (`source_key` set: youtube / plex / upload / site:<host>) is made the first time a
|
||||
user gets a file from that origin; an OWN folder (`name` set) is one the user made. Every file —
|
||||
a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
|
||||
file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
|
||||
|
||||
__tablename__ = "file_folders"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
|
||||
Index(
|
||||
"uq_file_folder_own_name",
|
||||
"user_id",
|
||||
func.lower(text("name")),
|
||||
unique=True,
|
||||
postgresql_where=text("name IS NOT NULL"),
|
||||
),
|
||||
CheckConstraint(
|
||||
"(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
|
||||
" OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
|
||||
name="ck_file_folder_kind",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
|
||||
# "site:" + a host name (DNS caps one at 253 characters).
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
@@ -974,6 +1007,16 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
|
||||
# folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
|
||||
# the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
|
||||
# never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
|
||||
class DownloadShare(Base, TimestampMixin):
|
||||
|
||||
@@ -10,4 +10,4 @@ from pathlib import Path
|
||||
|
||||
from app.config import settings
|
||||
|
||||
CACHE_DIR = Path(settings.download_root) / ".plex-img-cache"
|
||||
CACHE_DIR = Path(settings.files_root) / ".plex-img-cache"
|
||||
|
||||
@@ -41,7 +41,7 @@ log = logs.get_logger(__name__)
|
||||
_HLS_ROOT = (
|
||||
Path(settings.plex_hls_dir)
|
||||
if settings.plex_hls_dir
|
||||
else Path(settings.download_root) / ".plex-hls"
|
||||
else Path(settings.files_root) / ".plex-hls"
|
||||
)
|
||||
_SEG_SECONDS = 6
|
||||
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
|
||||
|
||||
@@ -145,6 +145,10 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"edit_spec": job.edit_spec,
|
||||
"can_download": ready and job.status == JobStatus.DONE,
|
||||
"expired": job.asset_id is None and job.status == JobStatus.DONE,
|
||||
# The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
|
||||
"folder_id": job.folder_id,
|
||||
"kept": job.kept,
|
||||
"expires_at": iso(job.expires_at),
|
||||
"asset": None
|
||||
if asset is None
|
||||
else {
|
||||
@@ -163,7 +167,6 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"media_kind": asset.media_kind,
|
||||
"uploader_url": asset.uploader_url,
|
||||
"thumbnail_url": asset.thumbnail_url,
|
||||
"expires_at": iso(asset.expires_at),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -495,7 +498,7 @@ async def spool_upload(
|
||||
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
|
||||
# check a connection straight back out.
|
||||
release_read_connection(db)
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging = Path(settings.files_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
@@ -871,7 +874,7 @@ def download_file(
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise not_found("file")
|
||||
|
||||
@@ -946,7 +949,7 @@ def storyboard_meta(
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.download_root)
|
||||
meta = editmod.ensure_storyboard(asset, settings.files_root)
|
||||
if meta is None:
|
||||
return {"available": False}
|
||||
db.commit() # persist storyboard_path if it was just generated
|
||||
@@ -967,7 +970,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
@@ -981,7 +984,7 @@ def storyboard_image(
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
|
||||
if path is None:
|
||||
raise not_found("filmstrip")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
@@ -73,7 +73,7 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
|
||||
raise refused(404, "video_gone")
|
||||
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
||||
# loads a player for a missing file — meta and file agree.
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise refused(404, "video_gone")
|
||||
return job, asset
|
||||
@@ -158,7 +158,7 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise refused(403, "share_password_required")
|
||||
asset = _ready_asset(db, link)
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise refused(404, "video_gone")
|
||||
|
||||
@@ -194,7 +194,7 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)
|
||||
job, asset = _ready_target(db, link)
|
||||
if not asset.poster_path:
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
path = storage.safe_abs_path(settings.files_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
@@ -715,7 +715,7 @@ def main() -> None:
|
||||
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
|
||||
threads: dict[int, threading.Thread] = {}
|
||||
level = _sync_knobs(threads, None)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root)
|
||||
log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.files_root)
|
||||
while not _stop.wait(_KNOB_SECONDS):
|
||||
level = _sync_knobs(threads, level)
|
||||
log.info("Download worker stopping…")
|
||||
|
||||
Reference in New Issue
Block a user