feat(files): every file sits in a library folder, and keeps its own expiry
The Files library starts here. A user's every download, clip and upload now lands in a source folder made per origin (YouTube, Plex clips, uploads, one per other site), decided once in downloads/folders.py at every place a job is made. How long a file stays moves from the shared asset to the user's job: stamped from its folder when it is done, never for a file the user keeps. The GC warns each user once, then takes the expired file out of their library; the cache cap never evicts a kept file, and an evicted one leaves every holder's library. The media mount is /files now (FILES_ROOT, FILES_HOST_PATH, volume siftlode_files); docs/self-hosting.md says how to carry an old volume over. Migration 0102 places every existing job and carries each done file's expiry over from its asset. safe_abs_path takes a missing path and answers None, which drops one baselined type error.
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
"""The folders of the Files library — where each of a user's files sits.
|
||||
|
||||
A folder is a row, never a directory: on disk a file stays where the download layout put it (one
|
||||
shared asset can be held by several users), so a rename or a move is a column update. Two kinds:
|
||||
|
||||
- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
|
||||
uploads, one per other site). Its `source_key` names the origin; the client renders its name
|
||||
from the key, in the reader's language.
|
||||
- an OWN folder is the user's, with the name they gave it.
|
||||
|
||||
Every file of a user is in exactly one folder. The folder a new file lands in is decided here
|
||||
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
|
||||
|
||||
Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
|
||||
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
|
||||
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
|
||||
kept it.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset
|
||||
|
||||
|
||||
class FolderKind(StrEnum):
|
||||
SOURCE = "source"
|
||||
OWN = "own"
|
||||
|
||||
|
||||
class SourceKey(StrEnum):
|
||||
"""The fixed origins. Any other site is `site:<host>` (see `site_key`)."""
|
||||
|
||||
YOUTUBE = "youtube"
|
||||
PLEX = "plex"
|
||||
UPLOAD = "upload"
|
||||
|
||||
|
||||
SITE_PREFIX = "site:"
|
||||
|
||||
# A host's leading label that only says "the same site, another front door".
|
||||
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
|
||||
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
|
||||
|
||||
|
||||
def site_key(url: str) -> str:
|
||||
"""The source key of a page on some site: `site:<host>` without its front door (www., m.),
|
||||
or the YouTube key for a YouTube address that arrived as a plain URL."""
|
||||
host = (urlsplit(url).hostname or "").lower().rstrip(".")
|
||||
for door in _FRONT_DOORS:
|
||||
if host.startswith(door) and host.count(".") > 1:
|
||||
host = host[len(door) :]
|
||||
break
|
||||
if host in _YOUTUBE_HOSTS:
|
||||
return SourceKey.YOUTUBE
|
||||
return f"{SITE_PREFIX}{host or 'unknown'}"
|
||||
|
||||
|
||||
def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
"""The origin a job's file came from — the source folder it belongs in.
|
||||
|
||||
youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL →
|
||||
its site; an edit → wherever its parent's file came from (its parent job, or, once that is
|
||||
deleted, the parent's asset)."""
|
||||
kind = job.source_kind
|
||||
if kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if kind == "clip":
|
||||
return (
|
||||
SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
|
||||
)
|
||||
if kind == "url":
|
||||
return site_key(job.source_ref)
|
||||
if kind == "edit":
|
||||
parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
|
||||
if parent is not None and parent.id != job.id:
|
||||
return source_key_for(db, parent)
|
||||
asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
|
||||
if asset is not None:
|
||||
if asset.source_kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
if asset.source_kind == "upload":
|
||||
return SourceKey.UPLOAD
|
||||
if asset.source_kind == "url":
|
||||
return site_key(asset.source_ref)
|
||||
return SourceKey.UPLOAD
|
||||
|
||||
|
||||
def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
"""The user's source folder for `key`, made on first use. A race on the unique (user, key) is
|
||||
settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
|
||||
stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
|
||||
folder = db.execute(stmt).scalar_one_or_none()
|
||||
if folder is not None:
|
||||
return folder
|
||||
retention = (
|
||||
None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
|
||||
)
|
||||
folder = FileFolder(
|
||||
user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
|
||||
)
|
||||
try:
|
||||
with db.begin_nested():
|
||||
db.add(folder)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race the savepoint exists for — another request made
|
||||
# the same folder first; re-selecting it is the whole handling.
|
||||
except IntegrityError:
|
||||
folder = db.execute(stmt).scalar_one()
|
||||
return folder
|
||||
|
||||
|
||||
def place(db: Session, job: DownloadJob) -> None:
|
||||
"""Put a new job's file in its source folder. Called wherever a job is created; the job must
|
||||
carry its `user_id`, `source_kind` and source refs already."""
|
||||
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||
|
||||
|
||||
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
|
||||
"""When a file that finishes now goes: its folder's retention from now, or never (kept, or a
|
||||
folder that keeps files)."""
|
||||
if kept or folder_id is None:
|
||||
return None
|
||||
days = db.execute(select(FileFolder.retention_days).where(FileFolder.id == folder_id)).scalar()
|
||||
return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
|
||||
|
||||
|
||||
def stamp_expiry(db: Session, job: DownloadJob) -> None:
|
||||
"""Stamp a job that just became done (in this session) with its expiry."""
|
||||
job.expires_at = done_expiry(db, job.folder_id, job.kept)
|
||||
job.expiry_warned = False
|
||||
Reference in New Issue
Block a user