"""The folders of the Files library — where each of a user's files sits. A folder is a row, never a directory: on disk a file stays where the download layout put it (one shared asset can be held by several users), so a rename or a move is a column update. Two kinds: - a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips, uploads, one per other site). Its `source_key` names the origin; the client renders its name from the key, in the reader's language. - an OWN folder is the user's, with the name they gave it. Every file of a user is in exactly one folder. The folder a new file lands in is decided here (`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells. Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user kept it. """ from datetime import datetime, timedelta, timezone from enum import StrEnum from urllib.parse import urlsplit from sqlalchemy import select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app import sysconfig from app.downloads.status import JobStatus from app.models import DownloadJob, FileFolder, MediaAsset class FolderKind(StrEnum): SOURCE = "source" OWN = "own" class SourceKey(StrEnum): """The fixed origins. Any other site is `site:` (see `site_key`).""" YOUTUBE = "youtube" PLEX = "plex" UPLOAD = "upload" SITE_PREFIX = "site:" # A host's leading label that only says "the same site, another front door". _FRONT_DOORS = ("www.", "m.", "mobile.", "web.") _YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"} def site_key(url: str) -> str: """The source key of a page on some site: `site:` without its front door (www., m.), or the YouTube key for a YouTube address that arrived as a plain URL.""" host = (urlsplit(url).hostname or "").lower().rstrip(".") for door in _FRONT_DOORS: if host.startswith(door) and host.count(".") > 1: host = host[len(door) :] break if host in _YOUTUBE_HOSTS: return SourceKey.YOUTUBE return f"{SITE_PREFIX}{host or 'unknown'}" def source_key_for(db: Session, job: DownloadJob) -> str: """The origin a job's file came from — the source folder it belongs in. youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file came from (its parent job, or, once that is deleted, the parent's asset).""" kind = job.source_kind if kind == "youtube": return SourceKey.YOUTUBE if kind == "upload": return SourceKey.UPLOAD if kind == "clip": return ( SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE ) if kind == "url": return site_key(job.source_ref) if kind == "edit": parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None if parent is not None and parent.id != job.id: return source_key_for(db, parent) asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None if asset is not None: if asset.source_kind == "youtube": return SourceKey.YOUTUBE if asset.source_kind == "upload": return SourceKey.UPLOAD if asset.source_kind == "url": return site_key(asset.source_ref) return SourceKey.UPLOAD def source_folder(db: Session, user_id: int, key: str) -> FileFolder: """The user's source folder for `key`, made on first use. A race on the unique (user, key) is settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive.""" stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key) folder = db.execute(stmt).scalar_one_or_none() if folder is not None: return folder retention = ( None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days") ) folder = FileFolder( user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention ) try: with db.begin_nested(): db.add(folder) db.flush() # silent-ok: the expected outcome of the race the savepoint exists for — another request made # the same folder first; re-selecting it is the whole handling. except IntegrityError: folder = db.execute(stmt).scalar_one() return folder def place(db: Session, job: DownloadJob) -> None: """Put a new job's file in its source folder. Called wherever a job is created; the job must carry its `user_id`, `source_kind` and source refs already.""" job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None: """When a file that finishes now goes: its folder's retention from now, or never (kept, or a folder that keeps files).""" if kept or folder_id is None: return None days = db.execute(select(FileFolder.retention_days).where(FileFolder.id == folder_id)).scalar() return None if days is None else datetime.now(timezone.utc) + timedelta(days=days) def stamp_expiry(db: Session, job: DownloadJob) -> None: """Stamp a job that just became done (in this session) with its expiry.""" job.expires_at = done_expiry(db, job.folder_id, job.kept) job.expiry_warned = False class FolderError(Exception): """A folder change the rules refuse; `reason` is the refusal the route answers with.""" def __init__(self, reason: str) -> None: super().__init__(reason) self.reason = reason def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder: """The folder an upload asked to land in, if it may: one of the user's own folders, or their Uploads folder. A file put in a site's folder would claim an origin it does not have.""" folder = db.get(FileFolder, folder_id) if folder is None or folder.user_id != user_id: raise FolderError("file_folder_gone") if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: raise FolderError("file_folder_not_its_source") return folder def restamp(db: Session, job: DownloadJob) -> None: """Start a file's clock again from now: it moved to another folder, or its user stopped keeping it. A file not done yet gets its expiry when it finishes (`stamp_expiry`), so it has none now.""" done = job.status == JobStatus.DONE job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None job.expiry_warned = False def set_kept(db: Session, job: DownloadJob, kept: bool) -> None: """Keep a file until its user deletes it, or hand it back to its folder's retention.""" job.kept = kept restamp(db, job) def move(db: Session, job: DownloadJob, folder: FileFolder) -> None: """Put a file in `folder` — one of its user's own folders, or the source folder it came from. The caller has checked that `folder` is the job's user's.""" if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job): raise FolderError("file_folder_not_its_source") if folder.id == job.folder_id: return job.folder_id = folder.id restamp(db, job) def set_retention(db: Session, folder: FileFolder, days: int | None) -> None: """Change how long `folder` keeps a file (None = until deleted). The done files already in it move by the difference — each keeps the age it has — but none ends up closer than the GC's warning window, so a shortened retention still warns before it takes anything.""" old = folder.retention_days if old == days: return folder.retention_days = days now = datetime.now(timezone.utc) floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days")) jobs = db.execute( select(DownloadJob).where( DownloadJob.folder_id == folder.id, DownloadJob.status == JobStatus.DONE, DownloadJob.kept.is_(False), ) ).scalars() for job in jobs: if days is None: job.expires_at = None elif old is None or job.expires_at is None: job.expires_at = max(now + timedelta(days=days), floor) else: job.expires_at = max(job.expires_at + timedelta(days=days - old), floor) job.expiry_warned = False def delete_own(db: Session, folder: FileFolder) -> int: """Delete one of a user's own folders. Its files are not deleted: each goes back to the source folder it came from, its clock starting there now. Returns how many moved.""" if folder.kind != FolderKind.OWN: raise FolderError("file_folder_not_own") jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all() for job in jobs: job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id restamp(db, job) db.flush() # the files leave before the folder goes: its foreign key RESTRICTs the delete db.delete(folder) return len(jobs)