siftlode/backend/app/downloads/folders.py

"""The folders of the Files library — where each of a user's files sits.

A folder is a row, never a directory: on disk a file stays where the download layout put it (one
shared asset can be held by several users), so a rename or a move is a column update. Two kinds:

- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
  uploads, one per other site). Its `source_key` names the origin; the client renders its name
  from the key, in the reader's language.
- an OWN folder is the user's, with the name they gave it.

Every file of a user is in exactly one folder. The folder a new file lands in is decided here
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
moves only into one of the user's own folders or back to its own source folder (`move`): a YouTube
file in the TikTok folder would be a lie the folder's name tells.

Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
kept it — and its `ready_at`, the moment it finished (`stamp_done`).
"""

from datetime import datetime, timedelta, timezone
from enum import StrEnum
from urllib.parse import urlsplit

from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session

from app import sysconfig
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset


class FolderKind(StrEnum):
    SOURCE = "source"
    OWN = "own"


class SourceKey(StrEnum):
    """The fixed origins. Any other site is `site:<host>` (see `site_key`)."""

    YOUTUBE = "youtube"
    PLEX = "plex"
    UPLOAD = "upload"


SITE_PREFIX = "site:"

# A host's leading label that only says "the same site, another front door".
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}


def site_key(url: str) -> str:
    """The source key of a page on some site: `site:<host>` without its front door (www., m.),
    or the YouTube key for a YouTube address that arrived as a plain URL."""
    host = (urlsplit(url).hostname or "").lower().rstrip(".")
    for door in _FRONT_DOORS:
        if host.startswith(door) and host.count(".") > 1:
            host = host[len(door) :]
            break
    if host in _YOUTUBE_HOSTS:
        return SourceKey.YOUTUBE
    return f"{SITE_PREFIX}{host or 'unknown'}"


def source_key_for(db: Session, job: DownloadJob) -> str:
    """The origin a job's file came from — the source folder it belongs in.

    The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
    (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
    came from (its parent job, or, once that is deleted, the parent's asset)."""
    if job.source_key:
        return job.source_key
    kind = job.source_kind
    if kind == "youtube":
        return SourceKey.YOUTUBE
    if kind == "upload":
        return SourceKey.UPLOAD
    if kind == "clip":
        return (
            SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
        )
    if kind == "url":
        return site_key(job.source_ref)
    if kind == "edit":
        parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
        if parent is not None and parent.id != job.id:
            return source_key_for(db, parent)
        asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
        if asset is not None:
            if asset.source_kind == "youtube":
                return SourceKey.YOUTUBE
            if asset.source_kind == "upload":
                return SourceKey.UPLOAD
            if asset.source_kind == "url":
                return site_key(asset.source_ref)
    return SourceKey.UPLOAD


def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
    """The user's source folder for `key`, made on first use. A race on the unique (user, key) is
    settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
    stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
    folder = db.execute(stmt).scalar_one_or_none()
    if folder is not None:
        return folder
    retention = (
        None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
    )
    folder = FileFolder(
        user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
    )
    try:
        with db.begin_nested():
            db.add(folder)
            db.flush()
    # silent-ok: the expected outcome of the race the savepoint exists for — another request made
    # the same folder first; re-selecting it is the whole handling.
    except IntegrityError:
        folder = db.execute(stmt).scalar_one()
    return folder


def place(db: Session, job: DownloadJob) -> None:
    """Put a new job's file in its source folder, and record that origin on it. Called wherever a
    job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
    key = source_key_for(db, job)
    job.source_key = key
    job.folder_id = source_folder(db, job.user_id, key).id


def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
    """When a file that finishes now goes: its folder's retention from now, or never (kept, or a
    folder that keeps files)."""
    if kept or folder_id is None:
        return None
    # Read under a share lock, held to the caller's commit: a retention change (`set_retention` locks
    # the folder) waits for this file and restamps it too, or this read waits for the change and
    # takes the new value — never the old value stamped on a file the change did not see.
    days = db.execute(
        select(FileFolder.retention_days)
        .where(FileFolder.id == folder_id)
        .with_for_update(read=True)
    ).scalar()
    return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)


def stamp_done(db: Session, job: DownloadJob) -> None:
    """Stamp a job that just became done (in this session): when it did, and its expiry."""
    job.ready_at = datetime.now(timezone.utc)
    job.expires_at = done_expiry(db, job.folder_id, job.kept)
    job.expiry_warned = False


class FolderError(Exception):
    """A folder change the rules refuse; `reason` is the refusal the route answers with."""

    def __init__(self, reason: str) -> None:
        super().__init__(reason)
        self.reason = reason


def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
    """The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
    caller's transaction ends, so a delete of the folder waits for the file and then takes it home
    with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
    only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
    folder answers like a missing one."""
    stmt = select(FileFolder).where(FileFolder.id == folder_id)
    if hold:
        stmt = stmt.with_for_update(read=True)
    folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
    if folder is None or folder.user_id != user_id:
        raise FolderError("file_folder_gone")
    return folder


def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
    """The folder an upload asked to land in, if it may: one of the user's own folders, or their
    Uploads folder. A file put in a site's folder would claim an origin it does not have."""
    folder = own_folder(db, user_id, folder_id, hold=hold)
    if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
        raise FolderError("file_folder_not_its_source")
    return folder


def restamp(db: Session, job: DownloadJob) -> None:
    """Start a file's clock again from now: it moved to another folder, or its user stopped keeping
    it. A file not done yet gets its expiry when it finishes (`stamp_done`), so it has none now."""
    done = job.status == JobStatus.DONE
    job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
    job.expiry_warned = False


def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
    """Keep a file until its user deletes it, or hand it back to its folder's retention."""
    job.kept = kept
    restamp(db, job)


def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
    """Put a file in `folder` — one of its user's own folders, or the source folder it came from.
    The caller has checked that `folder` is the job's user's."""
    if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
        raise FolderError("file_folder_not_its_source")
    if folder.id == job.folder_id:
        return
    job.folder_id = folder.id
    restamp(db, job)


def set_retention(db: Session, folder: FileFolder, days: int | None) -> None:
    """Change how long `folder` keeps a file (None = until deleted). The done files already in it
    move by the difference — each keeps the age it has — but none ends up closer than the GC's
    warning window, so a shortened retention still warns before it takes anything."""
    # Wait out a file being put in it or stamped from it right now (`own_folder`, `done_expiry`), so
    # that file is among the ones restamped below. Re-read: another change may have landed first.
    db.execute(
        select(FileFolder)
        .where(FileFolder.id == folder.id)
        .with_for_update()
        .execution_options(populate_existing=True)
    )
    old = folder.retention_days
    if old == days:
        return
    folder.retention_days = days
    now = datetime.now(timezone.utc)
    floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
    # Locked as they are read: one moved out or deleted meanwhile is no longer this folder's to time.
    jobs = db.execute(
        select(DownloadJob)
        .where(
            DownloadJob.folder_id == folder.id,
            DownloadJob.status == JobStatus.DONE,
            DownloadJob.kept.is_(False),
        )
        .with_for_update()
        .execution_options(populate_existing=True)
    ).scalars()
    for job in jobs:
        if days is None:
            job.expires_at = None
        elif old is None or job.expires_at is None:
            job.expires_at = max(now + timedelta(days=days), floor)
        else:
            job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
        job.expiry_warned = False


def delete_own(db: Session, folder: FileFolder) -> int:
    """Delete one of a user's own folders. Its files are not deleted: each goes back to the source
    folder it came from, its clock starting there now. Returns how many moved."""
    if folder.kind != FolderKind.OWN:
        raise FolderError("file_folder_not_own")
    # Wait out a file being put in it right now (`own_folder`), then take that one home too.
    db.execute(select(FileFolder.id).where(FileFolder.id == folder.id).with_for_update())
    # Locked as they are read: a file moved out meanwhile stays where it went, and one deleted
    # meanwhile is not written back (its gone row would fail the delete).
    jobs = (
        db.execute(
            select(DownloadJob)
            .where(DownloadJob.folder_id == folder.id)
            .with_for_update()
            .execution_options(populate_existing=True)
        )
        .scalars()
        .all()
    )
    for job in jobs:
        job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
        restamp(db, job)
    db.flush()  # the files leave before the folder goes: its foreign key RESTRICTs the delete
    db.delete(folder)
    return len(jobs)