siftlode/backend/app/downloads/folders.py

"""The folders of the Files library — where each of a user's files sits.

A folder is a row, never a directory: on disk a file stays where the download layout put it (one
shared asset can be held by several users), so a rename or a move is a column update. Three kinds:

- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
  uploads, one per other site). Its `source_key` names the origin; the client renders its name
  from the key, in the reader's language.
- an OWN folder is the user's, with the name they gave it.
- the HUB folder (one per user, made when they first give a machine a token for it, `hub_folder`) is
  where they swap files with that machine. Its name, like a source folder's, is the client's; a file
  may be moved in like into an own folder, and it keeps a file until it is deleted.

Every file of a user is in exactly one folder. The folder a new file lands in is decided here
(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once. A file
moves only into one of the user's own folders, their Hub folder, or back to its own source folder
(`move`): a YouTube file in the TikTok folder would be a lie the folder's name tells.

Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
folder was made; the user can change it), Uploads, own folders and the Hub keep it until it is deleted. A
done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
kept it — and its `ready_at`, the moment it finished (`stamp_done`).
"""

from datetime import datetime, timedelta, timezone
from enum import StrEnum
from urllib.parse import urlsplit

from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session

from app import sysconfig
from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset


class FolderKind(StrEnum):
    SOURCE = "source"
    OWN = "own"
    HUB = "hub"


class SourceKey(StrEnum):
    """The fixed origins. Any other site is `site:<host>` (see `site_key`)."""

    YOUTUBE = "youtube"
    PLEX = "plex"
    UPLOAD = "upload"


SITE_PREFIX = "site:"

# A host's leading label that only says "the same site, another front door".
_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}


def site_key(url: str) -> str:
    """The source key of a page on some site: `site:<host>` without its front door (www., m.),
    or the YouTube key for a YouTube address that arrived as a plain URL."""
    host = (urlsplit(url).hostname or "").lower().rstrip(".")
    for door in _FRONT_DOORS:
        if host.startswith(door) and host.count(".") > 1:
            host = host[len(door) :]
            break
    if host in _YOUTUBE_HOSTS:
        return SourceKey.YOUTUBE
    return f"{SITE_PREFIX}{host or 'unknown'}"


def source_key_for(db: Session, job: DownloadJob) -> str:
    """The origin a job's file came from — the source folder it belongs in.

    The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
    (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
    came from (its parent job, or, once that is deleted, the parent's asset)."""
    if job.source_key:
        return job.source_key
    kind = job.source_kind
    if kind == "youtube":
        return SourceKey.YOUTUBE
    if kind == "upload":
        return SourceKey.UPLOAD
    if kind == "clip":
        return (
            SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
        )
    if kind == "url":
        return site_key(job.source_ref)
    if kind == "edit":
        parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
        if parent is not None and parent.id != job.id:
            return source_key_for(db, parent)
        asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
        if asset is not None:
            if asset.source_kind == "youtube":
                return SourceKey.YOUTUBE
            if asset.source_kind == "upload":
                return SourceKey.UPLOAD
            if asset.source_kind == "url":
                return site_key(asset.source_ref)
    return SourceKey.UPLOAD


def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
    """The user's source folder for `key`, made on first use. A race on the unique (user, key) is
    settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
    stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
    folder = db.execute(stmt).scalar_one_or_none()
    if folder is not None:
        return folder
    retention = (
        None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
    )
    folder = FileFolder(
        user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
    )
    try:
        with db.begin_nested():
            db.add(folder)
            db.flush()
    # silent-ok: the expected outcome of the race the savepoint exists for — another request made
    # the same folder first; re-selecting it is the whole handling.
    except IntegrityError:
        folder = db.execute(stmt).scalar_one()
    return folder


def hub_folder(db: Session, user_id: int) -> FileFolder:
    """The user's Hub folder, made on first use; a race on its one-per-user index is settled like
    `source_folder`'s."""
    stmt = select(FileFolder).where(
        FileFolder.user_id == user_id, FileFolder.kind == FolderKind.HUB
    )
    folder = db.execute(stmt).scalar_one_or_none()
    if folder is not None:
        return folder
    folder = FileFolder(user_id=user_id, kind=FolderKind.HUB)
    try:
        with db.begin_nested():
            db.add(folder)
            db.flush()
    # silent-ok: another request made the Hub folder first; re-selecting it is the whole handling.
    except IntegrityError:
        folder = db.execute(stmt).scalar_one()
    return folder


def place(db: Session, job: DownloadJob) -> None:
    """Put a new job's file in its source folder, and record that origin on it. Called wherever a
    job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
    key = source_key_for(db, job)
    job.source_key = key
    job.folder_id = source_folder(db, job.user_id, key).id


def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
    """When a file that finishes now goes: its folder's retention from now, or never (kept, or a
    folder that keeps files)."""
    if kept or folder_id is None:
        return None
    # Read under a share lock, held to the caller's commit: a retention change (`set_retention` locks
    # the folder) waits for this file and restamps it too, or this read waits for the change and
    # takes the new value — never the old value stamped on a file the change did not see.
    days = db.execute(
        select(FileFolder.retention_days)
        .where(FileFolder.id == folder_id)
        .with_for_update(read=True)
    ).scalar()
    return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)


def stamp_done(db: Session, job: DownloadJob) -> None:
    """Stamp a job that just became done (in this session): when it did, and its expiry."""
    job.ready_at = datetime.now(timezone.utc)
    job.expires_at = done_expiry(db, job.folder_id, job.kept)
    job.expiry_warned = False


class FolderError(Exception):
    """A folder change the rules refuse; `reason` is the refusal the route answers with."""

    def __init__(self, reason: str) -> None:
        super().__init__(reason)
        self.reason = reason


def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
    """The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
    caller's transaction ends, so a delete of the folder waits for the file and then takes it home
    with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
    only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
    folder answers like a missing one."""
    stmt = select(FileFolder).where(FileFolder.id == folder_id)
    if hold:
        stmt = stmt.with_for_update(read=True)
    folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
    if folder is None or folder.user_id != user_id:
        raise FolderError("file_folder_gone")
    return folder


def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
    """The folder an upload asked to land in, if it may: one of the user's own folders, their Hub
    folder, or their Uploads folder. A file put in a site's folder would claim an origin it does not
    have."""
    folder = own_folder(db, user_id, folder_id, hold=hold)
    if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
        raise FolderError("file_folder_not_its_source")
    return folder


def restamp(db: Session, job: DownloadJob) -> None:
    """Start a file's clock again from now: it moved to another folder, or its user stopped keeping
    it. A file not done yet gets its expiry when it finishes (`stamp_done`), so it has none now."""
    done = job.status == JobStatus.DONE
    job.expires_at = done_expiry(db, job.folder_id, job.kept) if done else None
    job.expiry_warned = False


def set_kept(db: Session, job: DownloadJob, kept: bool) -> None:
    """Keep a file until its user deletes it, or hand it back to its folder's retention."""
    job.kept = kept
    restamp(db, job)


def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
    """Put a file in `folder` — one of its user's own folders, their Hub folder, or the source folder
    it came from.
    The caller has checked that `folder` is the job's user's."""
    if folder.kind == FolderKind.SOURCE and folder.source_key != source_key_for(db, job):
        raise FolderError("file_folder_not_its_source")
    if folder.id == job.folder_id:
        return
    job.folder_id = folder.id
    restamp(db, job)


def _hold_for_change(db: Session, folder: FileFolder) -> None:
    """Lock `folder` FOR UPDATE and read it afresh into the same object: a change waits out a file
    being put in it or stamped from it right now (`own_folder`, `done_expiry`), and works from what
    another change that landed first left, not from what the caller read before the wait. The row is
    consumed on purpose: an ORM result refreshes the object only as its rows are fetched."""
    fresh = db.execute(
        select(FileFolder)
        .where(FileFolder.id == folder.id)
        .with_for_update()
        .execution_options(populate_existing=True)
    ).scalar_one_or_none()
    if fresh is None:
        raise FolderError("file_folder_gone")


def set_retention(db: Session, folder: FileFolder, days: int | None) -> int | None:
    """Change how long `folder` keeps a file (None = until deleted). The done files already in it
    move by the difference — each keeps the age it has — but none ends up closer than the GC's
    warning window, so a shortened retention still warns before it takes anything. Returns the
    retention it replaced (equal to `days` when nothing changed)."""
    _hold_for_change(db, folder)  # so the file arriving now is among the ones restamped below
    old = folder.retention_days
    if old == days:
        return old
    folder.retention_days = days
    now = datetime.now(timezone.utc)
    floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
    # Locked as they are read: one moved out or deleted meanwhile is no longer this folder's to time.
    jobs = db.execute(
        select(DownloadJob)
        .where(
            DownloadJob.folder_id == folder.id,
            DownloadJob.status == JobStatus.DONE,
            DownloadJob.kept.is_(False),
        )
        .with_for_update()
        .execution_options(populate_existing=True)
    ).scalars()
    for job in jobs:
        if days is None:
            job.expires_at = None
        elif old is None or job.expires_at is None:
            job.expires_at = max(now + timedelta(days=days), floor)
        else:
            job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
        job.expiry_warned = False
    return old


def delete_own(db: Session, folder: FileFolder) -> int:
    """Delete one of a user's own folders. Its files are not deleted: each goes back to the source
    folder it came from, its clock starting there now. Returns how many moved."""
    if folder.kind != FolderKind.OWN:
        raise FolderError("file_folder_not_own")
    _hold_for_change(db, folder)  # so the file arriving now is taken home too
    # Locked as they are read: a file moved out meanwhile stays where it went, and one deleted
    # meanwhile is not written back (its gone row would fail the delete).
    jobs = (
        db.execute(
            select(DownloadJob)
            .where(DownloadJob.folder_id == folder.id)
            .with_for_update()
            .execution_options(populate_existing=True)
        )
        .scalars()
        .all()
    )
    for job in jobs:
        job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
        restamp(db, job)
    db.flush()  # the files leave before the folder goes: its foreign key RESTRICTs the delete
    db.delete(folder)
    return len(jobs)