siftlode/backend/app/downloads/gc.py

"""Download retention garbage collector (a scheduler job).

Five passes each run:
  1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
     once (gc_notified flag) so a download isn't silently deleted.
  2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
     holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
  3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
     ready assets (never an upload) until back under it.
  4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
     passes 1-3 skip them; benign row-count hygiene).
  5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
     re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file).
     It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
     so an in-flight download is never touched.

Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
"""

import time
from datetime import datetime, timedelta, timezone
from pathlib import Path

from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session

from app import logs, sysconfig
from app.config import settings
from app.downloads import quota, storage
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification

log = logs.get_logger(__name__)
# The one dot-prefixed top-level tree the orphan sweep enters. The others are not part of the asset
# layout (user clips, Plex HLS/img caches, in-progress staging, editor storyboards) and are skipped.
_UPLOAD_TREE = ".uploads"
_ORPHAN_MIN_AGE_S = 3600  # only sweep files older than this, so an in-flight download is never hit


def _holders(db: Session, asset_id: int) -> list[int]:
    return list(
        db.execute(
            select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
        ).scalars()
    )


def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
    title = asset.title or "download"
    for uid in _holders(db, asset.id):
        create_notification(
            db,
            user_id=uid,
            type=ntype,
            title=title,  # English fallback; the client renders a translated message from data
            data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
            commit=False,
        )


def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
    if asset.rel_path:
        storage.delete_asset_files(settings.download_root, asset.rel_path)
    _notify_holders(db, asset, ntype)
    db.delete(asset)  # jobs.asset_id -> NULL via FK ON DELETE SET NULL


def _sweep_orphan_files(db: Session, download_root: str) -> int:
    """Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
    owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
    channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
    row names at all. Skips the other dot-prefixed system trees and only removes files older than
    `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
    NOT owned by a surviving asset."""
    root = Path(download_root)
    if not root.is_dir():
        return 0
    keep: set[Path] = set()
    # An upload's file is on disk from ingest on, whatever its state (a queued conversion's original,
    # a failed one kept for Retry), so every upload row keeps its file — not only a ready one.
    for rel in db.execute(
        select(MediaAsset.rel_path).where(
            or_(MediaAsset.status == "ready", MediaAsset.source_kind == "upload"),
            MediaAsset.rel_path.is_not(None),
        )
    ).scalars():
        media = (root / rel).resolve()
        keep |= {media, media.with_suffix(".nfo"), media.with_suffix(".jpg")}

    cutoff = time.time() - _ORPHAN_MIN_AGE_S
    removed = 0
    # Throttle the whole-tree walk: this runs in the API process, and an unpaced rglob+stat over a
    # large download root monopolizes disk IO long enough to make web requests + playback stutter.
    # Yield the IO channel after every `batch` files touched so foreground work interleaves.
    pause_s = max(0, settings.download_gc_sweep_pause_ms) / 1000.0
    batch = max(1, settings.download_gc_sweep_batch)
    touched = 0

    def _throttle() -> None:
        nonlocal touched
        touched += 1
        if pause_s and touched % batch == 0:
            time.sleep(pause_s)

    unreclaimed = 0
    for entry in root.iterdir():
        # .edits / .plex-* / .staging / .storyboards + any dotfile — but `.uploads` is swept: a
        # conversion stopped between placing the new file and the commit that points at it (or the
        # unlink of the original after it) leaves a file no row names, and nothing else finds it.
        if entry.name.startswith(".") and entry.name != _UPLOAD_TREE:
            continue
        files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
        for f in files:
            _throttle()
            try:
                if f.resolve() in keep or f.stat().st_mtime > cutoff:
                    continue
                f.unlink()
                removed += 1
            except OSError:
                # silent-ok: counted, not logged per file — a broken mount would put one line
                # under every orphan. The total is reported once at the end of the sweep.
                unreclaimed += 1
        if entry.is_dir():  # prune emptied dirs (deepest first), then the channel dir itself
            for d in sorted(
                (p for p in entry.rglob("*") if p.is_dir()),
                key=lambda p: len(p.parts),
                reverse=True,
            ):
                try:
                    d.rmdir()
                # silent-ok: `rmdir` refusing means the directory still holds files, which is
                # the ordinary answer here — this prune only takes the ones that emptied.
                except OSError:
                    pass
            try:
                entry.rmdir()
            # silent-ok: the same prune one level up, at the channel directory.
            except OSError:
                pass
    if unreclaimed:
        log.warning(
            "some orphaned download files could not be reclaimed",
            extra=logs.event("download.gc.unreclaimed", files=unreclaimed, removed=removed),
        )
    return removed


def run_download_gc(db: Session) -> dict:
    now = datetime.now(timezone.utc)
    grace_days = sysconfig.get_int(db, "download_gc_grace_days")
    warned = expired = evicted = reaped = 0

    # 1. Pre-expiry warnings.
    warn_before = now + timedelta(days=grace_days)
    soon = (
        db.execute(
            select(MediaAsset).where(
                MediaAsset.status == "ready",
                MediaAsset.gc_notified.is_(False),
                MediaAsset.expires_at.is_not(None),
                MediaAsset.expires_at > now,
                MediaAsset.expires_at <= warn_before,
            )
        )
        .scalars()
        .all()
    )
    for asset in soon:
        _notify_holders(db, asset, "download_expiring")
        asset.gc_notified = True
        warned += 1
    if soon:
        db.commit()

    # 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
    dead = (
        db.execute(
            select(MediaAsset).where(
                MediaAsset.status.in_(("ready", "error")),
                MediaAsset.expires_at.is_not(None),
                MediaAsset.expires_at <= now,
            )
        )
        .scalars()
        .all()
    )
    for asset in dead:
        _delete_asset(db, asset, "download_expired")
        expired += 1
    if dead:
        db.commit()

    # 3. LRU eviction over the total cache cap.
    cap = sysconfig.get_int(db, "download_total_max_bytes")
    if cap > 0:
        total = (
            db.execute(
                select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
            ).scalar()
            or 0
        )
        if total > cap:
            # An upload counts toward the total (it is disk like any other) but is never evicted:
            # it has no source to fetch again, and it lives until its owner deletes it.
            candidates = (
                db.execute(
                    select(MediaAsset)
                    .where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
                    .order_by(MediaAsset.last_access_at.asc().nulls_first())
                )
                .scalars()
                .all()
            )
            for asset in candidates:
                if total <= cap:
                    break
                total -= asset.size_bytes or 0
                _delete_asset(db, asset, "download_evicted")
                evicted += 1
            db.commit()

    # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
    #    referenced errored row to pending on re-enqueue, so these only accumulate once every
    #    holder is gone; they carry no expires_at, so passes 1-3 never touch them.
    orphans = (
        db.execute(
            select(MediaAsset).where(
                MediaAsset.status == "error",
                MediaAsset.ref_count == 0,
                MediaAsset.rel_path.is_(None),
            )
        )
        .scalars()
        .all()
    )
    for asset in orphans:
        db.delete(asset)  # no file to unlink; FK SET NULL clears any stray job.asset_id
        reaped += 1
    if orphans:
        db.commit()

    # 4b. Reap unheld uploads. An upload carries no expires_at, is never evicted and never re-enqueued,
    #     so the passes above never free one that lost its last holder outside `release_asset` — a
    #     conversion that finished after its job was cancelled, or one cancelled mid-run (the asset
    #     was `downloading` then). Its file, ready or still the unconverted original, is at `rel_path`.
    unheld = (
        db.execute(
            select(MediaAsset).where(
                MediaAsset.source_kind == "upload",
                MediaAsset.ref_count == 0,
                MediaAsset.status != "downloading",
            )
        )
        .scalars()
        .all()
    )
    for asset in unheld:
        if asset.rel_path:  # no holder is left to tell, so no notice — just the file and the row
            storage.delete_asset_files(settings.download_root, asset.rel_path)
        db.delete(asset)
        reaped += 1
    if unheld:
        db.commit()

    # 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
    #    posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
    swept = _sweep_orphan_files(db, settings.download_root)

    result = {
        "warned": warned,
        "expired": expired,
        "evicted": evicted,
        "reaped": reaped,
        "swept": swept,
    }
    log.info("download_gc %s", result)
    return result