siftlode/backend/alembic/versions/0102_file_folders.py

"""The Files library: folders, and a file's retention on the user's job

Revision ID: 0102_file_folders
Revises: 0101_plex_text_plain
Create Date: 2026-10-11

Every download, clip and upload of a user now sits in a folder of their Files library
(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other
site) or one of their own. How long a file stays moves from the shared asset to each user's job
(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file
no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go.

This places every existing job in its source folder and carries each done job's expiry over from
the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from
`folders.source_key_for` as it is today — a migration must not follow the code's later changes.
"""

from urllib.parse import urlsplit

import sqlalchemy as sa
from alembic import op

revision: str = "0102_file_folders"
down_revision: str | None = "0101_plex_text_plain"
branch_labels: str | None = None
depends_on: str | None = None

_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
_DEFAULT_RETENTION_DAYS = 30  # config.download_retention_days on the day this was written


def _site_key(url: str) -> str:
    host = (urlsplit(url).hostname or "").lower().rstrip(".")
    for door in _FRONT_DOORS:
        if host.startswith(door) and host.count(".") > 1:
            host = host[len(door) :]
            break
    if host in _YOUTUBE_HOSTS:
        return "youtube"
    return f"site:{host or 'unknown'}"


def _asset_key(kind: str | None, ref: str | None) -> str:
    if kind == "youtube":
        return "youtube"
    if kind == "url" and ref:
        return _site_key(ref)
    return "upload"


def _key(job, jobs_by_id, assets) -> str:
    kind = job.source_kind
    if kind == "youtube":
        return "youtube"
    if kind == "upload":
        return "upload"
    if kind == "clip":
        spec = job.edit_spec or {}
        return "plex" if spec.get("source") == "plex" else "youtube"
    if kind == "url":
        return _site_key(job.source_ref)
    if kind == "edit":
        parent = jobs_by_id.get(job.source_job_id)
        if parent is not None and parent.id != job.id:
            return _key(parent, jobs_by_id, assets)
        asset = assets.get(job.source_asset_id)
        if asset is not None:
            return _asset_key(asset.source_kind, asset.source_ref)
    return "upload"


def upgrade() -> None:
    op.create_table(
        "file_folders",
        sa.Column("id", sa.Integer(), primary_key=True),
        sa.Column(
            "user_id",
            sa.Integer(),
            sa.ForeignKey("users.id", ondelete="CASCADE"),
            nullable=False,
        ),
        sa.Column("kind", sa.String(16), nullable=False),
        sa.Column("source_key", sa.String(260)),  # "site:" + a host (<= 253)
        sa.Column("name", sa.String(255)),
        sa.Column("retention_days", sa.Integer()),
        sa.Column(
            "created_at",
            sa.DateTime(timezone=True),
            server_default=sa.func.now(),
            nullable=False,
        ),
        sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
        sa.CheckConstraint(
            "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
            " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
            name="ck_file_folder_kind",
        ),
    )
    op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"])
    op.create_index(
        "uq_file_folder_own_name",
        "file_folders",
        ["user_id", sa.text("lower(name)")],
        unique=True,
        postgresql_where=sa.text("name IS NOT NULL"),
    )

    op.add_column(
        "download_jobs",
        sa.Column(
            "folder_id",
            sa.Integer(),
            sa.ForeignKey("file_folders.id", ondelete="RESTRICT"),
        ),
    )
    op.add_column(
        "download_jobs",
        sa.Column("kept", sa.Boolean(), server_default="false", nullable=False),
    )
    op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True)))
    op.add_column(
        "download_jobs",
        sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False),
    )
    op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"])
    op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"])

    conn = op.get_bind()
    raw = conn.execute(
        sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'")
    ).scalar()
    retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS

    jobs = conn.execute(
        sa.text(
            "SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id,"
            " j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at"
            " FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id"
        )
    ).fetchall()
    jobs_by_id = {j.id: j for j in jobs}
    assets = {
        a.id: a
        for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets"))
    }

    folder_ids: dict[tuple[int, str], int] = {}
    for job in jobs:
        key = _key(job, jobs_by_id, assets)
        slot = (job.user_id, key)
        if slot not in folder_ids:
            folder_ids[slot] = conn.execute(
                sa.text(
                    "INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
                    " VALUES (:u, 'source', :k, :r) RETURNING id"
                ),
                {"u": job.user_id, "k": key, "r": None if key == "upload" else retention},
            ).scalar_one()
        # A done file keeps the expiry it had (its asset's); an upload had none and keeps none.
        expires = job.asset_expires_at if job.status == "done" and key != "upload" else None
        conn.execute(
            sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"),
            {"f": folder_ids[slot], "e": expires, "id": job.id},
        )

    op.drop_column("media_assets", "expires_at")
    op.drop_column("media_assets", "gc_notified")


def downgrade() -> None:
    op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True)))
    op.add_column(
        "media_assets",
        sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
    )
    # The asset goes when its last holder would: the latest of its holders' expiries (a holder that
    # never expires keeps the asset unexpired).
    op.execute(
        "UPDATE media_assets a SET expires_at = sub.e FROM ("
        " SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END"
        " AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id"
        ") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'"
    )
    op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs")
    op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs")
    op.drop_column("download_jobs", "expiry_warned")
    op.drop_column("download_jobs", "expires_at")
    op.drop_column("download_jobs", "kept")
    op.drop_column("download_jobs", "folder_id")
    op.drop_index("uq_file_folder_own_name", table_name="file_folders")
    op.drop_index("ix_file_folders_user_id", table_name="file_folders")
    op.drop_table("file_folders")