diff --git a/.env.example b/.env.example
index fd17c2d5..95b3ad55 100644
--- a/.env.example
+++ b/.env.example
@@ -76,10 +76,10 @@ SCHEDULER_ENABLED=true
# --- Download center ---
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
-# Downloaded media defaults to a Docker-managed named volume. Set DOWNLOAD_HOST_PATH to a host
+# Downloaded media defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000
.
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
# --- Plex integration (optional) ---
# The optional Plex module browses/plays your Plex library, playing the LOCAL physical file
@@ -89,6 +89,6 @@ SCHEDULER_ENABLED=true
# to /plex-media inside the container (the local side of the admin "path map", whose default is
# /data=/plex-media). Leave unset if you don't use Plex.
# PLEX_MEDIA_HOST_PATH=/mnt/media
-# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to DOWNLOAD_ROOT; point it at
+# Optional: scratch dir for on-the-fly HLS remux segments. Defaults to FILES_ROOT; point it at
# fast local storage if your download volume is slow.
# PLEX_HLS_DIR=/var/tmp/plex-hls
diff --git a/.gitignore b/.gitignore
index 04718ff8..af6667dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,9 +29,9 @@ frontend/e2e/.auth/
*.sql.gz
backups/
-# Download center: locally downloaded media (the DOWNLOAD_ROOT bind mount at the repo root).
+# Download center: locally downloaded media (the FILES_ROOT bind mount at the repo root).
# Anchored with a leading slash so it does NOT also ignore the backend/app/downloads package.
-/downloads/
+/files/
# OS / editor
.DS_Store
diff --git a/Dockerfile b/Dockerfile
index 2a31b361..b04e23be 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -39,13 +39,13 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
WORKDIR /app
# Create the app user BEFORE the copies so files land already-owned via COPY --chown (a trailing
-# `chown -R /app` would rewrite every file into a duplicate layer). /downloads must be appuser-owned
+# `chown -R /app` would rewrite every file into a duplicate layer). /files must be appuser-owned
# so a fresh named volume mounted there inherits that ownership (Docker copies the image dir's
# ownership into a new empty volume) — the worker/API can then write without a manual chown. A bind
# mount to a host path instead needs that path writable by this uid (see docs/self-hosting.md).
RUN adduser --disabled-password --gecos "" appuser \
- && mkdir -p /downloads \
- && chown appuser /app /downloads
+ && mkdir -p /files \
+ && chown appuser /app /files
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json
index cc5f74f1..8f2e3b98 100644
--- a/backend/.basedpyright/baseline.json
+++ b/backend/.basedpyright/baseline.json
@@ -1196,16 +1196,6 @@
}
}
],
- "./app/routes/public.py": [
- {
- "code": "reportArgumentType",
- "range": {
- "startColumn": 57,
- "endColumn": 71,
- "lineCount": 1
- }
- }
- ],
"./app/routes/scheduler.py": [
{
"code": "reportArgumentType",
diff --git a/backend/alembic/versions/0050_asset_poster.py b/backend/alembic/versions/0050_asset_poster.py
index 7c210b7b..3059ea2b 100644
--- a/backend/alembic/versions/0050_asset_poster.py
+++ b/backend/alembic/versions/0050_asset_poster.py
@@ -6,7 +6,7 @@ Create Date: 2026-07-07
Some sources (a direct media URL like a reddit v.redd.it HLS manifest) carry no thumbnail, so the
download shows a blank image box. The worker now extracts a representative frame with ffmpeg and
-saves it as a poster sidecar; `poster_path` (relative to download_root) records it so the card /
+saves it as a poster sidecar; `poster_path` (relative to files_root) records it so the card /
watch page / link preview can fall back to it. Nullable — only set when there was no thumbnail.
"""
diff --git a/backend/alembic/versions/0102_file_folders.py b/backend/alembic/versions/0102_file_folders.py
new file mode 100644
index 00000000..30b965a6
--- /dev/null
+++ b/backend/alembic/versions/0102_file_folders.py
@@ -0,0 +1,193 @@
+"""The Files library: folders, and a file's retention on the user's job
+
+Revision ID: 0102_file_folders
+Revises: 0101_plex_text_plain
+Create Date: 2026-10-11
+
+Every download, clip and upload of a user now sits in a folder of their Files library
+(`app.downloads.folders`): a source folder made per origin (YouTube, Plex, uploads, one per other
+site) or one of their own. How long a file stays moves from the shared asset to each user's job
+(`download_jobs.expires_at`, stamped from the folder's `retention_days`), so one user keeping a file
+no longer depends on another's copy; the asset's own `expires_at` / `gc_notified` go.
+
+This places every existing job in its source folder and carries each done job's expiry over from
+the asset it holds (an upload keeps none). The source-key rule is copied here, frozen, from
+`folders.source_key_for` as it is today — a migration must not follow the code's later changes.
+"""
+
+from urllib.parse import urlsplit
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0102_file_folders"
+down_revision: str | None = "0101_plex_text_plain"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
+_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
+_DEFAULT_RETENTION_DAYS = 30 # config.download_retention_days on the day this was written
+
+
+def _site_key(url: str) -> str:
+ host = (urlsplit(url).hostname or "").lower().rstrip(".")
+ for door in _FRONT_DOORS:
+ if host.startswith(door) and host.count(".") > 1:
+ host = host[len(door) :]
+ break
+ if host in _YOUTUBE_HOSTS:
+ return "youtube"
+ return f"site:{host or 'unknown'}"
+
+
+def _asset_key(kind: str | None, ref: str | None) -> str:
+ if kind == "youtube":
+ return "youtube"
+ if kind == "url" and ref:
+ return _site_key(ref)
+ return "upload"
+
+
+def _key(job, jobs_by_id, assets) -> str:
+ kind = job.source_kind
+ if kind == "youtube":
+ return "youtube"
+ if kind == "upload":
+ return "upload"
+ if kind == "clip":
+ spec = job.edit_spec or {}
+ return "plex" if spec.get("source") == "plex" else "youtube"
+ if kind == "url":
+ return _site_key(job.source_ref)
+ if kind == "edit":
+ parent = jobs_by_id.get(job.source_job_id)
+ if parent is not None and parent.id != job.id:
+ return _key(parent, jobs_by_id, assets)
+ asset = assets.get(job.source_asset_id)
+ if asset is not None:
+ return _asset_key(asset.source_kind, asset.source_ref)
+ return "upload"
+
+
+def upgrade() -> None:
+ op.create_table(
+ "file_folders",
+ sa.Column("id", sa.Integer(), primary_key=True),
+ sa.Column(
+ "user_id",
+ sa.Integer(),
+ sa.ForeignKey("users.id", ondelete="CASCADE"),
+ nullable=False,
+ ),
+ sa.Column("kind", sa.String(16), nullable=False),
+ sa.Column("source_key", sa.String(260)), # "site:" + a host (<= 253)
+ sa.Column("name", sa.String(255)),
+ sa.Column("retention_days", sa.Integer()),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.func.now(),
+ nullable=False,
+ ),
+ sa.UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
+ sa.CheckConstraint(
+ "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
+ " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
+ name="ck_file_folder_kind",
+ ),
+ )
+ op.create_index("ix_file_folders_user_id", "file_folders", ["user_id"])
+ op.create_index(
+ "uq_file_folder_own_name",
+ "file_folders",
+ ["user_id", sa.text("lower(name)")],
+ unique=True,
+ postgresql_where=sa.text("name IS NOT NULL"),
+ )
+
+ op.add_column(
+ "download_jobs",
+ sa.Column(
+ "folder_id",
+ sa.Integer(),
+ sa.ForeignKey("file_folders.id", ondelete="RESTRICT"),
+ ),
+ )
+ op.add_column(
+ "download_jobs",
+ sa.Column("kept", sa.Boolean(), server_default="false", nullable=False),
+ )
+ op.add_column("download_jobs", sa.Column("expires_at", sa.DateTime(timezone=True)))
+ op.add_column(
+ "download_jobs",
+ sa.Column("expiry_warned", sa.Boolean(), server_default="false", nullable=False),
+ )
+ op.create_index("ix_download_jobs_folder_id", "download_jobs", ["folder_id"])
+ op.create_index("ix_download_jobs_expires_at", "download_jobs", ["expires_at"])
+
+ conn = op.get_bind()
+ raw = conn.execute(
+ sa.text("SELECT value FROM system_config WHERE key = 'download_retention_days'")
+ ).scalar()
+ retention = int(raw) if raw not in (None, "") else _DEFAULT_RETENTION_DAYS
+
+ jobs = conn.execute(
+ sa.text(
+ "SELECT j.id, j.user_id, j.source_kind, j.source_ref, j.edit_spec, j.source_job_id,"
+ " j.source_asset_id, j.status, j.asset_id, a.expires_at AS asset_expires_at"
+ " FROM download_jobs j LEFT JOIN media_assets a ON a.id = j.asset_id"
+ )
+ ).fetchall()
+ jobs_by_id = {j.id: j for j in jobs}
+ assets = {
+ a.id: a
+ for a in conn.execute(sa.text("SELECT id, source_kind, source_ref FROM media_assets"))
+ }
+
+ folder_ids: dict[tuple[int, str], int] = {}
+ for job in jobs:
+ key = _key(job, jobs_by_id, assets)
+ slot = (job.user_id, key)
+ if slot not in folder_ids:
+ folder_ids[slot] = conn.execute(
+ sa.text(
+ "INSERT INTO file_folders (user_id, kind, source_key, retention_days)"
+ " VALUES (:u, 'source', :k, :r) RETURNING id"
+ ),
+ {"u": job.user_id, "k": key, "r": None if key == "upload" else retention},
+ ).scalar_one()
+ # A done file keeps the expiry it had (its asset's); an upload had none and keeps none.
+ expires = job.asset_expires_at if job.status == "done" and key != "upload" else None
+ conn.execute(
+ sa.text("UPDATE download_jobs SET folder_id = :f, expires_at = :e WHERE id = :id"),
+ {"f": folder_ids[slot], "e": expires, "id": job.id},
+ )
+
+ op.drop_column("media_assets", "expires_at")
+ op.drop_column("media_assets", "gc_notified")
+
+
+def downgrade() -> None:
+ op.add_column("media_assets", sa.Column("expires_at", sa.DateTime(timezone=True)))
+ op.add_column(
+ "media_assets",
+ sa.Column("gc_notified", sa.Boolean(), server_default="false", nullable=False),
+ )
+ # The asset goes when its last holder would: the latest of its holders' expiries (a holder that
+ # never expires keeps the asset unexpired).
+ op.execute(
+ "UPDATE media_assets a SET expires_at = sub.e FROM ("
+ " SELECT asset_id, CASE WHEN bool_or(expires_at IS NULL) THEN NULL ELSE max(expires_at) END"
+ " AS e FROM download_jobs WHERE asset_id IS NOT NULL AND status = 'done' GROUP BY asset_id"
+ ") sub WHERE a.id = sub.asset_id AND a.source_kind <> 'upload'"
+ )
+ op.drop_index("ix_download_jobs_expires_at", table_name="download_jobs")
+ op.drop_index("ix_download_jobs_folder_id", table_name="download_jobs")
+ op.drop_column("download_jobs", "expiry_warned")
+ op.drop_column("download_jobs", "expires_at")
+ op.drop_column("download_jobs", "kept")
+ op.drop_column("download_jobs", "folder_id")
+ op.drop_index("uq_file_folder_own_name", table_name="file_folders")
+ op.drop_index("ix_file_folders_user_id", table_name="file_folders")
+ op.drop_table("file_folders")
diff --git a/backend/app/config.py b/backend/app/config.py
index 8edf93aa..4007159e 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -173,7 +173,7 @@ class Settings(BaseSettings):
# --- Download center (yt-dlp) ---
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
- download_root: str = "/downloads"
+ files_root: str = "/files"
# Whether THIS process runs the download worker loop. On for the dedicated worker
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
worker_enabled: bool = False
@@ -345,8 +345,8 @@ class Settings(BaseSettings):
plex_transcode_crf: int = 20
plex_transcode_max_height: int = 1080
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
- # dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
- # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
+ # dir under files_root. On localdev the files_root is a slow Windows bind-mount, so point
+ # this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's files_root is fast
# local disk, so the default is fine there.
plex_hls_dir: str = ""
diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py
index b793a770..13d035ab 100644
--- a/backend/app/downloads/backfill.py
+++ b/backend/app/downloads/backfill.py
@@ -89,7 +89,7 @@ def run() -> dict:
if a.poster_path is None:
# Record a self-hosted poster for every download (the reliable og:image source):
# ensure_poster returns the existing .jpg thumbnail sidecar, or cuts a frame.
- rel = editmod.ensure_poster(a, settings.download_root)
+ rel = editmod.ensure_poster(a, settings.files_root)
if rel:
a.poster_path = rel
filled_poster += 1
diff --git a/backend/app/downloads/edit.py b/backend/app/downloads/edit.py
index 21b82983..2ce485b1 100644
--- a/backend/app/downloads/edit.py
+++ b/backend/app/downloads/edit.py
@@ -732,7 +732,7 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
return dest if dest.exists() else None
-def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
+def ensure_poster(asset, files_root: str, timeout_s: int = 30) -> str | None:
"""Generate a poster frame for a ready asset that has no thumbnail (e.g. a direct media URL
like a reddit HLS manifest, which yt-dlp can't get a thumbnail for). Written as the media's
`.jpg` sidecar — so Plex uses it too and `delete_asset_files` cleans it up — and returned
@@ -740,7 +740,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
short clip) dodges black intro frames."""
if not asset.rel_path:
return None
- root = Path(download_root)
+ root = Path(files_root)
src = root / asset.rel_path
if not src.exists():
return None
@@ -756,7 +756,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
return rel if dest.exists() else None
-def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict | None:
+def ensure_storyboard(asset, files_root: str, timeout_s: int = 90) -> dict | None:
"""Return the filmstrip meta for a ready source asset, generating the sprite on first use.
Best-effort + time-bounded: for a very long/high-res source the one-pass decode can be slow,
@@ -768,7 +768,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
if not asset.rel_path:
return None
geom = storyboard_geometry(asset.duration_s)
- root = Path(download_root)
+ root = Path(files_root)
rel = f".storyboards/asset-{asset.id}.jpg"
dest = root / rel
diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py
new file mode 100644
index 00000000..b962bf65
--- /dev/null
+++ b/backend/app/downloads/folders.py
@@ -0,0 +1,139 @@
+"""The folders of the Files library — where each of a user's files sits.
+
+A folder is a row, never a directory: on disk a file stays where the download layout put it (one
+shared asset can be held by several users), so a rename or a move is a column update. Two kinds:
+
+- a SOURCE folder is made automatically, one per origin a user has files from (YouTube, Plex clips,
+ uploads, one per other site). Its `source_key` names the origin; the client renders its name
+ from the key, in the reader's language.
+- an OWN folder is the user's, with the name they gave it.
+
+Every file of a user is in exactly one folder. The folder a new file lands in is decided here
+(`source_key_for` + `source_folder`), at every place a job is created, so the rule lives once.
+
+Retention is the folder's: a source folder keeps a file `download_retention_days` (the value when the
+folder was made; the user can change it), Uploads and own folders keep it until it is deleted. A
+done file carries its own `expires_at`, stamped by `done_expiry` when it finishes — unless the user
+kept it.
+"""
+
+from datetime import datetime, timedelta, timezone
+from enum import StrEnum
+from urllib.parse import urlsplit
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app import sysconfig
+from app.models import DownloadJob, FileFolder, MediaAsset
+
+
+class FolderKind(StrEnum):
+ SOURCE = "source"
+ OWN = "own"
+
+
+class SourceKey(StrEnum):
+ """The fixed origins. Any other site is `site:` (see `site_key`)."""
+
+ YOUTUBE = "youtube"
+ PLEX = "plex"
+ UPLOAD = "upload"
+
+
+SITE_PREFIX = "site:"
+
+# A host's leading label that only says "the same site, another front door".
+_FRONT_DOORS = ("www.", "m.", "mobile.", "web.")
+_YOUTUBE_HOSTS = {"youtube.com", "youtu.be", "youtube-nocookie.com", "music.youtube.com"}
+
+
+def site_key(url: str) -> str:
+ """The source key of a page on some site: `site:` without its front door (www., m.),
+ or the YouTube key for a YouTube address that arrived as a plain URL."""
+ host = (urlsplit(url).hostname or "").lower().rstrip(".")
+ for door in _FRONT_DOORS:
+ if host.startswith(door) and host.count(".") > 1:
+ host = host[len(door) :]
+ break
+ if host in _YOUTUBE_HOSTS:
+ return SourceKey.YOUTUBE
+ return f"{SITE_PREFIX}{host or 'unknown'}"
+
+
+def source_key_for(db: Session, job: DownloadJob) -> str:
+ """The origin a job's file came from — the source folder it belongs in.
+
+ youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL →
+ its site; an edit → wherever its parent's file came from (its parent job, or, once that is
+ deleted, the parent's asset)."""
+ kind = job.source_kind
+ if kind == "youtube":
+ return SourceKey.YOUTUBE
+ if kind == "upload":
+ return SourceKey.UPLOAD
+ if kind == "clip":
+ return (
+ SourceKey.PLEX if (job.edit_spec or {}).get("source") == "plex" else SourceKey.YOUTUBE
+ )
+ if kind == "url":
+ return site_key(job.source_ref)
+ if kind == "edit":
+ parent = db.get(DownloadJob, job.source_job_id) if job.source_job_id else None
+ if parent is not None and parent.id != job.id:
+ return source_key_for(db, parent)
+ asset = db.get(MediaAsset, job.source_asset_id) if job.source_asset_id else None
+ if asset is not None:
+ if asset.source_kind == "youtube":
+ return SourceKey.YOUTUBE
+ if asset.source_kind == "upload":
+ return SourceKey.UPLOAD
+ if asset.source_kind == "url":
+ return site_key(asset.source_ref)
+ return SourceKey.UPLOAD
+
+
+def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
+ """The user's source folder for `key`, made on first use. A race on the unique (user, key) is
+ settled by re-selecting, inside a SAVEPOINT so the caller's own pending writes survive."""
+ stmt = select(FileFolder).where(FileFolder.user_id == user_id, FileFolder.source_key == key)
+ folder = db.execute(stmt).scalar_one_or_none()
+ if folder is not None:
+ return folder
+ retention = (
+ None if key == SourceKey.UPLOAD else sysconfig.get_int(db, "download_retention_days")
+ )
+ folder = FileFolder(
+ user_id=user_id, kind=FolderKind.SOURCE, source_key=key, retention_days=retention
+ )
+ try:
+ with db.begin_nested():
+ db.add(folder)
+ db.flush()
+ # silent-ok: the expected outcome of the race the savepoint exists for — another request made
+ # the same folder first; re-selecting it is the whole handling.
+ except IntegrityError:
+ folder = db.execute(stmt).scalar_one()
+ return folder
+
+
+def place(db: Session, job: DownloadJob) -> None:
+ """Put a new job's file in its source folder. Called wherever a job is created; the job must
+ carry its `user_id`, `source_kind` and source refs already."""
+ job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
+
+
+def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
+ """When a file that finishes now goes: its folder's retention from now, or never (kept, or a
+ folder that keeps files)."""
+ if kept or folder_id is None:
+ return None
+ days = db.execute(select(FileFolder.retention_days).where(FileFolder.id == folder_id)).scalar()
+ return None if days is None else datetime.now(timezone.utc) + timedelta(days=days)
+
+
+def stamp_expiry(db: Session, job: DownloadJob) -> None:
+ """Stamp a job that just became done (in this session) with its expiry."""
+ job.expires_at = done_expiry(db, job.folder_id, job.kept)
+ job.expiry_warned = False
diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py
index 7a9c2afd..6b22c2ea 100644
--- a/backend/app/downloads/gc.py
+++ b/backend/app/downloads/gc.py
@@ -1,12 +1,13 @@
"""Download retention garbage collector (a scheduler job).
Five passes each run:
- 1. Pre-expiry warning — for ready assets expiring within the grace window, notify each holder
- once (gc_notified flag) so a download isn't silently deleted.
- 2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
- holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
- 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
- ready assets (never an upload) until back under it.
+ 1. Pre-expiry warning — a done file whose `expires_at` (its folder's retention, `folders`) falls
+ within the grace window, and that its user did not keep, warns its user once (`expiry_warned`).
+ 2. Expiry — such a file past its `expires_at` leaves the library: the job is deleted and its hold
+ released, so the shared file goes with its last holder.
+ 3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed ready
+ assets (never an upload, never one a user kept) until back under it; each holder's file leaves
+ their library, with a notice.
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
passes 1-3 skip them; benign row-count hygiene).
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
@@ -14,7 +15,7 @@ Five passes each run:
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
so an in-flight download is never touched.
-Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
+Runs in the API process (which mounts FILES_ROOT). Pure disk/DB work — no YouTube quota.
"""
import time
@@ -26,7 +27,8 @@ from sqlalchemy.orm import Session
from app import logs, sysconfig
from app.config import settings
-from app.downloads import quota, storage
+from app.downloads import quota, service, storage
+from app.downloads.status import JobStatus
from app.models import DownloadJob, MediaAsset
from app.notifications import create_notification
@@ -37,42 +39,39 @@ _UPLOAD_TREE = ".uploads"
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
-def _holders(db: Session, asset_id: int) -> list[int]:
- return list(
- db.execute(
- select(DownloadJob.user_id).where(DownloadJob.asset_id == asset_id).distinct()
- ).scalars()
+def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) -> None:
+ title = job.display_name or (asset.title if asset else None) or "download"
+ create_notification(
+ db,
+ user_id=job.user_id,
+ type=ntype,
+ title=title, # English fallback; the client renders a translated message from data
+ data={
+ "kind": ntype,
+ "title": title,
+ "video_id": asset.source_ref if asset else None,
+ "job_id": job.id,
+ },
+ commit=False,
)
-def _notify_holders(db: Session, asset: MediaAsset, ntype: str) -> None:
- title = asset.title or "download"
- for uid in _holders(db, asset.id):
- create_notification(
- db,
- user_id=uid,
- type=ntype,
- title=title, # English fallback; the client renders a translated message from data
- data={"kind": ntype, "title": asset.title, "video_id": asset.source_ref},
- commit=False,
- )
+def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
+ """Take a file out of its user's library: tell them, release the job's hold on the asset (the
+ file goes with the last holder), delete the row (its shares and links cascade)."""
+ _notify(db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, ntype)
+ service.release_asset(db, job)
+ db.delete(job)
-def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
- if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
- _notify_holders(db, asset, ntype)
- db.delete(asset) # jobs.asset_id -> NULL via FK ON DELETE SET NULL
-
-
-def _sweep_orphan_files(db: Session, download_root: str) -> int:
- """Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
+def _sweep_orphan_files(db: Session, files_root: str) -> int:
+ """Delete media/sidecar files (and now-empty dirs) under FILES_ROOT that no `ready` asset
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
row names at all. Skips the other dot-prefixed system trees and only removes files older than
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
NOT owned by a surviving asset."""
- root = Path(download_root)
+ root = Path(files_root)
if not root.is_dir():
return 0
keep: set[Path] = set()
@@ -151,47 +150,52 @@ def run_download_gc(db: Session) -> dict:
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = reaped = 0
- # 1. Pre-expiry warnings.
+ # 1. Pre-expiry warnings — one per file, the first run inside the grace window.
warn_before = now + timedelta(days=grace_days)
soon = (
db.execute(
- select(MediaAsset).where(
- MediaAsset.status == "ready",
- MediaAsset.gc_notified.is_(False),
- MediaAsset.expires_at.is_not(None),
- MediaAsset.expires_at > now,
- MediaAsset.expires_at <= warn_before,
+ select(DownloadJob).where(
+ DownloadJob.status == JobStatus.DONE,
+ DownloadJob.kept.is_(False),
+ DownloadJob.expiry_warned.is_(False),
+ DownloadJob.expires_at.is_not(None),
+ DownloadJob.expires_at > now,
+ DownloadJob.expires_at <= warn_before,
)
)
.scalars()
.all()
)
- for asset in soon:
- _notify_holders(db, asset, "download_expiring")
- asset.gc_notified = True
+ for job in soon:
+ _notify(
+ db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
+ )
+ job.expiry_warned = True
warned += 1
if soon:
db.commit()
- # 2. Expiry deletion (ready assets past TTL, and any errored leftovers with a TTL).
+ # 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
dead = (
db.execute(
- select(MediaAsset).where(
- MediaAsset.status.in_(("ready", "error")),
- MediaAsset.expires_at.is_not(None),
- MediaAsset.expires_at <= now,
+ select(DownloadJob).where(
+ DownloadJob.status == JobStatus.DONE,
+ DownloadJob.kept.is_(False),
+ DownloadJob.expires_at.is_not(None),
+ DownloadJob.expires_at <= now,
)
)
.scalars()
.all()
)
- for asset in dead:
- _delete_asset(db, asset, "download_expired")
+ for job in dead:
+ _drop_job(db, job, "download_expired")
expired += 1
if dead:
db.commit()
- # 3. LRU eviction over the total cache cap.
+ # 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
+ # fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = (
@@ -201,12 +205,17 @@ def run_download_gc(db: Session) -> dict:
or 0
)
if total > cap:
- # An upload counts toward the total (it is disk like any other) but is never evicted:
- # it has no source to fetch again, and it lives until its owner deletes it.
+ kept_assets = select(DownloadJob.asset_id).where(
+ DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
+ )
candidates = (
db.execute(
select(MediaAsset)
- .where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
+ .where(
+ MediaAsset.status == "ready",
+ MediaAsset.source_kind != "upload",
+ MediaAsset.id.not_in(kept_assets),
+ )
.order_by(MediaAsset.last_access_at.asc().nulls_first())
)
.scalars()
@@ -216,7 +225,17 @@ def run_download_gc(db: Session) -> dict:
if total <= cap:
break
total -= asset.size_bytes or 0
- _delete_asset(db, asset, "download_evicted")
+ holders = (
+ db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
+ .scalars()
+ .all()
+ )
+ for job in holders:
+ _notify(db, job, asset, "download_evicted")
+ db.delete(job)
+ if asset.rel_path:
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
+ db.delete(asset)
evicted += 1
db.commit()
@@ -257,7 +276,7 @@ def run_download_gc(db: Session) -> dict:
)
for asset in unheld:
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
reaped += 1
if unheld:
@@ -265,7 +284,7 @@ def run_download_gc(db: Session) -> dict:
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
- swept = _sweep_orphan_files(db, settings.download_root)
+ swept = _sweep_orphan_files(db, settings.files_root)
result = {
"warned": warned,
diff --git a/backend/app/downloads/jobdb.py b/backend/app/downloads/jobdb.py
index a0935749..a8e95b29 100644
--- a/backend/app/downloads/jobdb.py
+++ b/backend/app/downloads/jobdb.py
@@ -68,8 +68,8 @@ def _reset_asset_pending(asset_id: int) -> None:
def _staging_dir(asset_id: int) -> Path:
- return Path(settings.download_root) / ".staging" / f"asset-{asset_id}"
+ return Path(settings.files_root) / ".staging" / f"asset-{asset_id}"
def _edit_staging_dir(asset_id: int) -> Path:
- return Path(settings.download_root) / ".staging" / f"edit-{asset_id}"
+ return Path(settings.files_root) / ".staging" / f"edit-{asset_id}"
diff --git a/backend/app/downloads/og.py b/backend/app/downloads/og.py
index 2bc7bf47..99cd2154 100644
--- a/backend/app/downloads/og.py
+++ b/backend/app/downloads/og.py
@@ -135,7 +135,7 @@ def _watch_tags(token: str, db: Session) -> str | None:
if image_dims is None and image.endswith("poster.jpg") and asset.poster_path:
# safe_abs_path returns None if the poster file is missing (gc'd) or escapes the root —
# guard it, else _jpeg_size(None) would raise an uncaught AttributeError on this public page.
- poster_abs = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ poster_abs = storage.safe_abs_path(settings.files_root, asset.poster_path)
image_dims = _jpeg_size(poster_abs) if poster_abs else None
if image_dims:
tags.append(_tag("og:image:width", str(image_dims[0])))
diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py
index 2b937438..3e792513 100644
--- a/backend/app/downloads/pipeline.py
+++ b/backend/app/downloads/pipeline.py
@@ -3,7 +3,7 @@
import re
import shutil
import time
-from datetime import date, datetime, timedelta, timezone
+from datetime import date, datetime, timezone
from pathlib import Path
import yt_dlp
@@ -434,14 +434,12 @@ def _extract_with_fallback(
raise last_exc # unreachable (loop either returns or raises)
-def _download_settings() -> tuple[str, int]:
- """Layout + retention days from the DB config (admin-overridable on the Configuration page),
- falling back to the env defaults. Read here, not from `settings`, so an admin edit takes effect."""
+def _download_layout() -> str:
+ """The on-disk layout from the DB config (admin-overridable on the Configuration page), falling
+ back to the env default. Read here, not from `settings`, so an admin edit takes effect. (How long
+ a file stays is its folder's, stamped on the job when it is done — `folders.done_expiry`.)"""
with SessionLocal() as db:
- return (
- sysconfig.get_str(db, "download_layout"),
- sysconfig.get_int(db, "download_retention_days"),
- )
+ return sysconfig.get_str(db, "download_layout")
def _ensure_phone_safe(
@@ -588,19 +586,18 @@ def _download(
thumbnail_url=info.get("thumbnail"),
)
- layout, retention_days = _download_settings()
+ layout = _download_layout()
rel = storage.rel_path(meta, ext, layout, asset_id=asset_id)
thumb = _find_thumbnail(staging, produced)
# Convert the raw thumbnail (webp/png/jpeg) to a real JPEG ourselves — robustly — for the
# poster + og:image sidecar, now that yt-dlp's fatal thumbnail convertor is gone. On failure
# this is None and write_sidecars writes no poster, so ensure_poster cuts a frame instead.
poster_jpg = editmod.thumbnail_to_jpg(thumb, staging / "poster.jpg")
- storage.place_file(produced, settings.download_root, rel)
- nfo_ok = storage.write_sidecars(settings.download_root, rel, meta, poster_jpg)
+ storage.place_file(produced, settings.files_root, rel)
+ nfo_ok = storage.write_sidecars(settings.files_root, rel, meta, poster_jpg)
- final = storage.abs_path(settings.download_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
- expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -626,12 +623,11 @@ def _download(
# (og:image) source — a remote thumbnail, esp. Facebook's signed CDN URL, expires
# and can't be relied on cross-origin. write_sidecars already wrote .jpg from
# the thumbnail; ensure_poster returns that, or cuts a frame if there was none.
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.source_webpage_url = info.get("webpage_url")
asset.nfo_written = nfo_ok
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
owner_id = job.user_id if job else None
@@ -840,7 +836,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
if not src_ready or not src_rel:
_fail_edit(job_id, asset_id, "The source download is no longer available.")
return
- src_path = storage.abs_path(settings.download_root, src_rel)
+ src_path = storage.abs_path(settings.files_root, src_rel)
if not src_path.exists():
_fail_edit(job_id, asset_id, "The source file is missing.")
return
@@ -887,13 +883,11 @@ def _process_edit(job_id: int, asset_id: int) -> None:
raise StatedFailure(dlerrors.CLIP_TOO_BIG_MESSAGE)
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
- storage.place_file(dest_staging, settings.download_root, rel)
- final = storage.abs_path(settings.download_root, rel)
+ storage.place_file(dest_staging, settings.files_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
size = final.stat().st_size if final.exists() else None
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
_, _, out_w, out_h, _ = probe.probe_media(final)
- _, retention_days = _download_settings()
- expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -912,7 +906,6 @@ def _process_edit(job_id: int, asset_id: int) -> None:
asset.width, asset.height = src_w, src_h
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = expires
job = db.get(DownloadJob, job_id)
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
owner_id = job.user_id if job else None
@@ -1086,10 +1079,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
raise _OwnerRefused(dlerrors.CLIP_TOO_BIG_MESSAGE)
ext = out.suffix.lstrip(".").lower() or "mp4"
rel = storage.edit_rel_path(user_id, asset_id, ext)
- storage.place_file(out, settings.download_root, rel)
- final = storage.abs_path(settings.download_root, rel)
+ storage.place_file(out, settings.files_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
vcodec, acodec, width, height, _ = probe.probe_media(final)
- _, retention_days = _download_settings()
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
@@ -1108,10 +1100,9 @@ def _process_clip(job_id: int, asset_id: int) -> None:
asset.media_kind = mediakind.derive(vcodec, acodec)
# The clip's own frame as its poster: the link preview shows the moment, and a Plex
# item has no public thumbnail at all.
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
asset.error = None
asset.last_access_at = datetime.now(timezone.utc)
- asset.expires_at = datetime.now(timezone.utc) + timedelta(days=retention_days)
fields: dict = {"status": JobStatus.DONE, "progress": 100, "phase": None}
# A video outside the catalog was queued as "Clip (clip)": the fetch has its title now.
placeholder = service.clip_title(None, None)
@@ -1230,7 +1221,7 @@ def _free_unheld_upload(asset_id: int) -> bool:
if asset is None or asset.ref_count != 0:
return False
if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
db.commit()
return True
@@ -1253,7 +1244,7 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
duration = asset.duration_s if asset else None
job = db.get(DownloadJob, job_id)
owner_id = job.user_id if job else None
- src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
+ src = storage.safe_abs_path(settings.files_root, old_rel) if old_rel else None
if old_rel is None or src is None or not src.exists():
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
return
@@ -1267,10 +1258,10 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
new_rel = storage.with_ext(old_rel, old_ext, ext)
if safe != src:
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
- storage.place_file(safe, settings.download_root, new_rel)
+ storage.place_file(safe, settings.files_root, new_rel)
if new_rel != old_rel:
- placed = storage.abs_path(settings.download_root, new_rel)
- final = storage.abs_path(settings.download_root, new_rel)
+ placed = storage.abs_path(settings.files_root, new_rel)
+ final = storage.abs_path(settings.files_root, new_rel)
vcodec, acodec, width, height, dur = probe.probe_media(final)
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py
index 96770af0..fd91d10f 100644
--- a/backend/app/downloads/service.py
+++ b/backend/app/downloads/service.py
@@ -19,7 +19,7 @@ from app.downloads import adapters
from app.downloads import clip as clipmod
from app.downloads import credentials
from app.downloads import edit as editmod
-from app.downloads import formats, mediakind, quota, storage
+from app.downloads import folders, formats, mediakind, quota, storage
from app.downloads.status import HOLDING, JobKind, JobStatus
from app.models import (
DISPLAY_TEXT_MAX,
@@ -198,12 +198,14 @@ def enqueue(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
# Cache hit: the file already exists — no worker round-trip needed.
if asset.status == "ready":
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -277,11 +279,13 @@ def enqueue_edit(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
if asset.status == "ready": # identical clip already produced for this user → instant
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -364,11 +368,13 @@ def enqueue_clip(
queue_pos=next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
incr_ref_count(db, asset.id)
if asset.status == "ready": # the same range was already cut for this user → instant
job.status = JobStatus.DONE
job.progress = 100
+ folders.stamp_expiry(db, job)
asset.last_access_at = datetime.now(timezone.utc)
db.commit()
@@ -392,6 +398,17 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
if not fields:
return False
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
+ if fields.get("status") == JobStatus.DONE:
+ # Done is when a file's retention starts: its folder's, unless the user kept it.
+ held = db.execute(
+ select(DownloadJob.folder_id, DownloadJob.kept).where(DownloadJob.id == job_id)
+ ).one_or_none()
+ if held is not None:
+ fields = {
+ **fields,
+ "expires_at": folders.done_expiry(db, held.folder_id, held.kept),
+ "expiry_warned": False,
+ }
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
@@ -436,7 +453,7 @@ def release_asset(db: Session, job: DownloadJob) -> None:
)
if new_count == 0 and freeable:
if asset.rel_path:
- storage.delete_asset_files(settings.download_root, asset.rel_path)
+ storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset)
@@ -565,4 +582,6 @@ def redownload(db: Session, job: DownloadJob) -> None:
# name would survive the rebuild).
job.display_name = None
job.queue_pos = next_queue_pos(db) # tail of the queue, not its stale original slot
+ job.expires_at = None # stamped again when the rebuild is done
+ job.expiry_warned = False
db.commit()
diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py
index 4a3ea17e..d2ec540e 100644
--- a/backend/app/downloads/storage.py
+++ b/backend/app/downloads/storage.py
@@ -122,7 +122,7 @@ def display_filename(name: str) -> str:
def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | None = None) -> str:
- """Path of the media file relative to DOWNLOAD_ROOT (forward slashes).
+ """Path of the media file relative to FILES_ROOT (forward slashes).
`asset_id` disambiguates the file. A MediaAsset is unique on (source_kind, source_ref,
format_sig), so the SAME video downloaded in two formats is two distinct assets — but without
@@ -144,7 +144,7 @@ def rel_path(meta: MediaMeta, ext: str, layout: str = "plex", asset_id: int | No
def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
- """On-disk path (relative to DOWNLOAD_ROOT) for a phase-2 editor derivative.
+ """On-disk path (relative to FILES_ROOT) for a phase-2 editor derivative.
Kept in a per-user `.edits/` tree — device-downloadable via the file endpoint, but NOT part of
the Plex library (these are user clips, not canonical episodes; there are no .nfo/poster
@@ -161,7 +161,7 @@ def _clean_upload_ext(filename: str) -> str:
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
- """On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
+ """On-disk path (relative to FILES_ROOT) for a locally-uploaded file.
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
@@ -250,24 +250,27 @@ def download_filename(display_name: str, container: str | None, path: Path) -> s
return f"{base}.{ext}"
-def abs_path(download_root: str, rel: str) -> Path:
- return Path(download_root) / rel
+def abs_path(files_root: str, rel: str) -> Path:
+ return Path(files_root) / rel
-def safe_abs_path(download_root: str, rel: str) -> Path | None:
- """The resolved absolute path for `rel`, but ONLY if it stays inside DOWNLOAD_ROOT and exists on
- disk — otherwise None. Centralizes the path-traversal + existence guard every file-serving endpoint
- shares, so the containment check lives in exactly one place."""
- root = Path(download_root).resolve()
- path = abs_path(download_root, rel).resolve()
+def safe_abs_path(files_root: str, rel: str | None) -> Path | None:
+ """The resolved absolute path for `rel`, but ONLY if it stays inside FILES_ROOT and exists on
+ disk — otherwise None (no `rel` at all included: an asset without a file has nothing to serve).
+ Centralizes the path-traversal + existence guard every file-serving endpoint shares, so the
+ containment check lives in exactly one place."""
+ if not rel:
+ return None
+ root = Path(files_root).resolve()
+ path = abs_path(files_root, rel).resolve()
if root not in path.parents or not path.exists():
return None
return path
-def place_file(staging_file: Path, download_root: str, rel: str) -> None:
- """Move a completed staging file into its final location under DOWNLOAD_ROOT."""
- dest = abs_path(download_root, rel)
+def place_file(staging_file: Path, files_root: str, rel: str) -> None:
+ """Move a completed staging file into its final location under FILES_ROOT."""
+ dest = abs_path(files_root, rel)
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(staging_file), str(dest))
@@ -301,9 +304,9 @@ def _sidecar(media: Path, ext: str) -> Path:
return media.with_suffix(ext)
-def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
+def write_sidecars(files_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None) -> bool:
"""Write `.nfo` + `.jpg` next to the media file. Returns True if written."""
- media = abs_path(download_root, rel)
+ media = abs_path(files_root, rel)
try:
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
if thumb_src and thumb_src.exists():
@@ -319,9 +322,9 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
return False
-def delete_asset_files(download_root: str, rel: str) -> None:
+def delete_asset_files(files_root: str, rel: str) -> None:
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
- media = abs_path(download_root, rel)
+ media = abs_path(files_root, rel)
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
try:
p.unlink()
@@ -335,7 +338,7 @@ def delete_asset_files(download_root: str, rel: str) -> None:
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
)
# Prune empty Season/Channel dirs up to (but not including) the root.
- root = Path(download_root).resolve()
+ root = Path(files_root).resolve()
parent = media.parent
while parent != root and root in parent.parents:
try:
diff --git a/backend/app/downloads/upload.py b/backend/app/downloads/upload.py
index 7612c4d3..c22ec9c6 100644
--- a/backend/app/downloads/upload.py
+++ b/backend/app/downloads/upload.py
@@ -33,7 +33,7 @@ from sqlalchemy.orm import Session
from app.config import settings
from app.downloads import edit as editmod
-from app.downloads import mediakind, playable, probe, quota, service, storage
+from app.downloads import folders, mediakind, playable, probe, quota, service, storage
from app.downloads.status import JobKind, JobStatus
from app.models import DownloadJob, MediaAsset, display_text
@@ -81,14 +81,14 @@ def ingest_upload(
db.flush() # need asset.id for the on-disk path
rel = storage.upload_rel_path(user_id, asset.id, filename)
- storage.place_file(src_path, settings.download_root, rel)
+ storage.place_file(src_path, settings.files_root, rel)
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
asset_id = asset.id
try:
- final = storage.abs_path(settings.download_root, rel)
+ final = storage.abs_path(settings.files_root, rel)
actual_size = final.stat().st_size if final.exists() else size
asset.status = "pending" if convert else "ready"
@@ -108,7 +108,7 @@ def ingest_upload(
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
# server side even for a codec the browser can't play).
- asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
+ asset.poster_path = editmod.ensure_poster(asset, settings.files_root)
job = DownloadJob(
user_id=user_id,
@@ -123,6 +123,9 @@ def ingest_upload(
queue_pos=service.next_queue_pos(db),
)
db.add(job)
+ folders.place(db, job)
+ if not convert:
+ folders.stamp_expiry(db, job)
service.incr_ref_count(db, asset.id)
db.commit()
except Exception:
@@ -139,7 +142,7 @@ def ingest_upload(
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
# when none was generated (audio/file, or a video whose poster never started).
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
- storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
+ storage.abs_path(settings.files_root, leaked).unlink(missing_ok=True)
raise
db.refresh(job)
return job
diff --git a/backend/app/models.py b/backend/app/models.py
index ec4c780c..45d57e29 100644
--- a/backend/app/models.py
+++ b/backend/app/models.py
@@ -18,6 +18,7 @@ from sqlalchemy import (
Text,
UniqueConstraint,
func,
+ text,
)
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
@@ -826,12 +827,13 @@ class MediaAsset(Base, TimestampMixin):
links to this ready asset instantly — no re-download, no cost. Only the SOURCE download is
shared; per-user edited derivatives (phase 2) are never cached here.
- `rel_path` is relative to `settings.download_root` (the mount), so the physical filename is
+ `rel_path` is relative to `settings.files_root` (the mount), so the physical filename is
system-decided and bound to the source id — never renamed (the user's custom name is display
metadata on `download_job`, applied only when downloading to their own machine). The naming
fields (`title`/`uploader`/`upload_date`) are denormalized so the Plex-tree path can be built
- uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here)
- plus `last_access_at`/`expires_at` drive retention + LRU eviction in the GC job."""
+ uniformly for both catalog videos and ad-hoc URLs. `ref_count` (active jobs pointing here) frees
+ the file with its last holder, and `last_access_at` orders the GC's LRU eviction. How long a file
+ stays is not the asset's: each holder's job carries its own `expires_at` (its library folder's)."""
__tablename__ = "media_assets"
__table_args__ = (
@@ -847,10 +849,10 @@ class MediaAsset(Base, TimestampMixin):
status: Mapped[str] = mapped_column(
String(16), default="pending", server_default="pending", index=True
) # "pending" | "ready" | "error"
- rel_path: Mapped[str | None] = mapped_column(Text) # relative to download_root
+ rel_path: Mapped[str | None] = mapped_column(Text) # relative to files_root
storyboard_path: Mapped[str | None] = mapped_column(Text) # phase-2 filmstrip mosaic
# A locally-generated poster frame (ffmpeg), set only when the source had NO thumbnail so the
- # card / watch page / link preview can still show an image. Relative to download_root.
+ # card / watch page / link preview can still show an image. Relative to files_root.
poster_path: Mapped[str | None] = mapped_column(Text)
size_bytes: Mapped[int | None] = mapped_column(BigInteger)
duration_s: Mapped[int | None] = mapped_column(Integer)
@@ -866,8 +868,6 @@ class MediaAsset(Base, TimestampMixin):
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
media_kind: Mapped[str | None] = mapped_column(String(16))
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
- # Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
- gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
# Denormalized naming/metadata for the Plex tree + ad-hoc display.
title: Mapped[str | None] = mapped_column(Text)
uploader: Mapped[str | None] = mapped_column(String(255))
@@ -883,7 +883,6 @@ class MediaAsset(Base, TimestampMixin):
error: Mapped[str | None] = mapped_column(Text)
ref_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
last_access_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
- expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class DownloadProfile(Base, TimestampMixin):
@@ -907,6 +906,40 @@ class DownloadProfile(Base, TimestampMixin):
sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+class FileFolder(Base, TimestampMixin):
+ """A folder of the Files library (`app.downloads.folders`): a row, never a directory.
+
+ A SOURCE folder (`source_key` set: youtube / plex / upload / site:) is made the first time a
+ user gets a file from that origin; an OWN folder (`name` set) is one the user made. Every file —
+ a `DownloadJob` — sits in exactly one (`DownloadJob.folder_id`). `retention_days` is how long a
+ file stays once done (NULL = until deleted); a file's own `expires_at` is stamped from it."""
+
+ __tablename__ = "file_folders"
+ __table_args__ = (
+ UniqueConstraint("user_id", "source_key", name="uq_file_folder_source"),
+ Index(
+ "uq_file_folder_own_name",
+ "user_id",
+ func.lower(text("name")),
+ unique=True,
+ postgresql_where=text("name IS NOT NULL"),
+ ),
+ CheckConstraint(
+ "(kind = 'source' AND source_key IS NOT NULL AND name IS NULL)"
+ " OR (kind = 'own' AND name IS NOT NULL AND source_key IS NULL)",
+ name="ck_file_folder_kind",
+ ),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
+ kind: Mapped[str] = mapped_column(String(16)) # folders.FolderKind
+ # "site:" + a host name (DNS caps one at 253 characters).
+ source_key: Mapped[str | None] = mapped_column(String(260))
+ name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
+ retention_days: Mapped[int | None] = mapped_column(Integer)
+
+
class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
"""A user's request to download one source — the per-user layer over `MediaAsset`.
@@ -974,6 +1007,16 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+ # The Files library: the folder this file sits in (`downloads.folders.place` at creation; an own
+ # folder is emptied back into the source folders before it is deleted, hence RESTRICT), whether
+ # the user keeps it past its folder's retention, and when it goes (stamped once done; NULL =
+ # never). `expiry_warned` marks the one warning the GC sends before it deletes the file.
+ folder_id: Mapped[int | None] = mapped_column(
+ ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
+ )
+ kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
+ expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
+ expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
class DownloadShare(Base, TimestampMixin):
diff --git a/backend/app/plex/cache.py b/backend/app/plex/cache.py
index b1aec046..1836f713 100644
--- a/backend/app/plex/cache.py
+++ b/backend/app/plex/cache.py
@@ -10,4 +10,4 @@ from pathlib import Path
from app.config import settings
-CACHE_DIR = Path(settings.download_root) / ".plex-img-cache"
+CACHE_DIR = Path(settings.files_root) / ".plex-img-cache"
diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py
index 1039f6af..4616bae0 100644
--- a/backend/app/plex/stream.py
+++ b/backend/app/plex/stream.py
@@ -41,7 +41,7 @@ log = logs.get_logger(__name__)
_HLS_ROOT = (
Path(settings.plex_hls_dir)
if settings.plex_hls_dir
- else Path(settings.download_root) / ".plex-hls"
+ else Path(settings.files_root) / ".plex-hls"
)
_SEG_SECONDS = 6
# Cold-start warm-up. A freshly-spawned ffmpeg has produced only the first segment (or two) by the
diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py
index 3e3af777..ff9acc32 100644
--- a/backend/app/routes/downloads.py
+++ b/backend/app/routes/downloads.py
@@ -145,6 +145,10 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"edit_spec": job.edit_spec,
"can_download": ready and job.status == JobStatus.DONE,
"expired": job.asset_id is None and job.status == JobStatus.DONE,
+ # The Files library: its folder, whether the user keeps it, and when it goes (NULL = never).
+ "folder_id": job.folder_id,
+ "kept": job.kept,
+ "expires_at": iso(job.expires_at),
"asset": None
if asset is None
else {
@@ -163,7 +167,6 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
"media_kind": asset.media_kind,
"uploader_url": asset.uploader_url,
"thumbnail_url": asset.thumbnail_url,
- "expires_at": iso(asset.expires_at),
},
}
@@ -495,7 +498,7 @@ async def spool_upload(
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
# check a connection straight back out.
release_read_connection(db)
- staging = Path(settings.download_root) / ".staging" / "uploads"
+ staging = Path(settings.files_root) / ".staging" / "uploads"
staging.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(staging))
tmp_path = Path(tmp)
@@ -871,7 +874,7 @@ def download_file(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise not_found("file")
@@ -946,7 +949,7 @@ def storyboard_meta(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(409, "download_not_ready")
- meta = editmod.ensure_storyboard(asset, settings.download_root)
+ meta = editmod.ensure_storyboard(asset, settings.files_root)
if meta is None:
return {"available": False}
db.commit() # persist storyboard_path if it was just generated
@@ -967,7 +970,7 @@ def poster_image(job_id: int, user: User = Depends(current_user), db: Session =
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.poster_path:
raise not_found("poster")
- path = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ path = storage.safe_abs_path(settings.files_root, asset.poster_path)
if path is None:
raise not_found("poster")
return FileResponse(path, media_type="image/jpeg")
@@ -981,7 +984,7 @@ def storyboard_image(
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
if asset is None or not asset.storyboard_path:
raise not_found("filmstrip")
- path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
+ path = storage.safe_abs_path(settings.files_root, asset.storyboard_path)
if path is None:
raise not_found("filmstrip")
return FileResponse(path, media_type="image/jpeg")
diff --git a/backend/app/routes/public.py b/backend/app/routes/public.py
index 42099a93..ddb6e8da 100644
--- a/backend/app/routes/public.py
+++ b/backend/app/routes/public.py
@@ -73,7 +73,7 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
raise refused(404, "video_gone")
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
# loads a player for a missing file — meta and file agree.
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise refused(404, "video_gone")
return job, asset
@@ -158,7 +158,7 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
raise refused(403, "share_password_required")
asset = _ready_asset(db, link)
- path = storage.safe_abs_path(settings.download_root, asset.rel_path)
+ path = storage.safe_abs_path(settings.files_root, asset.rel_path)
if path is None:
raise refused(404, "video_gone")
@@ -194,7 +194,7 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)
job, asset = _ready_target(db, link)
if not asset.poster_path:
raise not_found("poster")
- path = storage.safe_abs_path(settings.download_root, asset.poster_path)
+ path = storage.safe_abs_path(settings.files_root, asset.poster_path)
if path is None:
raise not_found("poster")
return FileResponse(path, media_type="image/jpeg")
diff --git a/backend/app/worker.py b/backend/app/worker.py
index dbff8e35..5f16d46b 100644
--- a/backend/app/worker.py
+++ b/backend/app/worker.py
@@ -715,7 +715,7 @@ def main() -> None:
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
threads: dict[int, threading.Thread] = {}
level = _sync_knobs(threads, None)
- log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.download_root)
+ log.info("Download worker started (concurrency=%d, root=%s).", _target, settings.files_root)
while not _stop.wait(_KNOB_SECONDS):
level = _sync_knobs(threads, level)
log.info("Download worker stopping…")
diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py
index 8566ca0d..9778e2cc 100644
--- a/backend/tests/test_clip.py
+++ b/backend/tests/test_clip.py
@@ -395,7 +395,7 @@ def test_a_clip_job_reaches_the_clip_entry(monkeypatch):
def worker(db, tmp_path, monkeypatch):
"""The worker path with its external programs replaced: ffmpeg writes the file it was given, the
probe answers what the test says, the phone-safe step and the poster pass through."""
- monkeypatch.setattr(settings, "download_root", str(tmp_path / "root"))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path / "root"))
seen: dict = {}
def ffmpeg(cmd, *_a, **_k):
diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py
index ed585e64..10ea902d 100644
--- a/backend/tests/test_download_failure_catalog.py
+++ b/backend/tests/test_download_failure_catalog.py
@@ -80,7 +80,7 @@ def _asset_and_job(db, user, *, sig, asset_status, job_status, error=None):
def _fail_the_child(monkeypatch, tmp_path, *, cookie: bool):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
cookiefile = tmp_path / "cookies.txt"
cookiefile.write_text("# Netscape HTTP Cookie File\n")
@@ -230,7 +230,7 @@ def test_a_download_yt_dlp_gave_no_duration_stores_the_files_duration(db, monkey
# row lost its trim action. Through the real finalize, with a real 3-second file.
u = _user(db, "fb-duration@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -350,7 +350,7 @@ def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeyp
# file name and the share preview all said "Video". The post's text is in the description.
u = _user(db, "fb-title@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -401,7 +401,7 @@ def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path):
# video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg).
u = _user(db, "fb-mp3@example.com")
a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running")
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
@@ -430,7 +430,7 @@ def test_a_video_download_is_still_made_browser_playable(db, monkeypatch, tmp_pa
a, j = _asset_and_job(
db, u, sig=f"mkv-{mode}", asset_status="downloading", job_status="running"
)
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs):
diff --git a/backend/tests/test_download_limits.py b/backend/tests/test_download_limits.py
index d2dfdc32..5ddbb948 100644
--- a/backend/tests/test_download_limits.py
+++ b/backend/tests/test_download_limits.py
@@ -259,7 +259,7 @@ def test_a_refusal_for_one_user_does_not_poison_the_shared_asset(db, monkeypatch
that shared row — correct for an upstream failure, wrong for "YOUR storage is full", which
`worker._process_job` would then hand verbatim to someone with an empty library.
"""
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
over, other = User(email="over@example.com"), User(email="other@example.com")
db.add_all([over, other])
db.flush()
@@ -306,7 +306,7 @@ def test_a_download_over_the_space_left_says_so_and_releases_the_shared_asset(
# yt-dlp refuses a file over `max_filesize` by saying so and returning with no file (pinned in
# test_formats). That is the owner's storage, not a processing fault: their refusal, in words
# true of a cap with space still left, and the shared asset goes back for the next job.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
u = User(email="nearcap@example.com")
db.add(u)
db.flush()
@@ -352,7 +352,7 @@ def test_a_download_that_outgrows_the_space_left_unannounced_is_refused_and_not_
# yt-dlp's `max_filesize` only stops a plain HTTP file that announces its size: YouTube's
# fragmented formats never meet it (measured on localdev: 718 MB fetched with 1 MB left). The
# file it produced is held to what is left by its real size, as a clip is.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
u = User(email="unannounced@example.com")
db.add(u)
db.flush()
diff --git a/backend/tests/test_edit_cap.py b/backend/tests/test_edit_cap.py
index 8f78c28d..fbc52efa 100644
--- a/backend/tests/test_edit_cap.py
+++ b/backend/tests/test_edit_cap.py
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg n
def test_an_editor_clip_larger_than_what_is_left_is_refused_and_not_placed(
db, make_user, tmp_path, monkeypatch
):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
src = tmp_path / "in.mp4"
subprocess.run(
[
diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py
index 431fc8d8..3ade4eee 100644
--- a/backend/tests/test_failure_lines.py
+++ b/backend/tests/test_failure_lines.py
@@ -258,7 +258,7 @@ def test_an_edit_that_crashes_is_said_once(sink, finalized, monkeypatch, tmp_pat
"""The line `_process_edit`'s handler used to log before `_fail_edit` logged it again."""
(tmp_path / "src.mp4").write_bytes(b"x")
monkeypatch.setattr(pipeline, "SessionLocal", _EditDb)
- monkeypatch.setattr(pipeline.settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(pipeline.settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline, "_edit_staging_dir", lambda asset_id: tmp_path / "staging")
def odd_spec(*a, **k):
@@ -300,7 +300,7 @@ def test_a_failed_download_line_does_not_put_back_what_the_adapter_took_out(sink
RuntimeError("dictionary changed size during iteration"),
RecursionError("maximum recursion depth exceeded"),
NotImplementedError(),
- FileNotFoundError(2, "No such file", "/downloads/Some_Channel/Season_2024/A_Private_Title"),
+ FileNotFoundError(2, "No such file", "/files/Some_Channel/Season_2024/A_Private_Title"),
],
ids=lambda e: type(e).__name__,
)
@@ -499,11 +499,9 @@ def test_a_failed_poster_or_storyboard_names_no_file(sink, tmp_path):
@pytest.mark.parametrize(
"err",
[
- subprocess.TimeoutExpired(
- ["ffmpeg", "-i", "/downloads/Some_Channel/A_Private_Title.mp4"], 30
- ),
- FileNotFoundError(2, "No such file", "/downloads/Some_Channel/A_Private_Title.jpg"),
- subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/downloads/Some_Channel/x.mp4"]),
+ subprocess.TimeoutExpired(["ffmpeg", "-i", "/files/Some_Channel/A_Private_Title.mp4"], 30),
+ FileNotFoundError(2, "No such file", "/files/Some_Channel/A_Private_Title.jpg"),
+ subprocess.CalledProcessError(1, ["ffmpeg", "-i", "/files/Some_Channel/x.mp4"]),
],
ids=lambda e: type(e).__name__,
)
@@ -523,9 +521,7 @@ def test_a_thumbnail_that_cannot_be_copied_names_no_file(sink, monkeypatch, tmp_
src.write_bytes(b"\xff\xd8\xff rest")
def refuse(a, b):
- raise PermissionError(
- 13, "Permission denied", "/downloads/Some_Channel/A_Private_Title.jpg"
- )
+ raise PermissionError(13, "Permission denied", "/files/Some_Channel/A_Private_Title.jpg")
monkeypatch.setattr(editmod.shutil, "copyfile", refuse)
assert editmod.thumbnail_to_jpg(src, tmp_path / "out.jpg") is None
diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py
new file mode 100644
index 00000000..7bd6681e
--- /dev/null
+++ b/backend/tests/test_file_folders.py
@@ -0,0 +1,263 @@
+"""The Files library's folders and a file's retention: which folder a new file lands in, when a done
+file goes, and the GC that takes it — warning once, never a file its user kept, never a file another
+user still holds.
+
+Pure-logic for the source-key rule; DB-backed for the rest.
+"""
+
+import hashlib
+from datetime import datetime, timedelta, timezone
+
+import pytest
+from sqlalchemy import func, select
+
+from app import sysconfig
+from app.config import settings
+from app.downloads import folders, formats, service
+from app.downloads import gc as gcmod
+from app.downloads.status import JobStatus
+from app.models import DownloadJob, FileFolder, MediaAsset, Notification
+
+NOW = datetime.now(timezone.utc)
+
+
+@pytest.mark.parametrize(
+ ("url", "key"),
+ [
+ ("https://www.tiktok.com/@a/video/1", "site:tiktok.com"),
+ ("https://m.facebook.com/groups/x/posts/2", "site:facebook.com"),
+ ("https://vimeo.com/123", "site:vimeo.com"),
+ ("https://youtu.be/abcdefghijk", "youtube"),
+ ("https://music.youtube.com/watch?v=abcdefghijk", "youtube"),
+ ("https://www.youtube.com/watch?v=abcdefghijk", "youtube"),
+ # A front door is only stripped when a real host remains behind it.
+ ("https://m.co/x", "site:m.co"),
+ ("not a url", "site:unknown"),
+ ],
+)
+def test_a_url_lands_in_its_sites_folder(url, key):
+ assert folders.site_key(url) == key
+
+
+@pytest.fixture
+def _root(tmp_path, monkeypatch):
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
+ return tmp_path
+
+
+def _asset(db, kind="youtube", ref="abcdefghijk", status="ready", rel="a.mp4", size=10):
+ asset = MediaAsset(
+ source_kind=kind,
+ source_ref=ref,
+ format_sig=hashlib.sha256(f"{kind}:{ref}".encode()).hexdigest()[:32],
+ status=status,
+ rel_path=rel,
+ size_bytes=size,
+ ref_count=0,
+ title="A title",
+ )
+ db.add(asset)
+ db.flush()
+ return asset
+
+
+def _job(db, user_id, asset, *, kind="youtube", ref=None, status=JobStatus.DONE, **kw):
+ job = DownloadJob(
+ user_id=user_id,
+ asset_id=asset.id if asset else None,
+ source_kind=kind,
+ source_ref=ref or (asset.source_ref if asset else "x"),
+ profile_snapshot={},
+ status=status,
+ **kw,
+ )
+ db.add(job)
+ db.flush()
+ folders.place(db, job)
+ if asset is not None:
+ service.incr_ref_count(db, asset.id)
+ db.commit()
+ return job
+
+
+def _folder(db, job) -> FileFolder:
+ return db.get(FileFolder, job.folder_id)
+
+
+def test_every_kind_of_job_lands_in_its_source_folder(db, make_user):
+ uid = make_user().id
+ yt = _job(db, uid, _asset(db))
+ site = _job(db, uid, _asset(db, "url", "https://www.tiktok.com/@a/video/1"), kind="url")
+ up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
+ plex = _job(
+ db, uid, _asset(db, "clip", "c1"), kind="clip", edit_spec={"source": "plex", "ref": "9"}
+ )
+ ytclip = _job(
+ db, uid, _asset(db, "clip", "c2"), kind="clip", edit_spec={"source": "youtube", "ref": "v"}
+ )
+ # An edit of a Plex clip: only its parent knows it came from Plex (the clip's asset says "clip").
+ edit = _job(
+ db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=plex.id,
+ source_asset_id=plex.asset_id,
+ ) # fmt: skip
+ keys = {j.id: _folder(db, j).source_key for j in (yt, site, up, plex, ytclip, edit)}
+ assert keys == {
+ yt.id: "youtube",
+ site.id: "site:tiktok.com",
+ up.id: "upload",
+ plex.id: "plex",
+ ytclip.id: "youtube",
+ edit.id: "plex", # an edit sits with the file it was cut from
+ }
+ # One folder per origin per user: the two YouTube files share theirs.
+ assert _folder(db, yt).id == _folder(db, ytclip).id
+
+
+def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
+ uid = make_user().id
+ src = _asset(db, "url", "https://vimeo.com/1")
+ edit = DownloadJob(
+ user_id=uid, source_kind="edit", source_ref="e", profile_snapshot={},
+ source_job_id=None, source_asset_id=src.id, status=JobStatus.DONE,
+ ) # fmt: skip
+ assert folders.source_key_for(db, edit) == "site:vimeo.com"
+
+
+def test_folders_are_per_user(db, make_user):
+ a, b = make_user().id, make_user().id
+ asset = _asset(db)
+ ja, jb = _job(db, a, asset), _job(db, b, asset)
+ assert ja.folder_id != jb.folder_id
+ assert (_folder(db, ja).user_id, _folder(db, jb).user_id) == (a, b)
+
+
+def test_a_source_folder_keeps_files_the_retention_days_and_uploads_forever(db, make_user):
+ sysconfig.set_value(db, "download_retention_days", 12)
+ db.commit()
+ uid = make_user().id
+ yt = _job(db, uid, _asset(db))
+ up = _job(db, uid, _asset(db, "upload", "u1"), kind="upload")
+ assert _folder(db, yt).retention_days == 12
+ assert _folder(db, up).retention_days is None
+
+
+def test_enqueue_places_the_job_and_a_cache_hit_is_stamped_at_once(db, make_user):
+ uid = make_user().id
+ spec = formats.normalize({})
+ first = service.enqueue(db, uid, "youtube", "abcdefghijk", spec)
+ assert _folder(db, first).source_key == "youtube"
+ assert first.expires_at is None # queued: its retention starts when it is done
+ asset = db.get(MediaAsset, first.asset_id)
+ asset.status, asset.rel_path = "ready", "a.mp4"
+ db.commit()
+ other = make_user().id
+ hit = service.enqueue(db, other, "youtube", "abcdefghijk", spec)
+ assert hit.status == JobStatus.DONE
+ days = _folder(db, hit).retention_days
+ assert hit.expires_at is not None
+ assert abs(hit.expires_at - (NOW + timedelta(days=days))) < timedelta(minutes=5)
+
+
+def test_a_finishing_worker_stamps_the_expiry_unless_kept(db, make_user):
+ uid = make_user().id
+ asset = _asset(db)
+ plain = _job(db, uid, asset, status=JobStatus.RUNNING)
+ kept = _job(db, uid, asset, status=JobStatus.RUNNING, kept=True)
+ for job in (plain, kept):
+ assert service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
+ db.commit()
+ db.refresh(plain)
+ db.refresh(kept)
+ assert plain.expires_at is not None
+ assert kept.expires_at is None
+
+
+def _gc(db):
+ return gcmod.run_download_gc(db)
+
+
+def _notices(db, uid, kind):
+ return db.execute(
+ select(func.count())
+ .select_from(Notification)
+ .where(Notification.user_id == uid, Notification.type == kind)
+ ).scalar()
+
+
+def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ uid = make_user().id
+ job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
+ _gc(db)
+ _gc(db)
+ assert _notices(db, uid, "download_expiring") == 1
+ job.expires_at = NOW - timedelta(minutes=1)
+ db.commit()
+ assert _gc(db)["expired"] == 1
+ assert db.get(DownloadJob, job.id) is None
+ assert _notices(db, uid, "download_expired") == 1
+ # Its last holder gone, the shared file and its row go too.
+ assert not (_root / "a.mp4").exists()
+ assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
+
+
+def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ uid = make_user().id
+ job = _job(db, uid, _asset(db), kept=True, expires_at=NOW - timedelta(days=1))
+ assert _gc(db)["expired"] == 0
+ assert db.get(DownloadJob, job.id) is not None
+
+
+def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
+ (_root / "a.mp4").write_bytes(b"x")
+ a, b = make_user().id, make_user().id
+ asset = _asset(db)
+ gone = _job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
+ stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
+ _gc(db)
+ assert db.get(DownloadJob, gone.id) is None
+ assert db.get(DownloadJob, stays.id) is not None
+ assert (_root / "a.mp4").exists()
+
+
+def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
+ db, make_user, _root
+):
+ sysconfig.set_value(db, "download_total_max_bytes", 15)
+ db.commit()
+ a, b = make_user().id, make_user().id
+ for name in ("old.mp4", "new.mp4"):
+ (_root / name).write_bytes(b"x")
+ old = _asset(db, ref="oldoldoldol", rel="old.mp4", size=10)
+ new = _asset(db, ref="newnewnewne", rel="new.mp4", size=10)
+ old.last_access_at = NOW - timedelta(days=9)
+ new.last_access_at = NOW - timedelta(days=1)
+ kept = _job(db, a, old, kept=True) # the older file, but kept: the newer one goes instead
+ ja, jb = _job(db, a, new), _job(db, b, new)
+ assert _gc(db)["evicted"] == 1
+ assert db.get(DownloadJob, kept.id) is not None
+ assert db.get(DownloadJob, ja.id) is None and db.get(DownloadJob, jb.id) is None
+ assert _notices(db, a, "download_evicted") == 1
+ assert _notices(db, b, "download_evicted") == 1
+ assert (_root / "old.mp4").exists() and not (_root / "new.mp4").exists()
+
+
+def test_deleting_a_user_takes_their_folders_and_files_with_them(db, make_user):
+ # Both tables cascade from users; the files' folder FK refuses deleting a folder that still holds
+ # files, which must not stop the account's own deletion.
+ user = make_user()
+ _job(db, user.id, _asset(db))
+ db.delete(user)
+ db.commit()
+ assert db.execute(select(func.count()).select_from(FileFolder)).scalar() == 0
+ assert db.execute(select(func.count()).select_from(DownloadJob)).scalar() == 0
+
+
+def test_the_longest_host_has_room_for_its_folder(db, make_user):
+ label = "a" * 63
+ host = ".".join([label, label, label, "b" * 58]) + ".hu" # 253 characters, the DNS maximum
+ assert len(host) == 253
+ uid = make_user().id
+ job = _job(db, uid, _asset(db, "url", f"https://{host}/v"), kind="url")
+ assert _folder(db, job).source_key == f"site:{host}"
diff --git a/backend/tests/test_migration_0102.py b/backend/tests/test_migration_0102.py
new file mode 100644
index 00000000..384bbcb7
--- /dev/null
+++ b/backend/tests/test_migration_0102.py
@@ -0,0 +1,73 @@
+"""Migration 0102's data step, DB-lane: every existing job lands in its source folder, and a done
+file's expiry moves from the shared asset to the user's job.
+
+The schema half is checked by test_migrations.py; a row placed in the wrong folder is invisible there.
+"""
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import text
+
+from app.db import engine
+
+
+def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user):
+ alice, bob = make_user().id, make_user().id
+ cfg = Config("alembic.ini")
+ command.downgrade(cfg, "0101_plex_text_plain")
+ try:
+ with engine.begin() as conn:
+
+ def asset(kind, ref, expires=None):
+ return conn.execute(
+ text(
+ "INSERT INTO media_assets (source_kind, source_ref, format_sig, status,"
+ " ref_count, expires_at) VALUES (:k, :r, :sig, 'ready', 1, :e)"
+ " RETURNING id"
+ ),
+ {"k": kind, "r": ref, "sig": kind + ref, "e": expires},
+ ).scalar_one()
+
+ def job(user, kind, ref, asset_id, status="done", spec=None, parent=None):
+ return conn.execute(
+ text(
+ "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
+ " profile_snapshot, status, edit_spec, source_job_id)"
+ " VALUES (:u, :a, :k, :r, '{}', :s, CAST(:spec AS json), :p) RETURNING id"
+ ),
+ {"u": user, "a": asset_id, "k": kind, "r": ref, "s": status,
+ "spec": spec, "p": parent},
+ ).scalar_one() # fmt: skip
+
+ yt = asset("youtube", "abcdefghijk", "2030-01-02T00:00:00+00:00")
+ tt = asset("url", "https://www.tiktok.com/@a/video/1", "2030-01-03T00:00:00+00:00")
+ up = asset("upload", "u1")
+ cl = asset("clip", "c1", "2030-01-04T00:00:00+00:00")
+ j_yt = job(alice, "youtube", "abcdefghijk", yt)
+ j_yt_bob = job(bob, "youtube", "abcdefghijk", yt, status="queued")
+ j_tt = job(alice, "url", "https://www.tiktok.com/@a/video/1", tt)
+ j_up = job(alice, "upload", "u1", up)
+ j_cl = job(alice, "clip", "c1", cl, spec='{"source": "plex", "ref": "9"}')
+ j_ed = job(alice, "edit", "e1", cl, parent=j_tt)
+ finally:
+ command.upgrade(cfg, "head")
+
+ with engine.connect() as conn:
+ rows = conn.execute(
+ text(
+ "SELECT j.id, f.user_id, f.source_key, f.retention_days, j.expires_at::date"
+ " FROM download_jobs j JOIN file_folders f ON f.id = j.folder_id"
+ )
+ ).all()
+ folders = conn.execute(text("SELECT count(*) FROM file_folders")).scalar()
+ got = {r[0]: (r[1], r[2], r[3], str(r[4]) if r[4] else None) for r in rows}
+ assert got == {
+ j_yt: (alice, "youtube", 30, "2030-01-02"),
+ j_yt_bob: (bob, "youtube", 30, None), # not done yet: no expiry until it is
+ j_tt: (alice, "site:tiktok.com", 30, "2030-01-03"),
+ j_up: (alice, "upload", None, None), # an upload never expires
+ j_cl: (alice, "plex", 30, "2030-01-04"),
+ j_ed: (alice, "site:tiktok.com", 30, "2030-01-04"),
+ }
+ # alice: youtube, tiktok, upload, plex; bob: youtube.
+ assert folders == 5
diff --git a/backend/tests/test_odd_frame.py b/backend/tests/test_odd_frame.py
index 13f13561..78b1cffe 100644
--- a/backend/tests/test_odd_frame.py
+++ b/backend/tests/test_odd_frame.py
@@ -46,7 +46,7 @@ def test_an_odd_sized_4_4_4_video_converts_to_the_even_frame_below_it(tmp_path):
def test_an_odd_crop_makes_a_clip_and_stores_the_frame_it_has(db, make_user, tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
src = tmp_path / "in.mp4"
_ffmpeg(
"-f", "lavfi", "-i", "testsrc2=size=160x120:rate=10:duration=2",
diff --git a/backend/tests/test_phone_safe_contract.py b/backend/tests/test_phone_safe_contract.py
index a15e1476..c40220fe 100644
--- a/backend/tests/test_phone_safe_contract.py
+++ b/backend/tests/test_phone_safe_contract.py
@@ -127,7 +127,7 @@ _FILES = {
def media_root(tmp_path, monkeypatch):
root = tmp_path / "media"
root.mkdir()
- monkeypatch.setattr(settings, "download_root", str(root))
+ monkeypatch.setattr(settings, "files_root", str(root))
return root
diff --git a/backend/tests/test_public_unlock_limit.py b/backend/tests/test_public_unlock_limit.py
index d4fc12d6..f5603cbc 100644
--- a/backend/tests/test_public_unlock_limit.py
+++ b/backend/tests/test_public_unlock_limit.py
@@ -19,7 +19,7 @@ _PW = "open sesame"
@pytest.fixture
def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
audio = {"codec_type": "audio", "codec_name": "aac"}
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
diff --git a/backend/tests/test_spa_entries.py b/backend/tests/test_spa_entries.py
index 7fab4d24..95837020 100644
--- a/backend/tests/test_spa_entries.py
+++ b/backend/tests/test_spa_entries.py
@@ -27,7 +27,7 @@ def pages(tmp_path, monkeypatch):
@pytest.fixture
def link_token(as_user, make_user, tmp_path, monkeypatch, upload_headers):
- monkeypatch.setattr(settings, "download_root", str(tmp_path / "media"))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path / "media"))
video = {"codec_type": "video", "codec_name": "h264", "pix_fmt": "yuv420p"}
audio = {"codec_type": "audio", "codec_name": "aac"}
data = {"streams": [video, audio], "format": {"format_name": "mov,mp4", "duration": "5"}}
diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py
index 108c26bc..d679e5ff 100644
--- a/backend/tests/test_storage.py
+++ b/backend/tests/test_storage.py
@@ -100,7 +100,7 @@ class TestDownloadFilename:
class TestSafeAbsPath:
- """The path-traversal guard: only paths that resolve INSIDE download_root AND exist are returned."""
+ """The path-traversal guard: only paths that resolve INSIDE files_root AND exist are returned."""
def test_returns_the_path_for_a_file_inside_the_root(self, tmp_path: Path):
(tmp_path / "sub").mkdir()
diff --git a/backend/tests/test_upload.py b/backend/tests/test_upload.py
index a240e67f..4d551232 100644
--- a/backend/tests/test_upload.py
+++ b/backend/tests/test_upload.py
@@ -5,7 +5,7 @@ generic file (force-download so uploaded HTML/SVG can't execute inline), quota a
size known up front, the per-file cap enforced while streaming, filename sanitization, and the two
guards that keep an upload out of paths that assume a re-fetchable source (edit / redownload).
-DB-backed (real Postgres) — skipped when no database is reachable. `settings.download_root` is
+DB-backed (real Postgres) — skipped when no database is reachable. `settings.files_root` is
pointed at a per-test tmp dir so ingest's on-disk moves are isolated. The ingest/route tests
monkeypatch the shared `probe.probe_json` so they don't shell out to ffprobe; the classifier's OWN
parsing (image → file, cover-art skip, dims, duration) is covered directly by `TestProbeClassifier`
@@ -51,7 +51,7 @@ def test_derive_media_kind():
@pytest.fixture
def _root(tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
@@ -761,7 +761,7 @@ def test_a_refused_upload_leaves_nothing_in_the_staging_directory(
sysconfig.set_value(db, "download_upload_max_bytes", 8)
db.commit()
- staging = Path(settings.download_root) / ".staging" / "uploads"
+ staging = Path(settings.files_root) / ".staging" / "uploads"
c = as_user(make_user())
r = c.post("/api/downloads/upload", content=b"x" * 4096, headers=upload_headers("leaky.bin"))
diff --git a/backend/tests/test_upload_conversion.py b/backend/tests/test_upload_conversion.py
index 8a8407f0..b4d84521 100644
--- a/backend/tests/test_upload_conversion.py
+++ b/backend/tests/test_upload_conversion.py
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
@pytest.fixture
def _root(tmp_path, monkeypatch):
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
return tmp_path
diff --git a/backend/tests/test_worker_dispatch.py b/backend/tests/test_worker_dispatch.py
index 8dc74b96..2593a306 100644
--- a/backend/tests/test_worker_dispatch.py
+++ b/backend/tests/test_worker_dispatch.py
@@ -32,7 +32,7 @@ def test_a_clip_the_phone_safe_step_changed_is_stored_under_its_new_extension(
):
# A fast trim stream-copies its source's container; when the phone-safe step remuxes it (an
# audio clip out of an .mp4 becomes an .m4a), the stored path and container follow the new file.
- monkeypatch.setattr(settings, "download_root", str(tmp_path))
+ monkeypatch.setattr(settings, "files_root", str(tmp_path))
data = {
"streams": [{"codec_type": "audio", "codec_name": "aac"}],
"format": {"format_name": "mov,mp4", "duration": "10"},
diff --git a/docker-compose.home.yml.example b/docker-compose.home.yml.example
index 69c40b84..fc30bfd8 100644
--- a/docker-compose.home.yml.example
+++ b/docker-compose.home.yml.example
@@ -17,7 +17,7 @@
# TOKEN_ENCRYPTION_KEY (Fernet), GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET,
# OAUTH_REDIRECT_URL=https://siftlode.example.com/auth/callback, ALLOWED_EMAILS, ADMIN_EMAILS,
# YOUTUBE_API_KEY. APP_VERSION is exported by the deploy step (selects the image tag).
-# Optional: DOWNLOAD_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir
+# Optional: FILES_HOST_PATH (host dir for the download tree) and PLEX_MEDIA_HOST_PATH (host dir
# holding the Plex library, mounted read-only for the optional Plex module).
name: siftlode
@@ -73,7 +73,7 @@ services:
SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips (read-only root is fine —
# it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -84,9 +84,9 @@ services:
# (10.x.x.x:8080) and on the trusted home LAN. Never forwarded to the public internet.
- "8080:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex media, read-only. Only used when the optional Plex module is enabled: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the Plex library (the local side
# of plex_path_map, default /plex-media). Left unset → an empty named volume (harmless; the
@@ -124,7 +124,7 @@ services:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode}
# The API owns the scheduler; the worker must not also run it.
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -137,7 +137,7 @@ services:
condition: service_started
networks: [internal]
volumes:
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex media (read-only) — same mount as the api; used by Plex clip jobs (a later phase).
- ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro
mem_limit: 2g
@@ -167,7 +167,7 @@ services:
volumes:
siftlode_pgdata:
- siftlode_downloads:
+ siftlode_files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
siftlode_plex_media:
diff --git a/docker-compose.localdev.yml b/docker-compose.localdev.yml
index 202ca5b8..c2be617e 100644
--- a/docker-compose.localdev.yml
+++ b/docker-compose.localdev.yml
@@ -21,21 +21,21 @@ services:
environment:
# This instance owns its scheduler (its own DB, so no double-write concern).
SCHEDULER_ENABLED: "true"
- # Plex HLS remux scratch on fast container-local storage (the /downloads bind-mount is a slow
+ # Plex HLS remux scratch on fast container-local storage (the /files bind-mount is a slow
# Windows Docker Desktop mount in dev; prod uses fast local disk so it keeps the default).
PLEX_HLS_DIR: /var/tmp/plex-hls
security_opt:
- apparmor:unconfined
volumes:
# Downloaded media lives on the host so you can inspect the generated Plex tree. Gitignored.
- - ./downloads:/downloads
+ - ./files:/files
- plex_media:/plex-media:ro
worker:
security_opt:
- apparmor:unconfined
volumes:
- - ./downloads:/downloads
+ - ./files:/files
- plex_media:/plex-media:ro
bgutil-pot:
diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml
index d3087d9d..2095000e 100644
--- a/docker-compose.selfhost.yml
+++ b/docker-compose.selfhost.yml
@@ -49,7 +49,7 @@ services:
SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
# the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -60,9 +60,9 @@ services:
# front for HTTPS / public exposure — and set OAUTH_REDIRECT_URL to the https URL.
- "${HTTP_PORT:-8080}:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the
# admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty
@@ -98,7 +98,7 @@ services:
environment:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-siftlode}
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -111,7 +111,7 @@ services:
condition: service_started
networks: [internal]
volumes:
- - ${DOWNLOAD_HOST_PATH:-siftlode_downloads}:/downloads
+ - ${FILES_HOST_PATH:-siftlode_files}:/files
# Same read-only Plex library as the API — the worker resolves library paths too.
- ${PLEX_MEDIA_HOST_PATH:-siftlode_plex_media}:/plex-media:ro
security_opt:
@@ -136,7 +136,7 @@ services:
volumes:
siftlode_pgdata:
- siftlode_downloads:
+ siftlode_files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
siftlode_plex_media:
diff --git a/docker-compose.yml b/docker-compose.yml
index fe760858..136fe5ca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
# the yt-dlp job loop.
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "false"
depends_on:
db:
@@ -45,9 +45,9 @@ services:
ports:
- "${APP_PORT:-8080}:8000"
volumes:
- # Downloaded media. Defaults to a Docker-managed named volume; set DOWNLOAD_HOST_PATH in .env
+ # Downloaded media. Defaults to a Docker-managed named volume; set FILES_HOST_PATH in .env
# to a host directory (e.g. one your Plex server reads) to keep the Plex-style tree there.
- - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads
+ - ${FILES_HOST_PATH:-files}:/files
# Plex library, read-only. The Plex module streams files itself, so it needs to SEE them: set
# PLEX_MEDIA_HOST_PATH in .env to the host directory holding the library (the local side of the
# admin "path map", whose default is /data=/plex-media). Left unset it falls back to an empty
@@ -80,7 +80,7 @@ services:
environment:
DATABASE_URL: postgresql+psycopg://${POSTGRES_USER:-siftlode}:${POSTGRES_PASSWORD:-siftlode}@db:5432/${POSTGRES_DB:-siftlode}
SCHEDULER_ENABLED: "false"
- DOWNLOAD_ROOT: /downloads
+ FILES_ROOT: /files
WORKER_ENABLED: "true"
depends_on:
db:
@@ -91,7 +91,7 @@ services:
bgutil-pot:
condition: service_started
volumes:
- - ${DOWNLOAD_HOST_PATH:-downloads}:/downloads
+ - ${FILES_HOST_PATH:-files}:/files
# Same read-only Plex library as the API — the worker resolves library paths too.
- ${PLEX_MEDIA_HOST_PATH:-plex_media}:/plex-media:ro
logging: *logging
@@ -107,6 +107,6 @@ services:
volumes:
pgdata:
- downloads:
+ files:
# Empty fallback for the Plex media mount when PLEX_MEDIA_HOST_PATH is unset (Plex module off).
plex_media:
diff --git a/docs/self-hosting.md b/docs/self-hosting.md
index b6265c73..1cb68b4d 100644
--- a/docs/self-hosting.md
+++ b/docs/self-hosting.md
@@ -105,12 +105,12 @@ them. It runs two extra containers that come up automatically — a `worker` (th
loop) and a small `bgutil-pot` sidecar (mints YouTube tokens so downloads aren't bot-blocked). No
configuration is required; per-user storage quotas are set on the admin **Downloads → System** page.
-By default the media lives in a Docker-managed volume (`siftlode_downloads`). To keep it somewhere
+By default the media lives in a Docker-managed volume (`siftlode_files`). To keep it somewhere
you can reach from other apps — e.g. a folder your **Plex** server indexes — point it at a host
directory by adding this to `.env` and re-running `up -d`:
```bash
-DOWNLOAD_HOST_PATH=/mnt/media/youtube
+FILES_HOST_PATH=/mnt/media/youtube
```
The directory must be writable by the container user (uid `1000`): `sudo chown -R 1000:1000 `.
@@ -164,8 +164,8 @@ then cannot read its own secrets.
token and every stored refresh token are encrypted with it *inside* the database. Restore a dump
against a different key and those values are lost — you would have to clear the affected rows from
`system_config` and re-enter every secret by hand. Keep `.env` beside the dump, and just as private.
-3. **Downloaded media** *(only if you use the Download Center)* — the `siftlode_downloads` volume, or
- whatever `DOWNLOAD_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their
+3. **Downloaded media** *(only if you use the Download Center)* — the `siftlode_files` volume, or
+ whatever `FILES_HOST_PATH` points at. Not needed to bring the instance back; the jobs and their
metadata are in the database, the files themselves are not.
Dump and restore the database with these two commands. They also exist as scripts
@@ -254,6 +254,18 @@ Then `pull` + `up -d` stays on that version until you change the line. Released
changed in them are listed in the app itself, under **Release Notes** in the account menu, and in
[`CHANGELOG.md`](../CHANGELOG.md).
+**The media mount was renamed** (the Files library): `DOWNLOAD_HOST_PATH` is now `FILES_HOST_PATH`,
+mounted at `/files`, and the default volume is `siftlode_files` instead of `siftlode_downloads`. If you
+set a host directory, rename the variable in `.env`. If you used the default volume, copy it over once
+before you start the new version (Docker cannot rename a volume), then remove the old one when the
+library looks right:
+
+```bash
+docker compose -f docker-compose.selfhost.yml down
+docker volume create siftlode_files
+docker run --rm -v siftlode_downloads:/from -v siftlode_files:/to alpine cp -a /from/. /to/
+```
+
**The registry keeps only the newest few releases, not every version the changelog lists.** A pin is
"decide when you take the next hop", not "stay here forever": once you are several releases behind,
your pinned tag can no longer be pulled — which matters the day you rebuild the host, prune your
diff --git a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
index 30833de2..f3ca2fc5 100644
--- a/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
+++ b/frontend/src/components/DownloadCenter.confirm.dom.test.tsx
@@ -74,6 +74,9 @@ const job = (id: number, status: DownloadJob["status"]): DownloadJob => ({
extra_links: [],
can_download: status === "done",
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
});
diff --git a/frontend/src/components/LibraryPlayer.dom.test.tsx b/frontend/src/components/LibraryPlayer.dom.test.tsx
index 1cae2383..17fbc086 100644
--- a/frontend/src/components/LibraryPlayer.dom.test.tsx
+++ b/frontend/src/components/LibraryPlayer.dom.test.tsx
@@ -54,6 +54,9 @@ const baseJob: DownloadJob = {
extra_links: [],
can_download: true,
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
};
@@ -73,7 +76,6 @@ const withAsset = (over: Partial>): DownloadJo
container: null,
media_kind: "video",
thumbnail_url: null,
- expires_at: null,
...over,
},
});
diff --git a/frontend/src/components/downloadRow.dom.test.tsx b/frontend/src/components/downloadRow.dom.test.tsx
index 39df6c0c..1acf0243 100644
--- a/frontend/src/components/downloadRow.dom.test.tsx
+++ b/frontend/src/components/downloadRow.dom.test.tsx
@@ -52,6 +52,9 @@ const baseJob: DownloadJob = {
extra_links: [],
can_download: true,
expired: false,
+ folder_id: null,
+ kept: false,
+ expires_at: null,
asset: null,
};
@@ -74,7 +77,6 @@ const withAsset = (
container,
media_kind: "video",
thumbnail_url: null,
- expires_at: null,
...over,
},
});
diff --git a/frontend/src/lib/api/downloads.ts b/frontend/src/lib/api/downloads.ts
index 805bf043..35cb5607 100644
--- a/frontend/src/lib/api/downloads.ts
+++ b/frontend/src/lib/api/downloads.ts
@@ -40,7 +40,6 @@ interface DownloadAsset {
// while still in flight (set at finalize). See backend downloads/mediakind.py.
media_kind: "video" | "audio" | "file" | null;
thumbnail_url: string | null;
- expires_at: string | null;
}
export type DownloadStatus = "queued" | "running" | "paused" | "done" | "error" | "canceled";
@@ -107,6 +106,11 @@ export interface DownloadJob {
edit_spec?: EditSpec | Omit | null;
can_download: boolean;
expired: boolean;
+ // The Files library (backend downloads/folders): its folder, whether the user keeps it past the
+ // folder's retention, and when it goes (null = never).
+ folder_id: number | null;
+ kept: boolean;
+ expires_at: string | null;
asset: DownloadAsset | null;
shared?: boolean;
user_email?: string;
diff --git a/install.ps1 b/install.ps1
index c3069dee..cf160749 100644
--- a/install.ps1
+++ b/install.ps1
@@ -31,7 +31,7 @@ OAUTH_REDIRECT_URL=$url/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 ).
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
"@ | Set-Content -Path .env -Encoding ascii
Write-Host "Wrote .env (secrets generated)."
}
diff --git a/install.sh b/install.sh
index c6ad65be..b275e9de 100755
--- a/install.sh
+++ b/install.sh
@@ -29,7 +29,7 @@ OAUTH_REDIRECT_URL=$URL/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 ).
-# DOWNLOAD_HOST_PATH=/mnt/media/youtube
+# FILES_HOST_PATH=/mnt/media/youtube
EOF
chmod 600 .env
echo "Wrote .env (secrets generated)."