From 815083bcc4cfef427087bc1b6599943a2ff85609 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Tue, 13 Oct 2026 00:14:09 +0200 Subject: [PATCH] =?UTF-8?q?fix(files):=20review=20round=201,=20the=20serve?= =?UTF-8?q?r=20=E2=80=94=20the=20tray's=20window,=20the=20GC's=20locks,=20?= =?UTF-8?q?a=20file's=20origin,=20the=20upgrade=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The queue keeps a done job by when it became done (ready_at), not by its last write: renaming, keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished. - GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a second release that frees a file another user still holds. - A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit whose parent expired still goes back to its folder, not to Uploads. - A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes that file home too, instead of failing on the foreign key. - The hash route reads at most two files at once (429 past that) and holds no connection meanwhile. - 0104's backfill dates old files by their request date only: no notice prod wrote names its job. - The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up. - Stale comments about the /downloads mount and the Downloads page. --- .env.example | 6 +- backend/alembic/versions/0104_job_ready_at.py | 12 +- .../alembic/versions/0105_job_source_key.py | 37 +++++ backend/app/config.py | 10 +- backend/app/downloads/folders.py | 39 +++-- backend/app/downloads/gc.py | 82 ++++++++--- backend/app/downloads/service.py | 2 +- backend/app/downloads/storage.py | 20 +++ backend/app/main.py | 5 + backend/app/models.py | 7 +- backend/app/routes/downloads.py | 6 +- backend/app/routes/files.py | 40 ++++-- backend/app/worker.py | 8 +- backend/tests/test_file_folders.py | 133 ++++++++++++++++++ backend/tests/test_files_api.py | 56 +++++++- backend/tests/test_migration_0102.py | 68 +++++++++ backend/tests/test_migration_0104.py | 20 +-- backend/tests/test_migration_0105.py | 45 ++++++ backend/tests/test_storage.py | 22 +++ backend/tests/test_worker_knobs.py | 15 ++ docker-compose.home.yml.example | 4 +- docker-compose.selfhost.yml | 4 +- docker-compose.yml | 2 +- docs/self-hosting.md | 36 +++-- .../src/components/{ => ui}/RenameField.tsx | 0 frontend/src/i18n/locales/en/errors.json | 1 + frontend/src/i18n/locales/hu/errors.json | 1 + install.ps1 | 2 +- install.sh | 2 +- 29 files changed, 593 insertions(+), 92 deletions(-) create mode 100644 backend/alembic/versions/0105_job_source_key.py create mode 100644 backend/tests/test_migration_0105.py rename frontend/src/components/{ => ui}/RenameField.tsx (100%) diff --git a/.env.example b/.env.example index 95b3ad55..376f9315 100644 --- a/.env.example +++ b/.env.example @@ -73,10 +73,10 @@ SMTP_FROM= # of them and false on the rest, to avoid double quota use and write races. SCHEDULER_ENABLED=true -# --- Download center --- -# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small +# --- Files library & downloads --- +# Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small # `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose. -# Downloaded media defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host +# The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host # directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there. # The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 . # FILES_HOST_PATH=/mnt/media/youtube diff --git a/backend/alembic/versions/0104_job_ready_at.py b/backend/alembic/versions/0104_job_ready_at.py index 4f6ba8f2..3be74ad3 100644 --- a/backend/alembic/versions/0104_job_ready_at.py +++ b/backend/alembic/versions/0104_job_ready_at.py @@ -8,9 +8,8 @@ Create Date: 2026-10-12 download that failed for days and then went through used to sit days down the list, under its request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`. -A done job from before has no record of that moment, except where its "ready to watch" notice is -still in the owner's inbox (read notices are pruned past a cap): that notice was written the moment -it finished. The rest take the request date, which is where they already sat. +A done job from before has no record of that moment (its "ready to watch" notice never named the +job, so it cannot be matched back), so it takes its request date, which is where it already sat. """ import sqlalchemy as sa @@ -26,12 +25,7 @@ def upgrade() -> None: op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True))) op.execute( """ - UPDATE download_jobs j SET ready_at = COALESCE( - (SELECT max(n.created_at) FROM notifications n - WHERE n.type = 'download_done' AND n.user_id = j.user_id - AND n.data->>'job_id' = j.id::text), - j.created_at) - WHERE j.status = 'done' + UPDATE download_jobs SET ready_at = created_at WHERE status = 'done' """ ) op.create_check_constraint( diff --git a/backend/alembic/versions/0105_job_source_key.py b/backend/alembic/versions/0105_job_source_key.py new file mode 100644 index 00000000..6f9cc019 --- /dev/null +++ b/backend/alembic/versions/0105_job_source_key.py @@ -0,0 +1,37 @@ +"""A file remembers where it came from + +Revision ID: 0105_job_source_key +Revises: 0104_job_ready_at +Create Date: 2026-10-12 + +`download_jobs.source_key` is the origin a file came from — the source folder it may go back to — +recorded when it is placed. An edit used to work its origin out through its parent each time, and +once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could +no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever. + +A file in a source folder takes that folder's key. One already in an own folder is left NULL and +keeps being worked out (`folders.source_key_for`), as before. +""" + +import sqlalchemy as sa +from alembic import op + +revision: str = "0105_job_source_key" +down_revision: str | None = "0104_job_ready_at" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.add_column("download_jobs", sa.Column("source_key", sa.String(260))) + op.execute( + """ + UPDATE download_jobs j SET source_key = f.source_key + FROM file_folders f + WHERE f.id = j.folder_id AND f.kind = 'source' + """ + ) + + +def downgrade() -> None: + op.drop_column("download_jobs", "source_key") diff --git a/backend/app/config.py b/backend/app/config.py index 4007159e..fd477645 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import Literal, get_args, get_origin -from pydantic import Field, ValidationInfo, field_validator, model_validator +from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict # Shipped placeholder; production must override it. Used by the startup guard below. @@ -171,9 +171,13 @@ class Settings(BaseSettings): feed_default_hidden_live: str = "live,upcoming" # --- Download center (yt-dlp) --- - # Root directory (inside the container) for downloaded media — a Docker volume/bind mount. + # Root directory (inside the container) for the files library — a Docker volume/bind mount. # Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it. - files_root: str = "/files" + # DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at + # /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image. + files_root: str = Field( + default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root") + ) # Whether THIS process runs the download worker loop. On for the dedicated worker # container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled. worker_enabled: bool = False diff --git a/backend/app/downloads/folders.py b/backend/app/downloads/folders.py index cbc32733..570d9598 100644 --- a/backend/app/downloads/folders.py +++ b/backend/app/downloads/folders.py @@ -68,9 +68,11 @@ def site_key(url: str) -> str: def source_key_for(db: Session, job: DownloadJob) -> str: """The origin a job's file came from — the source folder it belongs in. - youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL → - its site; an edit → wherever its parent's file came from (its parent job, or, once that is - deleted, the parent's asset).""" + The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source + (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file + came from (its parent job, or, once that is deleted, the parent's asset).""" + if job.source_key: + return job.source_key kind = job.source_kind if kind == "youtube": return SourceKey.YOUTUBE @@ -122,9 +124,11 @@ def source_folder(db: Session, user_id: int, key: str) -> FileFolder: def place(db: Session, job: DownloadJob) -> None: - """Put a new job's file in its source folder. Called wherever a job is created; the job must - carry its `user_id`, `source_kind` and source refs already.""" - job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id + """Put a new job's file in its source folder, and record that origin on it. Called wherever a + job is created; the job must carry its `user_id`, `source_kind` and source refs already.""" + key = source_key_for(db, job) + job.source_key = key + job.folder_id = source_folder(db, job.user_id, key).id def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None: @@ -151,12 +155,25 @@ class FolderError(Exception): self.reason = reason -def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder: - """The folder an upload asked to land in, if it may: one of the user's own folders, or their - Uploads folder. A file put in a site's folder would claim an origin it does not have.""" - folder = db.get(FileFolder, folder_id) +def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: + """The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the + caller's transaction ends, so a delete of the folder waits for the file and then takes it home + with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False` + only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's + folder answers like a missing one.""" + stmt = select(FileFolder).where(FileFolder.id == folder_id) + if hold: + stmt = stmt.with_for_update(read=True) + folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none() if folder is None or folder.user_id != user_id: raise FolderError("file_folder_gone") + return folder + + +def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder: + """The folder an upload asked to land in, if it may: one of the user's own folders, or their + Uploads folder. A file put in a site's folder would claim an origin it does not have.""" + folder = own_folder(db, user_id, folder_id, hold=hold) if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: raise FolderError("file_folder_not_its_source") return folder @@ -219,6 +236,8 @@ def delete_own(db: Session, folder: FileFolder) -> int: folder it came from, its clock starting there now. Returns how many moved.""" if folder.kind != FolderKind.OWN: raise FolderError("file_folder_not_own") + # Wait out a file being put in it right now (`own_folder`), then take that one home too. + db.execute(select(FileFolder.id).where(FileFolder.id == folder.id).with_for_update()) jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all() for job in jobs: job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id diff --git a/backend/app/downloads/gc.py b/backend/app/downloads/gc.py index 6b22c2ea..981c4b85 100644 --- a/backend/app/downloads/gc.py +++ b/backend/app/downloads/gc.py @@ -56,6 +56,32 @@ def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str) ) +def _relock_expired( + db: Session, job_id: int, asset_id: int | None, now: datetime +) -> DownloadJob | None: + """Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the + asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a + user may have deleted or re-downloaded it (dropping it again would release its asset a second + time, freeing a file another job still holds) or kept or moved it (it is no longer expired).""" + if asset_id is not None: + db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update()) + job = db.execute( + select(DownloadJob) + .where(DownloadJob.id == job_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() + still = ( + job is not None + and job.asset_id == asset_id + and job.status == JobStatus.DONE + and not job.kept + and job.expires_at is not None + and job.expires_at <= now + ) + return job if still else None + + def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None: """Take a file out of its user's library: tell them, release the job's hold on the asset (the file goes with the last holder), delete the row (its shares and links cascade).""" @@ -176,22 +202,22 @@ def run_download_gc(db: Session) -> dict: db.commit() # 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1). - dead = ( - db.execute( - select(DownloadJob).where( - DownloadJob.status == JobStatus.DONE, - DownloadJob.kept.is_(False), - DownloadJob.expires_at.is_not(None), - DownloadJob.expires_at <= now, - ) + # Each file is re-checked under its locks and committed on its own: the release unlinks the + # file at once, so it must never act on what the SELECT saw a while ago. + dead = db.execute( + select(DownloadJob.id, DownloadJob.asset_id).where( + DownloadJob.status == JobStatus.DONE, + DownloadJob.kept.is_(False), + DownloadJob.expires_at.is_not(None), + DownloadJob.expires_at <= now, ) - .scalars() - .all() - ) - for job in dead: - _drop_job(db, job, "download_expired") - expired += 1 - if dead: + ).all() + db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below + for job_id, asset_id in dead: + job = _relock_expired(db, job_id, asset_id, now) + if job is not None: + _drop_job(db, job, "download_expired") + expired += 1 db.commit() # 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to @@ -221,15 +247,33 @@ def run_download_gc(db: Session) -> dict: .scalars() .all() ) - for asset in candidates: + candidate_ids = [a.id for a in candidates] + db.rollback() + for asset_id in candidate_ids: if total <= cap: break - total -= asset.size_bytes or 0 + # Locked and re-read like an expiry: a holder may have kept it, or every holder + # deleted it (the file is then already gone and counted out) since the SELECT. + asset = db.execute( + select(MediaAsset) + .where(MediaAsset.id == asset_id) + .with_for_update() + .execution_options(populate_existing=True) + ).scalar_one_or_none() holders = ( - db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id)) + db.execute( + select(DownloadJob) + .where(DownloadJob.asset_id == asset_id) + .with_for_update() + .execution_options(populate_existing=True) + ) .scalars() .all() ) + if asset is None or asset.status != "ready" or any(j.kept for j in holders): + db.commit() + continue + total -= asset.size_bytes or 0 for job in holders: _notify(db, job, asset, "download_evicted") db.delete(job) @@ -237,7 +281,7 @@ def run_download_gc(db: Session) -> dict: storage.delete_asset_files(settings.files_root, asset.rel_path) db.delete(asset) evicted += 1 - db.commit() + db.commit() # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still- # referenced errored row to pending on re-enqueue, so these only accumulate once every diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index 691dd944..3da8fd6b 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -49,7 +49,7 @@ def source_url(source_kind: str, source_ref: str) -> str: def reference_url(job, asset) -> str | None: - """The clean "downloaded from" URL for a job — shared by the private Downloads page and the public + """The clean "downloaded from" URL for a job — shared by the Files library and the public watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's own earlier download, not a web page) and for a Plex clip (the library is not a public page); a YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py index d2ec540e..8a003117 100644 --- a/backend/app/downloads/storage.py +++ b/backend/app/downloads/storage.py @@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide date-based — the community-standard mapping for YouTube-in-Plex. """ +import os import re import shutil import unicodedata @@ -67,6 +68,25 @@ _FOLD_MAP = str.maketrans( ) +def files_root_problem(root: str, system_root: str = "/") -> str | None: + """Why `root` cannot be the files library's volume, or None when it can. A root that sits on the + container's own filesystem is the image's empty directory, not a mounted volume: the library + would answer 404 for every file, and what a download writes would go when the container does — + the state an install is in when its compose file still mounts the library somewhere else.""" + path = Path(root) + if not path.is_dir(): + return f"{root} does not exist" + try: + if path.stat().st_dev == Path(system_root).stat().st_dev: + return f"{root} is not a mounted volume (it is the container's own empty directory)" + # silent-ok: the failure IS the answer — returned as the reason, which the caller logs. + except OSError as e: + return f"{root} cannot be read ({e})" + if not os.access(path, os.W_OK): + return f"{root} is not writable" + return None + + def _ascii_fold(text: str) -> str: """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have diff --git a/backend/app/main.py b/backend/app/main.py index 0da3d6c2..cd7cbba2 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -28,6 +28,7 @@ log = logs.get_logger(__name__) from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first from app.config import settings # noqa: E402 — logging is configured first from app.db import SessionLocal, engine # noqa: E402 — logging is configured first +from app.downloads import storage # noqa: E402 — logging is configured first from app.plex import stream as plex_stream # noqa: E402 — logging is configured first from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first from app.routes import ( # noqa: E402 — logging is configured first @@ -64,6 +65,10 @@ from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — @asynccontextmanager async def lifespan(app: FastAPI): log.info("Siftlode starting up") + problem = storage.files_root_problem(settings.files_root) + if problem: + # The rest of the app still serves; the files library cannot (the worker refuses to start). + log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem) # The stored copy of the log (log_events). The entrypoint has already migrated the schema. logsink.start() # First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print diff --git a/backend/app/models.py b/backend/app/models.py index 242db29d..6fe9726f 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -877,7 +877,7 @@ class MediaAsset(Base, TimestampMixin): upload_date: Mapped[date | None] = mapped_column(Date) thumbnail_url: Mapped[str | None] = mapped_column(Text) # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" - # reference shown on the Downloads page. Null until the worker extracts it (older/queued rows + # reference shown in the Files library. Null until the worker extracts it (older/queued rows # fall back to a URL derived from source_kind+source_ref). source_webpage_url: Mapped[str | None] = mapped_column(Text) error: Mapped[str | None] = mapped_column(Text) @@ -1036,6 +1036,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin): folder_id: Mapped[int | None] = mapped_column( ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True ) + # The origin the file came from (the source folder's key), recorded where it is placed: an edit + # traces its origin through its parent, and that parent may be deleted or expire later — the + # file must still know which source folder it may go back to. NULL only for a file placed before + # the column existed outside a source folder; `folders.source_key_for` works those out. + source_key: Mapped[str | None] = mapped_column(String(260)) kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") diff --git a/backend/app/routes/downloads.py b/backend/app/routes/downloads.py index 6870cda7..931df5e6 100644 --- a/backend/app/routes/downloads.py +++ b/backend/app/routes/downloads.py @@ -399,6 +399,8 @@ def enqueue_clip( # How long a finished job stays in the queue: the tray polls it from every page and must see a job # turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's. +# A done job is timed from when it became done, never from its last write: a rename, a move or the +# GC's expiry warning touches an old file's row, and the tray would announce it as just finished. QUEUE_RECENT = timedelta(minutes=10) @@ -411,7 +413,9 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get select(DownloadJob) .where( DownloadJob.user_id == user.id, - DownloadJob.status.not_in(FINISHED) | (DownloadJob.updated_at >= recent), + DownloadJob.status.not_in(FINISHED) + | ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent)) + | ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)), ) .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) .limit(MAX_LIST) diff --git a/backend/app/routes/files.py b/backend/app/routes/files.py index 7c03aadc..3c67f709 100644 --- a/backend/app/routes/files.py +++ b/backend/app/routes/files.py @@ -13,6 +13,7 @@ import base64 import hashlib import os import tempfile +import threading from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass from datetime import datetime, timezone @@ -23,7 +24,7 @@ from urllib.parse import unquote, urlparse from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import FileResponse, Response from pydantic import BaseModel, Field -from sqlalchemy import func, or_, select +from sqlalchemy import func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -59,6 +60,9 @@ MAX_LIST = 500 # The longest a folder may keep a file, in days — ten years is "forever" in all but name. _MAX_RETENTION_DAYS = 3650 _HASH_CHUNK = 1 << 20 +# How many files the hash route reads at once: each holds a threadpool thread for the whole read. +_HASHERS = 2 +_HASHING = threading.BoundedSemaphore(_HASHERS) # --- shared with routes.downloads ------------------------------------------------------- @@ -235,7 +239,7 @@ def _upload_folder(db: Session, user_id: int, raw: str) -> int | None: if not raw.strip(): return None try: - return folders.upload_target(db, user_id, int(raw)).id + return folders.upload_target(db, user_id, int(raw), hold=False).id except ValueError as e: raise refused(409, "file_folder_gone") from e except folders.FolderError as e: @@ -664,12 +668,9 @@ def update_file( if "folder_id" in sent: if body.folder_id is None: raise refused(400, "file_folder_gone") - folder = db.get(FileFolder, body.folder_id) - if folder is None or folder.user_id != user.id: - raise refused(409, "file_folder_gone") before = job.folder_id try: - folders.move(db, job, folder) + folders.move(db, job, folders.own_folder(db, user.id, body.folder_id)) except folders.FolderError as e: raise refused(409, e.reason) from e if job.folder_id != before: @@ -842,13 +843,30 @@ def get_sha256( job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: """The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder. - A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool.""" + A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and + every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller + past that is told to come back), and none of them holds a database connection while it reads.""" asset, path = _ready_file(db, accessible_job(db, user, job_id)) - if not asset.sha256: + if asset.sha256: + return {"sha256": asset.sha256} + asset_id = asset.id + if not _HASHING.acquire(blocking=False): + raise refused(429, "file_hash_busy") + try: + release_read_connection(db) h = hashlib.sha256() with path.open("rb") as f: while chunk := f.read(_HASH_CHUNK): h.update(chunk) - asset.sha256 = h.hexdigest() - db.commit() - return {"sha256": asset.sha256} + except OSError as e: + raise not_found("file") from e + finally: + _HASHING.release() + digest = h.hexdigest() + db.execute( + update(MediaAsset) + .where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None)) + .values(sha256=digest) + ) + db.commit() + return {"sha256": digest} diff --git a/backend/app/worker.py b/backend/app/worker.py index 5f16d46b..b8180456 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -37,7 +37,7 @@ from app import sysconfig from app.config import settings from app.db import SessionLocal from app.downloads import edit as editmod -from app.downloads import credentials, formats, quota, service +from app.downloads import credentials, formats, quota, service, storage from app.downloads import errors as dlerrors from app.downloads.jobdb import ( _edit_staging_dir, @@ -710,6 +710,12 @@ def main() -> None: # `log_events`, so on the deploy that first adds the table the worker may start before the API # has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks. logsink.start() + problem = storage.files_root_problem(settings.files_root) + if problem: + # Refuse, loudly: downloading into a root that is not the volume loses every file with the + # container. The queue waits until the mount is fixed (see docs/self-hosting.md). + log.critical("The files library cannot be written: %s. The worker will not start.", problem) + raise SystemExit(1) _recover_orphans() global _target _target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer diff --git a/backend/tests/test_file_folders.py b/backend/tests/test_file_folders.py index 94e245c3..4c1b068f 100644 --- a/backend/tests/test_file_folders.py +++ b/backend/tests/test_file_folders.py @@ -15,6 +15,7 @@ from app import sysconfig from app.config import settings from app.downloads import folders, formats, service from app.downloads import gc as gcmod +from app.db import SessionLocal from app.downloads.status import JobStatus from app.models import DownloadJob, FileFolder, MediaAsset, Notification @@ -123,6 +124,68 @@ def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user): assert folders.source_key_for(db, edit) == "site:vimeo.com" +def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user): + # The parent download expires, its file goes with its last holder, and both of the edit's + # pointers to it are cleared: the edit must still go back to the YouTube folder it came from. + uid = make_user().id + parent = _job(db, uid, _asset(db)) + edit = _job( + db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id, + source_asset_id=parent.asset_id, + ) # fmt: skip + home = _folder(db, edit) + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.flush() + folders.move(db, edit, own) + edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves + db.delete(parent) + db.commit() + folders.move(db, edit, home) + assert edit.folder_id == home.id + folders.move(db, edit, own) + folders.delete_own(db, own) + assert edit.folder_id == home.id + + +def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user): + # Before, the delete missed the file still on its way in and failed on the foreign key (a 500), + # or the file's insert did after the folder went. + import threading + + uid = make_user().id + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.commit() + own_id = own.id + job = _job(db, uid, _asset(db)) + home = job.folder_id + target = folders.own_folder(db, uid, own_id) # held from the check until this commits + outcome: list[object] = [] + + def delete_it(): + other = SessionLocal() + try: + outcome.append(folders.delete_own(other, other.get(FileFolder, own_id))) + other.commit() + except Exception as e: # noqa: BLE001 — the assertion below names what went wrong + outcome.append(e) + finally: + other.close() + + t = threading.Thread(target=delete_it) + t.start() + t.join(timeout=1) + assert t.is_alive() # the delete waits for the file, even before it is written + folders.move(db, job, target) + db.commit() + t.join(timeout=10) + assert outcome == [1] + db.expire_all() + assert db.get(DownloadJob, job.id).folder_id == home + assert db.get(FileFolder, own_id) is None + + def test_folders_are_per_user(db, make_user): a, b = make_user().id, make_user().id asset = _asset(db) @@ -201,6 +264,30 @@ def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user, assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0 +@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"]) +def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart): + # The GC warns once per expiry; a file whose expiry was moved must be warned before the new one. + (_root / "a.mp4").write_bytes(b"x") + uid = make_user().id + job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12)) + _gc(db) + assert job.expiry_warned + if restart == "unkeep": + folders.set_kept(db, job, True) + folders.set_kept(db, job, False) + elif restart == "move": + home = _folder(db, job) + own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine") + db.add(own) + db.flush() + folders.move(db, job, own) + folders.move(db, job, home) + else: + folders.set_retention(db, _folder(db, job), 90) + db.commit() + assert not job.expiry_warned + + def test_the_gc_never_takes_a_kept_file(db, make_user, _root): (_root / "a.mp4").write_bytes(b"x") uid = make_user().id @@ -221,6 +308,52 @@ def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root): assert (_root / "a.mp4").exists() +def _meanwhile(monkeypatch, act): + """Run `act` in another session (another request) between the GC's expiry SELECT and its + handling of each file it found — the window a user's click can land in.""" + real = gcmod._relock_expired + + def relock(db, job_id, asset_id, now): + other = SessionLocal() + try: + act(other, job_id) + other.commit() + finally: + other.close() + return real(db, job_id, asset_id, now) + + monkeypatch.setattr(gcmod, "_relock_expired", relock) + + +def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch): + (_root / "a.mp4").write_bytes(b"x") + job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1)) + _meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True)) + assert _gc(db)["expired"] == 0 + db.expire_all() + assert db.get(DownloadJob, job.id) is not None + assert (_root / "a.mp4").exists() + + +def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch): + # Released twice, the shared file would go from under the other user's copy. + (_root / "a.mp4").write_bytes(b"x") + a, b = make_user().id, make_user().id + asset = _asset(db) + _job(db, a, asset, expires_at=NOW - timedelta(minutes=1)) + stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20)) + + def user_deletes(other, jid): + service.delete(other, other.get(DownloadJob, jid)) + + _meanwhile(monkeypatch, user_deletes) + _gc(db) + db.expire_all() + assert db.get(DownloadJob, stays.id).asset_id == asset.id + assert db.get(MediaAsset, asset.id).ref_count == 1 + assert (_root / "a.mp4").exists() + + def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library( db, make_user, _root ): diff --git a/backend/tests/test_files_api.py b/backend/tests/test_files_api.py index b02a46bd..5a3ddf3e 100644 --- a/backend/tests/test_files_api.py +++ b/backend/tests/test_files_api.py @@ -421,6 +421,21 @@ def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as assert (row.actor_id, row.target_id) == (alice.id, str(job.id)) +def test_a_file_shared_with_someone_is_theirs_to_watch_not_to_change(db, make_user, as_user, _root): + # The read routes let a share's recipient in; the write routes must not. + alice, friend = make_user(), make_user() + job = _file(db, alice.id, title="Mine") + db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id)) + db.commit() + c = as_user(friend) + assert c.patch(f"/api/files/{job.id}", json={"name": "Theirs"}).status_code == 404 + assert c.patch(f"/api/files/{job.id}", json={"kept": True}).status_code == 404 + assert c.delete(f"/api/files/{job.id}").status_code == 404 + db.expire_all() + still = db.get(DownloadJob, job.id) + assert still is not None and still.display_name is None and not still.kept + + # --- upload, the file, its hash ------------------------------------------------------------ @@ -532,6 +547,26 @@ def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404 +def test_only_a_few_files_are_hashed_at_once(db, make_user, as_user, _root): + # Each hash holds a thread of the pool every sync route shares for the whole read; a caller + # past the few that may run is told to come back, not queued behind them. + from app.routes import files as filesroute + + alice = make_user() + job = _file(db, alice.id, rel="v.mp4") + (_root / "v.mp4").write_bytes(b"bytes") + held = [filesroute._HASHING.acquire(blocking=False) for _ in range(filesroute._HASHERS)] + try: + assert all(held) + busy = as_user(alice).get(f"/api/files/{job.id}/sha256") + finally: + for _ in held: + filesroute._HASHING.release() + assert busy.status_code == 429 + assert busy.json()["detail"]["reason"] == "file_hash_busy" + assert as_user(alice).get(f"/api/files/{job.id}/sha256").status_code == 200 + + def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user): job = _file(db, make_user().id) asset = db.get(MediaAsset, job.asset_id) @@ -562,7 +597,7 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library( db.execute( update(DownloadJob) .where(DownloadJob.id == job.id) - .values(updated_at=NOW - timedelta(hours=1)) + .values(updated_at=NOW - timedelta(hours=1), ready_at=NOW - timedelta(hours=1)) ) db.commit() rows = as_user(user).get("/api/downloads").json() @@ -570,3 +605,22 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library( assert {row["id"] for row in rows} == {fresh.id, waiting.id, stuck.id, paused.id, stopped.id} # The tray names where each came from, as the library does. assert {row["source_key"] for row in rows} == {"youtube"} + + +def test_an_old_file_written_to_does_not_come_back_as_just_finished(db, make_user, as_user): + # Renaming, keeping or moving touches the row; the tray would announce the file as ready again. + from sqlalchemy import update + + user = make_user() + old = _file(db, user.id, title="Old") + db.execute( + update(DownloadJob) + .where(DownloadJob.id == old.id) + .values(updated_at=NOW - timedelta(days=30), ready_at=NOW - timedelta(days=30)) + ) + db.commit() + c = as_user(user) + assert ( + c.patch(f"/api/files/{old.id}", json={"name": "Renamed", "kept": True}).status_code == 200 + ) + assert [row["id"] for row in c.get("/api/downloads").json()] == [] diff --git a/backend/tests/test_migration_0102.py b/backend/tests/test_migration_0102.py index 384bbcb7..a2a40512 100644 --- a/backend/tests/test_migration_0102.py +++ b/backend/tests/test_migration_0102.py @@ -71,3 +71,71 @@ def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user): } # alice: youtube, tiktok, upload, plex; bob: youtube. assert folders == 5 + + +def test_the_admins_retention_is_kept_and_the_downgrade_gives_the_asset_back_its_expiry( + db, make_user +): + alice, bob = make_user().id, make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0101_plex_text_plain") + try: + with engine.begin() as conn: + conn.execute( + text( + "INSERT INTO system_config (key, value) VALUES ('download_retention_days', '7')" + " ON CONFLICT (key) DO UPDATE SET value = '7'" + ) + ) + yt = conn.execute( + text( + "INSERT INTO media_assets (source_kind, source_ref, format_sig, status," + " ref_count, expires_at) VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 2," + " '2030-01-02T00:00:00+00:00') RETURNING id" + ) + ).scalar_one() + for user in (alice, bob): + conn.execute( + text( + "INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref," + " profile_snapshot, status) VALUES (:u, :a, 'youtube', 'abcdefghijk'," + " '{}', 'done')" + ), + {"u": user, "a": yt}, + ) + command.upgrade(cfg, "head") + with engine.connect() as conn: + retention = conn.execute( + text( + "SELECT DISTINCT retention_days FROM file_folders WHERE source_key = 'youtube'" + ) + ).all() + + def asset_expiry_after_downgrade(alice_at, bob_at): + with engine.begin() as conn: + for user, at in ((alice, alice_at), (bob, bob_at)): + conn.execute( + text( + "UPDATE download_jobs SET expires_at = CAST(:at AS timestamptz)" + " WHERE asset_id = :a AND user_id = :u" + ), + {"at": at, "a": yt, "u": user}, + ) + command.downgrade(cfg, "0101_plex_text_plain") + with engine.connect() as conn: + got = conn.execute( + text("SELECT expires_at::date FROM media_assets WHERE id = :a"), {"a": yt} + ).scalar() + command.upgrade(cfg, "head") + return str(got) if got else None + + # The shared file goes when its last holder's copy would; one holder keeping it keeps it. + latest = asset_expiry_after_downgrade( + "2030-02-01T00:00:00+00:00", "2030-03-01T00:00:00+00:00" + ) + kept_by_one = asset_expiry_after_downgrade("2030-02-01T00:00:00+00:00", None) + finally: + command.upgrade(cfg, "head") + assert retention == [(7,)] + assert latest == "2030-03-01" + assert kept_by_one is None diff --git a/backend/tests/test_migration_0104.py b/backend/tests/test_migration_0104.py index 09b5517d..86f5ac2a 100644 --- a/backend/tests/test_migration_0104.py +++ b/backend/tests/test_migration_0104.py @@ -1,5 +1,5 @@ -"""Migration 0104's data step, DB-lane: a done file from before is dated by when it finished where -its "ready to watch" notice says so, else by when it was asked for; a job not done has no date.""" +"""Migration 0104's data step, DB-lane: a done file from before is dated by when it was asked for +(where it already sat); a job not done has no date.""" from alembic import command from alembic.config import Config @@ -8,7 +8,7 @@ from sqlalchemy import text from app.db import engine -def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user): +def test_a_done_file_takes_its_request_time(db, make_user): uid = make_user().id cfg = Config("alembic.ini") command.downgrade(cfg, "0103_files_sha256") @@ -25,17 +25,8 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user): {"u": uid, "s": status, "c": created}, ).scalar_one() - retried = job("done", "2026-10-01T10:00:00+00:00") - quiet = job("done", "2026-10-02T10:00:00+00:00") + done = job("done", "2026-10-02T10:00:00+00:00") failed = job("error", "2026-10-03T10:00:00+00:00") - for at in ("2026-10-05T09:00:00+00:00", "2026-10-06T09:00:00+00:00"): - conn.execute( - text( - "INSERT INTO notifications (user_id, type, title, data, created_at) VALUES" - " (:u, 'download_done', 't', CAST(:d AS json), CAST(:c AS timestamptz))" - ), - {"u": uid, "d": f'{{"job_id": {retried}}}', "c": at}, - ) finally: command.upgrade(cfg, "head") @@ -47,6 +38,5 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user): ) ).all() ) - assert rows[retried] == "10-06 09" # its last finish - assert rows[quiet] == "10-02 10" # no notice left: where it already sat + assert rows[done] == "10-02 10" assert rows[failed] is None diff --git a/backend/tests/test_migration_0105.py b/backend/tests/test_migration_0105.py new file mode 100644 index 00000000..937d6d01 --- /dev/null +++ b/backend/tests/test_migration_0105.py @@ -0,0 +1,45 @@ +"""Migration 0105's data step, DB-lane: a file in a source folder records that folder's key as its +origin; a file in an own folder is left to be worked out, as before.""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_a_file_in_a_source_folder_records_its_origin(db, make_user): + uid = make_user().id + cfg = Config("alembic.ini") + command.downgrade(cfg, "0104_job_ready_at") + try: + with engine.begin() as conn: + + def folder(kind, key, name): + return conn.execute( + text( + "INSERT INTO file_folders (user_id, kind, source_key, name) VALUES" + " (:u, :k, :s, :n) RETURNING id" + ), + {"u": uid, "k": kind, "s": key, "n": name}, + ).scalar_one() + + def job(folder_id): + return conn.execute( + text( + "INSERT INTO download_jobs (user_id, source_kind, source_ref," + " profile_snapshot, status, folder_id) VALUES (:u, 'url', 'x', '{}'," + " 'queued', :f) RETURNING id" + ), + {"u": uid, "f": folder_id}, + ).scalar_one() + + in_source = job(folder("source", "site:vimeo.com", None)) + in_own = job(folder("own", None, "Mine")) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + rows = dict(conn.execute(text("SELECT id, source_key FROM download_jobs")).all()) + assert rows[in_source] == "site:vimeo.com" + assert rows[in_own] is None diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py index d679e5ff..7525e773 100644 --- a/backend/tests/test_storage.py +++ b/backend/tests/test_storage.py @@ -6,6 +6,7 @@ from datetime import date from app.downloads.storage import ( MediaMeta, download_filename, + files_root_problem, rel_path, safe_abs_path, sanitize, @@ -174,3 +175,24 @@ def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path assert f"{stem}.nfo" in names, names assert f"{stem}.jpg" in names, names assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan + + +def test_a_files_root_on_the_containers_own_disk_is_refused(tmp_path): + # The image's empty /files, left unmounted by an old compose file: every file would 404 and + # every download would go with the container. + assert "not a mounted volume" in files_root_problem( + str(tmp_path), system_root=str(tmp_path.parent) + ) + assert "does not exist" in files_root_problem(str(tmp_path / "gone")) + # On another filesystem than the container's own, and writable: the volume. + assert files_root_problem(str(tmp_path), system_root="/proc") is None + + +def test_the_old_name_of_the_files_root_still_finds_the_files(monkeypatch): + from app.config import Settings + + monkeypatch.delenv("FILES_ROOT", raising=False) + monkeypatch.setenv("DOWNLOAD_ROOT", "/downloads") + assert Settings().files_root == "/downloads" + monkeypatch.setenv("FILES_ROOT", "/files") + assert Settings().files_root == "/files" diff --git a/backend/tests/test_worker_knobs.py b/backend/tests/test_worker_knobs.py index b423510d..c8cb192e 100644 --- a/backend/tests/test_worker_knobs.py +++ b/backend/tests/test_worker_knobs.py @@ -67,6 +67,7 @@ def test_worker_boots_with_the_stored_concurrency_and_log_level(monkeypatch, sto monkeypatch.setattr(worker, "_wait_for_schema", lambda: True) monkeypatch.setattr(worker.logsink, "start", lambda: None) monkeypatch.setattr(worker, "_recover_orphans", lambda: None) + monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: None) threading.Timer(0.5, worker._stop.set).start() worker.main() @@ -107,3 +108,17 @@ def test_an_unreadable_database_keeps_what_runs(monkeypatch, levels, loops): assert worker._target == 2 assert sorted(threads) == [0, 1] # the loops still run on the boot value assert levels == [] + + +def test_the_worker_will_not_start_on_a_files_root_that_is_not_the_volume(monkeypatch, loops): + # Files downloaded into the container's own empty /files go with the container: refuse, loudly. + monkeypatch.setattr(worker.settings, "worker_enabled", True) + monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None) + monkeypatch.setattr(worker.signal, "signal", lambda *_a: None) + monkeypatch.setattr(worker, "_wait_for_schema", lambda: True) + monkeypatch.setattr(worker.logsink, "start", lambda: None) + monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: "not a mounted volume") + with pytest.raises(SystemExit) as exit_: + worker.main() + assert exit_.value.code == 1 + assert loops == [] diff --git a/docker-compose.home.yml.example b/docker-compose.home.yml.example index fc30bfd8..ea55314f 100644 --- a/docker-compose.home.yml.example +++ b/docker-compose.home.yml.example @@ -72,7 +72,7 @@ services: # The public instance owns the background scheduler (single writer). SCHEDULER_ENABLED: "true" # Download center: the API serves finished files + builds filmstrips (read-only root is fine — - # it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below. + # it only writes to the /files mount + tmpfs). The yt-dlp job loop runs in `worker` below. FILES_ROOT: /files WORKER_ENABLED: "false" depends_on: @@ -113,7 +113,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to + # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # $HOME caches + the staging dir); heavier limits since it does the video encoding. worker: image: registry.example.com/you/siftlode:${APP_VERSION:-latest} diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml index 2095000e..fecf70a1 100644 --- a/docker-compose.selfhost.yml +++ b/docker-compose.selfhost.yml @@ -48,7 +48,7 @@ services: # This instance owns the background scheduler (single writer). SCHEDULER_ENABLED: "true" # Download center: the API serves finished files + builds filmstrips; the worker (below) runs - # the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs. + # the yt-dlp job loop. Read-only root is fine — it only writes to the /files mount + tmpfs. FILES_ROOT: /files WORKER_ENABLED: "false" depends_on: @@ -85,7 +85,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to + # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # $HOME caches + the staging dir). worker: image: git.b1fr0st.eu/peter/siftlode:${IMAGE_TAG:-latest} diff --git a/docker-compose.yml b/docker-compose.yml index 136fe5ca..673dfed1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: restart: unless-stopped # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the - # DB (job queue) + the downloads mount with the API. + # DB (job queue) + the /files mount with the API. worker: build: context: . diff --git a/docs/self-hosting.md b/docs/self-hosting.md index c253daa3..3180bcbc 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -254,17 +254,33 @@ Then `pull` + `up -d` stays on that version until you change the line. Released changed in them are listed in the app itself, under **Release Notes** in the account menu, and in [`CHANGELOG.md`](../CHANGELOG.md). -**The media mount was renamed** (the Files library): `DOWNLOAD_HOST_PATH` is now `FILES_HOST_PATH`, -mounted at `/files`, and the default volume is `siftlode_files` instead of `siftlode_downloads`. If you -set a host directory, rename the variable in `.env`. If you used the default volume, copy it over once -before you start the new version (Docker cannot rename a volume), then remove the old one when the -library looks right: +**The media mount was renamed** (the Files library): it is mounted at `/files` (`FILES_ROOT`), set +from `FILES_HOST_PATH`, and the default volume is `siftlode_files`. Until you move, nothing breaks: a +compose file from before still mounts `/downloads` and sets `DOWNLOAD_ROOT`, and the app still reads +that name. To move to the new layout, download the new +[`docker-compose.selfhost.yml`](../docker-compose.selfhost.yml) over your old one, then: -```bash -docker compose -f docker-compose.selfhost.yml down -docker volume create siftlode_files -docker run --rm -v siftlode_downloads:/from -v siftlode_files:/to alpine cp -a /from/. /to/ -``` +- **If you set a host directory**, rename `DOWNLOAD_HOST_PATH` to `FILES_HOST_PATH` in `.env`. +- **If you used the default volume**, copy it over once before you start the new version (Docker + cannot rename a volume). Compose names a volume after the project, which is your folder's name + unless you set one, so look the old one up rather than guessing it: + + ```bash + docker compose -f docker-compose.selfhost.yml down + docker volume ls --format '{{.Name}}' | grep 'siftlode_downloads$' # e.g. mysiftlode_siftlode_downloads + OLD=mysiftlode_siftlode_downloads # the name it printed + NEW="${OLD%siftlode_downloads}siftlode_files" + docker volume create "$NEW" + docker run --rm -v "$OLD":/from -v "$NEW":/to alpine cp -a /from/. /to/ + docker compose -f docker-compose.selfhost.yml up -d + ``` + + Open the Files library and check your files play; only then remove the old volume + (`docker volume rm "$OLD"`). + +If the files library is not mounted where the app looks, the API says so in its log at start +(`The files library is not reachable`), and the download worker refuses to start rather than write +files into a container that throws them away. **The registry keeps only the newest few releases, not every version the changelog lists.** A pin is "decide when you take the next hop", not "stay here forever": once you are several releases behind, diff --git a/frontend/src/components/RenameField.tsx b/frontend/src/components/ui/RenameField.tsx similarity index 100% rename from frontend/src/components/RenameField.tsx rename to frontend/src/components/ui/RenameField.tsx diff --git a/frontend/src/i18n/locales/en/errors.json b/frontend/src/i18n/locales/en/errors.json index 075e833e..9da019e3 100644 --- a/frontend/src/i18n/locales/en/errors.json +++ b/frontend/src/i18n/locales/en/errors.json @@ -237,6 +237,7 @@ "file_folder_name_taken": "You already have a folder with that name", "file_folder_not_own": "Only a folder you made can be renamed or deleted", "file_folder_gone": "That folder no longer exists", + "file_hash_busy": "Too many files are being checked right now — try again in a moment", "file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from", "file_no_preview": "There is no preview for this file" } diff --git a/frontend/src/i18n/locales/hu/errors.json b/frontend/src/i18n/locales/hu/errors.json index 2e2ce05e..97d11cac 100644 --- a/frontend/src/i18n/locales/hu/errors.json +++ b/frontend/src/i18n/locales/hu/errors.json @@ -237,6 +237,7 @@ "file_folder_name_taken": "Már van ilyen nevű mappád", "file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted", "file_folder_gone": "Ez a mappa már nem létezik", + "file_hash_busy": "Most túl sok fájl ellenőrzése fut — próbáld újra kicsit később", "file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet", "file_no_preview": "Ehhez a fájlhoz nincs előnézet" } diff --git a/install.ps1 b/install.ps1 index cf160749..5a55f695 100644 --- a/install.ps1 +++ b/install.ps1 @@ -29,7 +29,7 @@ SECRET_KEY=$secretKey TOKEN_ENCRYPTION_KEY=$fernet OAUTH_REDIRECT_URL=$url/auth/callback -# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead +# Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead # of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 ). # FILES_HOST_PATH=/mnt/media/youtube "@ | Set-Content -Path .env -Encoding ascii diff --git a/install.sh b/install.sh index b275e9de..b9ddc901 100755 --- a/install.sh +++ b/install.sh @@ -27,7 +27,7 @@ SECRET_KEY=$SECRET_KEY TOKEN_ENCRYPTION_KEY=$TOKEN_ENCRYPTION_KEY OAUTH_REDIRECT_URL=$URL/auth/callback -# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead +# Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead # of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 ). # FILES_HOST_PATH=/mnt/media/youtube EOF