fix(files): review round 1, the server — the tray's window, the GC's locks, a file's origin, the upgrade path
- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming, keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished. - GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a second release that frees a file another user still holds. - A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit whose parent expired still goes back to its folder, not to Uploads. - A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes that file home too, instead of failing on the foreign key. - The hash route reads at most two files at once (429 past that) and holds no connection meanwhile. - 0104's backfill dates old files by their request date only: no notice prod wrote names its job. - The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up. - Stale comments about the /downloads mount and the Downloads page.
This commit is contained in:
+3
-3
@@ -73,10 +73,10 @@ SMTP_FROM=
|
||||
# of them and false on the rest, to avoid double quota use and write races.
|
||||
SCHEDULER_ENABLED=true
|
||||
|
||||
# --- Download center ---
|
||||
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
||||
# --- Files library & downloads ---
|
||||
# Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
|
||||
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
|
||||
# Downloaded media defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
|
||||
# The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
|
||||
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
|
||||
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>.
|
||||
# FILES_HOST_PATH=/mnt/media/youtube
|
||||
|
||||
@@ -8,9 +8,8 @@ Create Date: 2026-10-12
|
||||
download that failed for days and then went through used to sit days down the list, under its
|
||||
request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`.
|
||||
|
||||
A done job from before has no record of that moment, except where its "ready to watch" notice is
|
||||
still in the owner's inbox (read notices are pruned past a cap): that notice was written the moment
|
||||
it finished. The rest take the request date, which is where they already sat.
|
||||
A done job from before has no record of that moment (its "ready to watch" notice never named the
|
||||
job, so it cannot be matched back), so it takes its request date, which is where it already sat.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
@@ -26,12 +25,7 @@ def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE download_jobs j SET ready_at = COALESCE(
|
||||
(SELECT max(n.created_at) FROM notifications n
|
||||
WHERE n.type = 'download_done' AND n.user_id = j.user_id
|
||||
AND n.data->>'job_id' = j.id::text),
|
||||
j.created_at)
|
||||
WHERE j.status = 'done'
|
||||
UPDATE download_jobs SET ready_at = created_at WHERE status = 'done'
|
||||
"""
|
||||
)
|
||||
op.create_check_constraint(
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A file remembers where it came from
|
||||
|
||||
Revision ID: 0105_job_source_key
|
||||
Revises: 0104_job_ready_at
|
||||
Create Date: 2026-10-12
|
||||
|
||||
`download_jobs.source_key` is the origin a file came from — the source folder it may go back to —
|
||||
recorded when it is placed. An edit used to work its origin out through its parent each time, and
|
||||
once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could
|
||||
no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever.
|
||||
|
||||
A file in a source folder takes that folder's key. One already in an own folder is left NULL and
|
||||
keeps being worked out (`folders.source_key_for`), as before.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0105_job_source_key"
|
||||
down_revision: str | None = "0104_job_ready_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("source_key", sa.String(260)))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE download_jobs j SET source_key = f.source_key
|
||||
FROM file_folders f
|
||||
WHERE f.id = j.folder_id AND f.kind = 'source'
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "source_key")
|
||||
@@ -1,7 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -171,9 +171,13 @@ class Settings(BaseSettings):
|
||||
feed_default_hidden_live: str = "live,upcoming"
|
||||
|
||||
# --- Download center (yt-dlp) ---
|
||||
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount.
|
||||
# Root directory (inside the container) for the files library — a Docker volume/bind mount.
|
||||
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
|
||||
files_root: str = "/files"
|
||||
# DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at
|
||||
# /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image.
|
||||
files_root: str = Field(
|
||||
default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root")
|
||||
)
|
||||
# Whether THIS process runs the download worker loop. On for the dedicated worker
|
||||
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
|
||||
worker_enabled: bool = False
|
||||
|
||||
@@ -68,9 +68,11 @@ def site_key(url: str) -> str:
|
||||
def source_key_for(db: Session, job: DownloadJob) -> str:
|
||||
"""The origin a job's file came from — the source folder it belongs in.
|
||||
|
||||
youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL →
|
||||
its site; an edit → wherever its parent's file came from (its parent job, or, once that is
|
||||
deleted, the parent's asset)."""
|
||||
The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
|
||||
(YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
|
||||
came from (its parent job, or, once that is deleted, the parent's asset)."""
|
||||
if job.source_key:
|
||||
return job.source_key
|
||||
kind = job.source_kind
|
||||
if kind == "youtube":
|
||||
return SourceKey.YOUTUBE
|
||||
@@ -122,9 +124,11 @@ def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
|
||||
|
||||
|
||||
def place(db: Session, job: DownloadJob) -> None:
|
||||
"""Put a new job's file in its source folder. Called wherever a job is created; the job must
|
||||
carry its `user_id`, `source_kind` and source refs already."""
|
||||
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||
"""Put a new job's file in its source folder, and record that origin on it. Called wherever a
|
||||
job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
|
||||
key = source_key_for(db, job)
|
||||
job.source_key = key
|
||||
job.folder_id = source_folder(db, job.user_id, key).id
|
||||
|
||||
|
||||
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
|
||||
@@ -151,12 +155,25 @@ class FolderError(Exception):
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder:
|
||||
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their
|
||||
Uploads folder. A file put in a site's folder would claim an origin it does not have."""
|
||||
folder = db.get(FileFolder, folder_id)
|
||||
def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||
"""The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
|
||||
caller's transaction ends, so a delete of the folder waits for the file and then takes it home
|
||||
with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
|
||||
only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
|
||||
folder answers like a missing one."""
|
||||
stmt = select(FileFolder).where(FileFolder.id == folder_id)
|
||||
if hold:
|
||||
stmt = stmt.with_for_update(read=True)
|
||||
folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
|
||||
if folder is None or folder.user_id != user_id:
|
||||
raise FolderError("file_folder_gone")
|
||||
return folder
|
||||
|
||||
|
||||
def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
|
||||
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their
|
||||
Uploads folder. A file put in a site's folder would claim an origin it does not have."""
|
||||
folder = own_folder(db, user_id, folder_id, hold=hold)
|
||||
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
|
||||
raise FolderError("file_folder_not_its_source")
|
||||
return folder
|
||||
@@ -219,6 +236,8 @@ def delete_own(db: Session, folder: FileFolder) -> int:
|
||||
folder it came from, its clock starting there now. Returns how many moved."""
|
||||
if folder.kind != FolderKind.OWN:
|
||||
raise FolderError("file_folder_not_own")
|
||||
# Wait out a file being put in it right now (`own_folder`), then take that one home too.
|
||||
db.execute(select(FileFolder.id).where(FileFolder.id == folder.id).with_for_update())
|
||||
jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all()
|
||||
for job in jobs:
|
||||
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
|
||||
|
||||
+63
-19
@@ -56,6 +56,32 @@ def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str)
|
||||
)
|
||||
|
||||
|
||||
def _relock_expired(
|
||||
db: Session, job_id: int, asset_id: int | None, now: datetime
|
||||
) -> DownloadJob | None:
|
||||
"""Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the
|
||||
asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a
|
||||
user may have deleted or re-downloaded it (dropping it again would release its asset a second
|
||||
time, freeing a file another job still holds) or kept or moved it (it is no longer expired)."""
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
still = (
|
||||
job is not None
|
||||
and job.asset_id == asset_id
|
||||
and job.status == JobStatus.DONE
|
||||
and not job.kept
|
||||
and job.expires_at is not None
|
||||
and job.expires_at <= now
|
||||
)
|
||||
return job if still else None
|
||||
|
||||
|
||||
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||
@@ -176,22 +202,22 @@ def run_download_gc(db: Session) -> dict:
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||
dead = (
|
||||
db.execute(
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at <= now,
|
||||
)
|
||||
# Each file is re-checked under its locks and committed on its own: the release unlinks the
|
||||
# file at once, so it must never act on what the SELECT saw a while ago.
|
||||
dead = db.execute(
|
||||
select(DownloadJob.id, DownloadJob.asset_id).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at <= now,
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in dead:
|
||||
_drop_job(db, job, "download_expired")
|
||||
expired += 1
|
||||
if dead:
|
||||
).all()
|
||||
db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below
|
||||
for job_id, asset_id in dead:
|
||||
job = _relock_expired(db, job_id, asset_id, now)
|
||||
if job is not None:
|
||||
_drop_job(db, job, "download_expired")
|
||||
expired += 1
|
||||
db.commit()
|
||||
|
||||
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
|
||||
@@ -221,15 +247,33 @@ def run_download_gc(db: Session) -> dict:
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in candidates:
|
||||
candidate_ids = [a.id for a in candidates]
|
||||
db.rollback()
|
||||
for asset_id in candidate_ids:
|
||||
if total <= cap:
|
||||
break
|
||||
total -= asset.size_bytes or 0
|
||||
# Locked and re-read like an expiry: a holder may have kept it, or every holder
|
||||
# deleted it (the file is then already gone and counted out) since the SELECT.
|
||||
asset = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
holders = (
|
||||
db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id))
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if asset is None or asset.status != "ready" or any(j.kept for j in holders):
|
||||
db.commit()
|
||||
continue
|
||||
total -= asset.size_bytes or 0
|
||||
for job in holders:
|
||||
_notify(db, job, asset, "download_evicted")
|
||||
db.delete(job)
|
||||
@@ -237,7 +281,7 @@ def run_download_gc(db: Session) -> dict:
|
||||
storage.delete_asset_files(settings.files_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
evicted += 1
|
||||
db.commit()
|
||||
db.commit()
|
||||
|
||||
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
|
||||
# referenced errored row to pending on re-enqueue, so these only accumulate once every
|
||||
|
||||
@@ -49,7 +49,7 @@ def source_url(source_kind: str, source_ref: str) -> str:
|
||||
|
||||
|
||||
def reference_url(job, asset) -> str | None:
|
||||
"""The clean "downloaded from" URL for a job — shared by the private Downloads page and the public
|
||||
"""The clean "downloaded from" URL for a job — shared by the Files library and the public
|
||||
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
|
||||
own earlier download, not a web page) and for a Plex clip (the library is not a public page); a
|
||||
YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else
|
||||
|
||||
@@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide
|
||||
date-based — the community-standard mapping for YouTube-in-Plex.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
@@ -67,6 +68,25 @@ _FOLD_MAP = str.maketrans(
|
||||
)
|
||||
|
||||
|
||||
def files_root_problem(root: str, system_root: str = "/") -> str | None:
|
||||
"""Why `root` cannot be the files library's volume, or None when it can. A root that sits on the
|
||||
container's own filesystem is the image's empty directory, not a mounted volume: the library
|
||||
would answer 404 for every file, and what a download writes would go when the container does —
|
||||
the state an install is in when its compose file still mounts the library somewhere else."""
|
||||
path = Path(root)
|
||||
if not path.is_dir():
|
||||
return f"{root} does not exist"
|
||||
try:
|
||||
if path.stat().st_dev == Path(system_root).stat().st_dev:
|
||||
return f"{root} is not a mounted volume (it is the container's own empty directory)"
|
||||
# silent-ok: the failure IS the answer — returned as the reason, which the caller logs.
|
||||
except OSError as e:
|
||||
return f"{root} cannot be read ({e})"
|
||||
if not os.access(path, os.W_OK):
|
||||
return f"{root} is not writable"
|
||||
return None
|
||||
|
||||
|
||||
def _ascii_fold(text: str) -> str:
|
||||
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
|
||||
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
|
||||
|
||||
@@ -28,6 +28,7 @@ log = logs.get_logger(__name__)
|
||||
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
||||
from app.config import settings # noqa: E402 — logging is configured first
|
||||
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
||||
from app.downloads import storage # noqa: E402 — logging is configured first
|
||||
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
||||
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
||||
from app.routes import ( # noqa: E402 — logging is configured first
|
||||
@@ -64,6 +65,10 @@ from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 —
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
problem = storage.files_root_problem(settings.files_root)
|
||||
if problem:
|
||||
# The rest of the app still serves; the files library cannot (the worker refuses to start).
|
||||
log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
|
||||
@@ -877,7 +877,7 @@ class MediaAsset(Base, TimestampMixin):
|
||||
upload_date: Mapped[date | None] = mapped_column(Date)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(Text)
|
||||
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
|
||||
# reference shown on the Downloads page. Null until the worker extracts it (older/queued rows
|
||||
# reference shown in the Files library. Null until the worker extracts it (older/queued rows
|
||||
# fall back to a URL derived from source_kind+source_ref).
|
||||
source_webpage_url: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1036,6 +1036,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
folder_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
|
||||
)
|
||||
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
|
||||
# traces its origin through its parent, and that parent may be deleted or expire later — the
|
||||
# file must still know which source folder it may go back to. NULL only for a file placed before
|
||||
# the column existed outside a source folder; `folders.source_key_for` works those out.
|
||||
source_key: Mapped[str | None] = mapped_column(String(260))
|
||||
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
|
||||
@@ -399,6 +399,8 @@ def enqueue_clip(
|
||||
|
||||
# How long a finished job stays in the queue: the tray polls it from every page and must see a job
|
||||
# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's.
|
||||
# A done job is timed from when it became done, never from its last write: a rename, a move or the
|
||||
# GC's expiry warning touches an old file's row, and the tray would announce it as just finished.
|
||||
QUEUE_RECENT = timedelta(minutes=10)
|
||||
|
||||
|
||||
@@ -411,7 +413,9 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get
|
||||
select(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.not_in(FINISHED) | (DownloadJob.updated_at >= recent),
|
||||
DownloadJob.status.not_in(FINISHED)
|
||||
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
|
||||
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
|
||||
)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(MAX_LIST)
|
||||
|
||||
+29
-11
@@ -13,6 +13,7 @@ import base64
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
@@ -23,7 +24,7 @@ from urllib.parse import unquote, urlparse
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -59,6 +60,9 @@ MAX_LIST = 500
|
||||
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
||||
_MAX_RETENTION_DAYS = 3650
|
||||
_HASH_CHUNK = 1 << 20
|
||||
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
|
||||
_HASHERS = 2
|
||||
_HASHING = threading.BoundedSemaphore(_HASHERS)
|
||||
|
||||
|
||||
# --- shared with routes.downloads -------------------------------------------------------
|
||||
@@ -235,7 +239,7 @@ def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
|
||||
if not raw.strip():
|
||||
return None
|
||||
try:
|
||||
return folders.upload_target(db, user_id, int(raw)).id
|
||||
return folders.upload_target(db, user_id, int(raw), hold=False).id
|
||||
except ValueError as e:
|
||||
raise refused(409, "file_folder_gone") from e
|
||||
except folders.FolderError as e:
|
||||
@@ -664,12 +668,9 @@ def update_file(
|
||||
if "folder_id" in sent:
|
||||
if body.folder_id is None:
|
||||
raise refused(400, "file_folder_gone")
|
||||
folder = db.get(FileFolder, body.folder_id)
|
||||
if folder is None or folder.user_id != user.id:
|
||||
raise refused(409, "file_folder_gone")
|
||||
before = job.folder_id
|
||||
try:
|
||||
folders.move(db, job, folder)
|
||||
folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
if job.folder_id != before:
|
||||
@@ -842,13 +843,30 @@ def get_sha256(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
|
||||
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool."""
|
||||
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
|
||||
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
|
||||
past that is told to come back), and none of them holds a database connection while it reads."""
|
||||
asset, path = _ready_file(db, accessible_job(db, user, job_id))
|
||||
if not asset.sha256:
|
||||
if asset.sha256:
|
||||
return {"sha256": asset.sha256}
|
||||
asset_id = asset.id
|
||||
if not _HASHING.acquire(blocking=False):
|
||||
raise refused(429, "file_hash_busy")
|
||||
try:
|
||||
release_read_connection(db)
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
while chunk := f.read(_HASH_CHUNK):
|
||||
h.update(chunk)
|
||||
asset.sha256 = h.hexdigest()
|
||||
db.commit()
|
||||
return {"sha256": asset.sha256}
|
||||
except OSError as e:
|
||||
raise not_found("file") from e
|
||||
finally:
|
||||
_HASHING.release()
|
||||
digest = h.hexdigest()
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
db.commit()
|
||||
return {"sha256": digest}
|
||||
|
||||
@@ -37,7 +37,7 @@ from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import credentials, formats, quota, service
|
||||
from app.downloads import credentials, formats, quota, service, storage
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
@@ -710,6 +710,12 @@ def main() -> None:
|
||||
# `log_events`, so on the deploy that first adds the table the worker may start before the API
|
||||
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
|
||||
logsink.start()
|
||||
problem = storage.files_root_problem(settings.files_root)
|
||||
if problem:
|
||||
# Refuse, loudly: downloading into a root that is not the volume loses every file with the
|
||||
# container. The queue waits until the mount is fixed (see docs/self-hosting.md).
|
||||
log.critical("The files library cannot be written: %s. The worker will not start.", problem)
|
||||
raise SystemExit(1)
|
||||
_recover_orphans()
|
||||
global _target
|
||||
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
|
||||
|
||||
@@ -15,6 +15,7 @@ from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import folders, formats, service
|
||||
from app.downloads import gc as gcmod
|
||||
from app.db import SessionLocal
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, FileFolder, MediaAsset, Notification
|
||||
|
||||
@@ -123,6 +124,68 @@ def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
|
||||
assert folders.source_key_for(db, edit) == "site:vimeo.com"
|
||||
|
||||
|
||||
def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user):
|
||||
# The parent download expires, its file goes with its last holder, and both of the edit's
|
||||
# pointers to it are cleared: the edit must still go back to the YouTube folder it came from.
|
||||
uid = make_user().id
|
||||
parent = _job(db, uid, _asset(db))
|
||||
edit = _job(
|
||||
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id,
|
||||
source_asset_id=parent.asset_id,
|
||||
) # fmt: skip
|
||||
home = _folder(db, edit)
|
||||
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
||||
db.add(own)
|
||||
db.flush()
|
||||
folders.move(db, edit, own)
|
||||
edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves
|
||||
db.delete(parent)
|
||||
db.commit()
|
||||
folders.move(db, edit, home)
|
||||
assert edit.folder_id == home.id
|
||||
folders.move(db, edit, own)
|
||||
folders.delete_own(db, own)
|
||||
assert edit.folder_id == home.id
|
||||
|
||||
|
||||
def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user):
|
||||
# Before, the delete missed the file still on its way in and failed on the foreign key (a 500),
|
||||
# or the file's insert did after the folder went.
|
||||
import threading
|
||||
|
||||
uid = make_user().id
|
||||
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
||||
db.add(own)
|
||||
db.commit()
|
||||
own_id = own.id
|
||||
job = _job(db, uid, _asset(db))
|
||||
home = job.folder_id
|
||||
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
|
||||
outcome: list[object] = []
|
||||
|
||||
def delete_it():
|
||||
other = SessionLocal()
|
||||
try:
|
||||
outcome.append(folders.delete_own(other, other.get(FileFolder, own_id)))
|
||||
other.commit()
|
||||
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
|
||||
outcome.append(e)
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
t = threading.Thread(target=delete_it)
|
||||
t.start()
|
||||
t.join(timeout=1)
|
||||
assert t.is_alive() # the delete waits for the file, even before it is written
|
||||
folders.move(db, job, target)
|
||||
db.commit()
|
||||
t.join(timeout=10)
|
||||
assert outcome == [1]
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id).folder_id == home
|
||||
assert db.get(FileFolder, own_id) is None
|
||||
|
||||
|
||||
def test_folders_are_per_user(db, make_user):
|
||||
a, b = make_user().id, make_user().id
|
||||
asset = _asset(db)
|
||||
@@ -201,6 +264,30 @@ def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user,
|
||||
assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"])
|
||||
def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart):
|
||||
# The GC warns once per expiry; a file whose expiry was moved must be warned before the new one.
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
uid = make_user().id
|
||||
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
|
||||
_gc(db)
|
||||
assert job.expiry_warned
|
||||
if restart == "unkeep":
|
||||
folders.set_kept(db, job, True)
|
||||
folders.set_kept(db, job, False)
|
||||
elif restart == "move":
|
||||
home = _folder(db, job)
|
||||
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
|
||||
db.add(own)
|
||||
db.flush()
|
||||
folders.move(db, job, own)
|
||||
folders.move(db, job, home)
|
||||
else:
|
||||
folders.set_retention(db, _folder(db, job), 90)
|
||||
db.commit()
|
||||
assert not job.expiry_warned
|
||||
|
||||
|
||||
def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
uid = make_user().id
|
||||
@@ -221,6 +308,52 @@ def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
|
||||
assert (_root / "a.mp4").exists()
|
||||
|
||||
|
||||
def _meanwhile(monkeypatch, act):
|
||||
"""Run `act` in another session (another request) between the GC's expiry SELECT and its
|
||||
handling of each file it found — the window a user's click can land in."""
|
||||
real = gcmod._relock_expired
|
||||
|
||||
def relock(db, job_id, asset_id, now):
|
||||
other = SessionLocal()
|
||||
try:
|
||||
act(other, job_id)
|
||||
other.commit()
|
||||
finally:
|
||||
other.close()
|
||||
return real(db, job_id, asset_id, now)
|
||||
|
||||
monkeypatch.setattr(gcmod, "_relock_expired", relock)
|
||||
|
||||
|
||||
def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch):
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1))
|
||||
_meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True))
|
||||
assert _gc(db)["expired"] == 0
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, job.id) is not None
|
||||
assert (_root / "a.mp4").exists()
|
||||
|
||||
|
||||
def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch):
|
||||
# Released twice, the shared file would go from under the other user's copy.
|
||||
(_root / "a.mp4").write_bytes(b"x")
|
||||
a, b = make_user().id, make_user().id
|
||||
asset = _asset(db)
|
||||
_job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
|
||||
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
|
||||
|
||||
def user_deletes(other, jid):
|
||||
service.delete(other, other.get(DownloadJob, jid))
|
||||
|
||||
_meanwhile(monkeypatch, user_deletes)
|
||||
_gc(db)
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, stays.id).asset_id == asset.id
|
||||
assert db.get(MediaAsset, asset.id).ref_count == 1
|
||||
assert (_root / "a.mp4").exists()
|
||||
|
||||
|
||||
def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
|
||||
db, make_user, _root
|
||||
):
|
||||
|
||||
@@ -421,6 +421,21 @@ def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as
|
||||
assert (row.actor_id, row.target_id) == (alice.id, str(job.id))
|
||||
|
||||
|
||||
def test_a_file_shared_with_someone_is_theirs_to_watch_not_to_change(db, make_user, as_user, _root):
|
||||
# The read routes let a share's recipient in; the write routes must not.
|
||||
alice, friend = make_user(), make_user()
|
||||
job = _file(db, alice.id, title="Mine")
|
||||
db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
|
||||
db.commit()
|
||||
c = as_user(friend)
|
||||
assert c.patch(f"/api/files/{job.id}", json={"name": "Theirs"}).status_code == 404
|
||||
assert c.patch(f"/api/files/{job.id}", json={"kept": True}).status_code == 404
|
||||
assert c.delete(f"/api/files/{job.id}").status_code == 404
|
||||
db.expire_all()
|
||||
still = db.get(DownloadJob, job.id)
|
||||
assert still is not None and still.display_name is None and not still.kept
|
||||
|
||||
|
||||
# --- upload, the file, its hash ------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -532,6 +547,26 @@ def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask
|
||||
assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404
|
||||
|
||||
|
||||
def test_only_a_few_files_are_hashed_at_once(db, make_user, as_user, _root):
|
||||
# Each hash holds a thread of the pool every sync route shares for the whole read; a caller
|
||||
# past the few that may run is told to come back, not queued behind them.
|
||||
from app.routes import files as filesroute
|
||||
|
||||
alice = make_user()
|
||||
job = _file(db, alice.id, rel="v.mp4")
|
||||
(_root / "v.mp4").write_bytes(b"bytes")
|
||||
held = [filesroute._HASHING.acquire(blocking=False) for _ in range(filesroute._HASHERS)]
|
||||
try:
|
||||
assert all(held)
|
||||
busy = as_user(alice).get(f"/api/files/{job.id}/sha256")
|
||||
finally:
|
||||
for _ in held:
|
||||
filesroute._HASHING.release()
|
||||
assert busy.status_code == 429
|
||||
assert busy.json()["detail"]["reason"] == "file_hash_busy"
|
||||
assert as_user(alice).get(f"/api/files/{job.id}/sha256").status_code == 200
|
||||
|
||||
|
||||
def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user):
|
||||
job = _file(db, make_user().id)
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
@@ -562,7 +597,7 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library(
|
||||
db.execute(
|
||||
update(DownloadJob)
|
||||
.where(DownloadJob.id == job.id)
|
||||
.values(updated_at=NOW - timedelta(hours=1))
|
||||
.values(updated_at=NOW - timedelta(hours=1), ready_at=NOW - timedelta(hours=1))
|
||||
)
|
||||
db.commit()
|
||||
rows = as_user(user).get("/api/downloads").json()
|
||||
@@ -570,3 +605,22 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library(
|
||||
assert {row["id"] for row in rows} == {fresh.id, waiting.id, stuck.id, paused.id, stopped.id}
|
||||
# The tray names where each came from, as the library does.
|
||||
assert {row["source_key"] for row in rows} == {"youtube"}
|
||||
|
||||
|
||||
def test_an_old_file_written_to_does_not_come_back_as_just_finished(db, make_user, as_user):
|
||||
# Renaming, keeping or moving touches the row; the tray would announce the file as ready again.
|
||||
from sqlalchemy import update
|
||||
|
||||
user = make_user()
|
||||
old = _file(db, user.id, title="Old")
|
||||
db.execute(
|
||||
update(DownloadJob)
|
||||
.where(DownloadJob.id == old.id)
|
||||
.values(updated_at=NOW - timedelta(days=30), ready_at=NOW - timedelta(days=30))
|
||||
)
|
||||
db.commit()
|
||||
c = as_user(user)
|
||||
assert (
|
||||
c.patch(f"/api/files/{old.id}", json={"name": "Renamed", "kept": True}).status_code == 200
|
||||
)
|
||||
assert [row["id"] for row in c.get("/api/downloads").json()] == []
|
||||
|
||||
@@ -71,3 +71,71 @@ def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user):
|
||||
}
|
||||
# alice: youtube, tiktok, upload, plex; bob: youtube.
|
||||
assert folders == 5
|
||||
|
||||
|
||||
def test_the_admins_retention_is_kept_and_the_downgrade_gives_the_asset_back_its_expiry(
|
||||
db, make_user
|
||||
):
|
||||
alice, bob = make_user().id, make_user().id
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0101_plex_text_plain")
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO system_config (key, value) VALUES ('download_retention_days', '7')"
|
||||
" ON CONFLICT (key) DO UPDATE SET value = '7'"
|
||||
)
|
||||
)
|
||||
yt = conn.execute(
|
||||
text(
|
||||
"INSERT INTO media_assets (source_kind, source_ref, format_sig, status,"
|
||||
" ref_count, expires_at) VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 2,"
|
||||
" '2030-01-02T00:00:00+00:00') RETURNING id"
|
||||
)
|
||||
).scalar_one()
|
||||
for user in (alice, bob):
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
|
||||
" profile_snapshot, status) VALUES (:u, :a, 'youtube', 'abcdefghijk',"
|
||||
" '{}', 'done')"
|
||||
),
|
||||
{"u": user, "a": yt},
|
||||
)
|
||||
command.upgrade(cfg, "head")
|
||||
with engine.connect() as conn:
|
||||
retention = conn.execute(
|
||||
text(
|
||||
"SELECT DISTINCT retention_days FROM file_folders WHERE source_key = 'youtube'"
|
||||
)
|
||||
).all()
|
||||
|
||||
def asset_expiry_after_downgrade(alice_at, bob_at):
|
||||
with engine.begin() as conn:
|
||||
for user, at in ((alice, alice_at), (bob, bob_at)):
|
||||
conn.execute(
|
||||
text(
|
||||
"UPDATE download_jobs SET expires_at = CAST(:at AS timestamptz)"
|
||||
" WHERE asset_id = :a AND user_id = :u"
|
||||
),
|
||||
{"at": at, "a": yt, "u": user},
|
||||
)
|
||||
command.downgrade(cfg, "0101_plex_text_plain")
|
||||
with engine.connect() as conn:
|
||||
got = conn.execute(
|
||||
text("SELECT expires_at::date FROM media_assets WHERE id = :a"), {"a": yt}
|
||||
).scalar()
|
||||
command.upgrade(cfg, "head")
|
||||
return str(got) if got else None
|
||||
|
||||
# The shared file goes when its last holder's copy would; one holder keeping it keeps it.
|
||||
latest = asset_expiry_after_downgrade(
|
||||
"2030-02-01T00:00:00+00:00", "2030-03-01T00:00:00+00:00"
|
||||
)
|
||||
kept_by_one = asset_expiry_after_downgrade("2030-02-01T00:00:00+00:00", None)
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
assert retention == [(7,)]
|
||||
assert latest == "2030-03-01"
|
||||
assert kept_by_one is None
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Migration 0104's data step, DB-lane: a done file from before is dated by when it finished where
|
||||
its "ready to watch" notice says so, else by when it was asked for; a job not done has no date."""
|
||||
"""Migration 0104's data step, DB-lane: a done file from before is dated by when it was asked for
|
||||
(where it already sat); a job not done has no date."""
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
@@ -8,7 +8,7 @@ from sqlalchemy import text
|
||||
from app.db import engine
|
||||
|
||||
|
||||
def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user):
|
||||
def test_a_done_file_takes_its_request_time(db, make_user):
|
||||
uid = make_user().id
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0103_files_sha256")
|
||||
@@ -25,17 +25,8 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user):
|
||||
{"u": uid, "s": status, "c": created},
|
||||
).scalar_one()
|
||||
|
||||
retried = job("done", "2026-10-01T10:00:00+00:00")
|
||||
quiet = job("done", "2026-10-02T10:00:00+00:00")
|
||||
done = job("done", "2026-10-02T10:00:00+00:00")
|
||||
failed = job("error", "2026-10-03T10:00:00+00:00")
|
||||
for at in ("2026-10-05T09:00:00+00:00", "2026-10-06T09:00:00+00:00"):
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO notifications (user_id, type, title, data, created_at) VALUES"
|
||||
" (:u, 'download_done', 't', CAST(:d AS json), CAST(:c AS timestamptz))"
|
||||
),
|
||||
{"u": uid, "d": f'{{"job_id": {retried}}}', "c": at},
|
||||
)
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
@@ -47,6 +38,5 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user):
|
||||
)
|
||||
).all()
|
||||
)
|
||||
assert rows[retried] == "10-06 09" # its last finish
|
||||
assert rows[quiet] == "10-02 10" # no notice left: where it already sat
|
||||
assert rows[done] == "10-02 10"
|
||||
assert rows[failed] is None
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Migration 0105's data step, DB-lane: a file in a source folder records that folder's key as its
|
||||
origin; a file in an own folder is left to be worked out, as before."""
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import engine
|
||||
|
||||
|
||||
def test_a_file_in_a_source_folder_records_its_origin(db, make_user):
|
||||
uid = make_user().id
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0104_job_ready_at")
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
|
||||
def folder(kind, key, name):
|
||||
return conn.execute(
|
||||
text(
|
||||
"INSERT INTO file_folders (user_id, kind, source_key, name) VALUES"
|
||||
" (:u, :k, :s, :n) RETURNING id"
|
||||
),
|
||||
{"u": uid, "k": kind, "s": key, "n": name},
|
||||
).scalar_one()
|
||||
|
||||
def job(folder_id):
|
||||
return conn.execute(
|
||||
text(
|
||||
"INSERT INTO download_jobs (user_id, source_kind, source_ref,"
|
||||
" profile_snapshot, status, folder_id) VALUES (:u, 'url', 'x', '{}',"
|
||||
" 'queued', :f) RETURNING id"
|
||||
),
|
||||
{"u": uid, "f": folder_id},
|
||||
).scalar_one()
|
||||
|
||||
in_source = job(folder("source", "site:vimeo.com", None))
|
||||
in_own = job(folder("own", None, "Mine"))
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
with engine.connect() as conn:
|
||||
rows = dict(conn.execute(text("SELECT id, source_key FROM download_jobs")).all())
|
||||
assert rows[in_source] == "site:vimeo.com"
|
||||
assert rows[in_own] is None
|
||||
@@ -6,6 +6,7 @@ from datetime import date
|
||||
from app.downloads.storage import (
|
||||
MediaMeta,
|
||||
download_filename,
|
||||
files_root_problem,
|
||||
rel_path,
|
||||
safe_abs_path,
|
||||
sanitize,
|
||||
@@ -174,3 +175,24 @@ def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path
|
||||
assert f"{stem}.nfo" in names, names
|
||||
assert f"{stem}.jpg" in names, names
|
||||
assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan
|
||||
|
||||
|
||||
def test_a_files_root_on_the_containers_own_disk_is_refused(tmp_path):
|
||||
# The image's empty /files, left unmounted by an old compose file: every file would 404 and
|
||||
# every download would go with the container.
|
||||
assert "not a mounted volume" in files_root_problem(
|
||||
str(tmp_path), system_root=str(tmp_path.parent)
|
||||
)
|
||||
assert "does not exist" in files_root_problem(str(tmp_path / "gone"))
|
||||
# On another filesystem than the container's own, and writable: the volume.
|
||||
assert files_root_problem(str(tmp_path), system_root="/proc") is None
|
||||
|
||||
|
||||
def test_the_old_name_of_the_files_root_still_finds_the_files(monkeypatch):
|
||||
from app.config import Settings
|
||||
|
||||
monkeypatch.delenv("FILES_ROOT", raising=False)
|
||||
monkeypatch.setenv("DOWNLOAD_ROOT", "/downloads")
|
||||
assert Settings().files_root == "/downloads"
|
||||
monkeypatch.setenv("FILES_ROOT", "/files")
|
||||
assert Settings().files_root == "/files"
|
||||
|
||||
@@ -67,6 +67,7 @@ def test_worker_boots_with_the_stored_concurrency_and_log_level(monkeypatch, sto
|
||||
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
|
||||
monkeypatch.setattr(worker.logsink, "start", lambda: None)
|
||||
monkeypatch.setattr(worker, "_recover_orphans", lambda: None)
|
||||
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: None)
|
||||
|
||||
threading.Timer(0.5, worker._stop.set).start()
|
||||
worker.main()
|
||||
@@ -107,3 +108,17 @@ def test_an_unreadable_database_keeps_what_runs(monkeypatch, levels, loops):
|
||||
assert worker._target == 2
|
||||
assert sorted(threads) == [0, 1] # the loops still run on the boot value
|
||||
assert levels == []
|
||||
|
||||
|
||||
def test_the_worker_will_not_start_on_a_files_root_that_is_not_the_volume(monkeypatch, loops):
|
||||
# Files downloaded into the container's own empty /files go with the container: refuse, loudly.
|
||||
monkeypatch.setattr(worker.settings, "worker_enabled", True)
|
||||
monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None)
|
||||
monkeypatch.setattr(worker.signal, "signal", lambda *_a: None)
|
||||
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
|
||||
monkeypatch.setattr(worker.logsink, "start", lambda: None)
|
||||
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: "not a mounted volume")
|
||||
with pytest.raises(SystemExit) as exit_:
|
||||
worker.main()
|
||||
assert exit_.value.code == 1
|
||||
assert loops == []
|
||||
|
||||
@@ -72,7 +72,7 @@ services:
|
||||
# The public instance owns the background scheduler (single writer).
|
||||
SCHEDULER_ENABLED: "true"
|
||||
# Download center: the API serves finished files + builds filmstrips (read-only root is fine —
|
||||
# it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below.
|
||||
# it only writes to the /files mount + tmpfs). The yt-dlp job loop runs in `worker` below.
|
||||
FILES_ROOT: /files
|
||||
WORKER_ENABLED: "false"
|
||||
depends_on:
|
||||
@@ -113,7 +113,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
|
||||
# DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
|
||||
# DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
|
||||
# $HOME caches + the staging dir); heavier limits since it does the video encoding.
|
||||
worker:
|
||||
image: registry.example.com/you/siftlode:${APP_VERSION:-latest}
|
||||
|
||||
@@ -48,7 +48,7 @@ services:
|
||||
# This instance owns the background scheduler (single writer).
|
||||
SCHEDULER_ENABLED: "true"
|
||||
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs
|
||||
# the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs.
|
||||
# the yt-dlp job loop. Read-only root is fine — it only writes to the /files mount + tmpfs.
|
||||
FILES_ROOT: /files
|
||||
WORKER_ENABLED: "false"
|
||||
depends_on:
|
||||
@@ -85,7 +85,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
|
||||
# DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
|
||||
# DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
|
||||
# $HOME caches + the staging dir).
|
||||
worker:
|
||||
image: git.b1fr0st.eu/peter/siftlode:${IMAGE_TAG:-latest}
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
|
||||
# DB (job queue) + the downloads mount with the API.
|
||||
# DB (job queue) + the /files mount with the API.
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
|
||||
+26
-10
@@ -254,17 +254,33 @@ Then `pull` + `up -d` stays on that version until you change the line. Released
|
||||
changed in them are listed in the app itself, under **Release Notes** in the account menu, and in
|
||||
[`CHANGELOG.md`](../CHANGELOG.md).
|
||||
|
||||
**The media mount was renamed** (the Files library): `DOWNLOAD_HOST_PATH` is now `FILES_HOST_PATH`,
|
||||
mounted at `/files`, and the default volume is `siftlode_files` instead of `siftlode_downloads`. If you
|
||||
set a host directory, rename the variable in `.env`. If you used the default volume, copy it over once
|
||||
before you start the new version (Docker cannot rename a volume), then remove the old one when the
|
||||
library looks right:
|
||||
**The media mount was renamed** (the Files library): it is mounted at `/files` (`FILES_ROOT`), set
|
||||
from `FILES_HOST_PATH`, and the default volume is `siftlode_files`. Until you move, nothing breaks: a
|
||||
compose file from before still mounts `/downloads` and sets `DOWNLOAD_ROOT`, and the app still reads
|
||||
that name. To move to the new layout, download the new
|
||||
[`docker-compose.selfhost.yml`](../docker-compose.selfhost.yml) over your old one, then:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.selfhost.yml down
|
||||
docker volume create siftlode_files
|
||||
docker run --rm -v siftlode_downloads:/from -v siftlode_files:/to alpine cp -a /from/. /to/
|
||||
```
|
||||
- **If you set a host directory**, rename `DOWNLOAD_HOST_PATH` to `FILES_HOST_PATH` in `.env`.
|
||||
- **If you used the default volume**, copy it over once before you start the new version (Docker
|
||||
cannot rename a volume). Compose names a volume after the project, which is your folder's name
|
||||
unless you set one, so look the old one up rather than guessing it:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.selfhost.yml down
|
||||
docker volume ls --format '{{.Name}}' | grep 'siftlode_downloads$' # e.g. mysiftlode_siftlode_downloads
|
||||
OLD=mysiftlode_siftlode_downloads # the name it printed
|
||||
NEW="${OLD%siftlode_downloads}siftlode_files"
|
||||
docker volume create "$NEW"
|
||||
docker run --rm -v "$OLD":/from -v "$NEW":/to alpine cp -a /from/. /to/
|
||||
docker compose -f docker-compose.selfhost.yml up -d
|
||||
```
|
||||
|
||||
Open the Files library and check your files play; only then remove the old volume
|
||||
(`docker volume rm "$OLD"`).
|
||||
|
||||
If the files library is not mounted where the app looks, the API says so in its log at start
|
||||
(`The files library is not reachable`), and the download worker refuses to start rather than write
|
||||
files into a container that throws them away.
|
||||
|
||||
**The registry keeps only the newest few releases, not every version the changelog lists.** A pin is
|
||||
"decide when you take the next hop", not "stay here forever": once you are several releases behind,
|
||||
|
||||
@@ -237,6 +237,7 @@
|
||||
"file_folder_name_taken": "You already have a folder with that name",
|
||||
"file_folder_not_own": "Only a folder you made can be renamed or deleted",
|
||||
"file_folder_gone": "That folder no longer exists",
|
||||
"file_hash_busy": "Too many files are being checked right now — try again in a moment",
|
||||
"file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from",
|
||||
"file_no_preview": "There is no preview for this file"
|
||||
}
|
||||
|
||||
@@ -237,6 +237,7 @@
|
||||
"file_folder_name_taken": "Már van ilyen nevű mappád",
|
||||
"file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted",
|
||||
"file_folder_gone": "Ez a mappa már nem létezik",
|
||||
"file_hash_busy": "Most túl sok fájl ellenőrzése fut — próbáld újra kicsit később",
|
||||
"file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet",
|
||||
"file_no_preview": "Ehhez a fájlhoz nincs előnézet"
|
||||
}
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ SECRET_KEY=$secretKey
|
||||
TOKEN_ENCRYPTION_KEY=$fernet
|
||||
OAUTH_REDIRECT_URL=$url/auth/callback
|
||||
|
||||
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
|
||||
# Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead
|
||||
# of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 <dir>).
|
||||
# FILES_HOST_PATH=/mnt/media/youtube
|
||||
"@ | Set-Content -Path .env -Encoding ascii
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ SECRET_KEY=$SECRET_KEY
|
||||
TOKEN_ENCRYPTION_KEY=$TOKEN_ENCRYPTION_KEY
|
||||
OAUTH_REDIRECT_URL=$URL/auth/callback
|
||||
|
||||
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead
|
||||
# Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead
|
||||
# of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 <dir>).
|
||||
# FILES_HOST_PATH=/mnt/media/youtube
|
||||
EOF
|
||||
|
||||
Reference in New Issue
Block a user