fix(files): review round 1, the server — the tray's window, the GC's locks, a file's origin, the upgrade path

- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming,
  keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished.
- GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit
  one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a
  second release that frees a file another user still holds.
- A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit
  whose parent expired still goes back to its folder, not to Uploads.
- A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes
  that file home too, instead of failing on the foreign key.
- The hash route reads at most two files at once (429 past that) and holds no connection meanwhile.
- 0104's backfill dates old files by their request date only: no notice prod wrote names its job.
- The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the
  worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up.
- Stale comments about the /downloads mount and the Downloads page.
This commit is contained in:
2026-10-13 00:14:09 +02:00
parent 96509fe356
commit 815083bcc4
29 changed files with 593 additions and 92 deletions
+3 -3
View File
@@ -73,10 +73,10 @@ SMTP_FROM=
# of them and false on the rest, to avoid double quota use and write races. # of them and false on the rest, to avoid double quota use and write races.
SCHEDULER_ENABLED=true SCHEDULER_ENABLED=true
# --- Download center --- # --- Files library & downloads ---
# The Download Center adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small # Downloading adds a `worker` container (runs the yt-dlp/ffmpeg job loop) and a small
# `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose. # `bgutil-pot` sidecar (mints YouTube tokens) — both come up automatically with docker compose.
# Downloaded media defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host # The Files library defaults to a Docker-managed named volume. Set FILES_HOST_PATH to a host
# directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there. # directory instead — e.g. one your Plex server can read — to keep the Plex-style tree there.
# The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>. # The path must be writable by the container user (uid of `appuser`, 1000): chown 1000:1000 <dir>.
# FILES_HOST_PATH=/mnt/media/youtube # FILES_HOST_PATH=/mnt/media/youtube
@@ -8,9 +8,8 @@ Create Date: 2026-10-12
download that failed for days and then went through used to sit days down the list, under its download that failed for days and then went through used to sit days down the list, under its
request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`. request date. Every path to done stamps it, held by `ck_download_jobs_ready_at`.
A done job from before has no record of that moment, except where its "ready to watch" notice is A done job from before has no record of that moment (its "ready to watch" notice never named the
still in the owner's inbox (read notices are pruned past a cap): that notice was written the moment job, so it cannot be matched back), so it takes its request date, which is where it already sat.
it finished. The rest take the request date, which is where they already sat.
""" """
import sqlalchemy as sa import sqlalchemy as sa
@@ -26,12 +25,7 @@ def upgrade() -> None:
op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True))) op.add_column("download_jobs", sa.Column("ready_at", sa.DateTime(timezone=True)))
op.execute( op.execute(
""" """
UPDATE download_jobs j SET ready_at = COALESCE( UPDATE download_jobs SET ready_at = created_at WHERE status = 'done'
(SELECT max(n.created_at) FROM notifications n
WHERE n.type = 'download_done' AND n.user_id = j.user_id
AND n.data->>'job_id' = j.id::text),
j.created_at)
WHERE j.status = 'done'
""" """
) )
op.create_check_constraint( op.create_check_constraint(
@@ -0,0 +1,37 @@
"""A file remembers where it came from
Revision ID: 0105_job_source_key
Revises: 0104_job_ready_at
Create Date: 2026-10-12
`download_jobs.source_key` is the origin a file came from — the source folder it may go back to —
recorded when it is placed. An edit used to work its origin out through its parent each time, and
once the parent was deleted or expired it fell back to "upload": moved into an own folder, it could
no longer go back to its folder, and deleting that folder sent it to Uploads to stay forever.
A file in a source folder takes that folder's key. One already in an own folder is left NULL and
keeps being worked out (`folders.source_key_for`), as before.
"""
import sqlalchemy as sa
from alembic import op
revision: str = "0105_job_source_key"
down_revision: str | None = "0104_job_ready_at"
branch_labels: str | None = None
depends_on: str | None = None
def upgrade() -> None:
op.add_column("download_jobs", sa.Column("source_key", sa.String(260)))
op.execute(
"""
UPDATE download_jobs j SET source_key = f.source_key
FROM file_folders f
WHERE f.id = j.folder_id AND f.kind = 'source'
"""
)
def downgrade() -> None:
op.drop_column("download_jobs", "source_key")
+7 -3
View File
@@ -1,7 +1,7 @@
from pathlib import Path from pathlib import Path
from typing import Literal, get_args, get_origin from typing import Literal, get_args, get_origin
from pydantic import Field, ValidationInfo, field_validator, model_validator from pydantic import AliasChoices, Field, ValidationInfo, field_validator, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
# Shipped placeholder; production must override it. Used by the startup guard below. # Shipped placeholder; production must override it. Used by the startup guard below.
@@ -171,9 +171,13 @@ class Settings(BaseSettings):
feed_default_hidden_live: str = "live,upcoming" feed_default_hidden_live: str = "live,upcoming"
# --- Download center (yt-dlp) --- # --- Download center (yt-dlp) ---
# Root directory (inside the container) for downloaded media — a Docker volume/bind mount. # Root directory (inside the container) for the files library — a Docker volume/bind mount.
# Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it. # Env-only (a mount path can't live in the DB). The worker lays a Plex-style tree under it.
files_root: str = "/files" # DOWNLOAD_ROOT is its old name: an install still on the old compose file (mounted at
# /downloads, DOWNLOAD_ROOT=/downloads) keeps finding its files after it pulls a new image.
files_root: str = Field(
default="/files", validation_alias=AliasChoices("FILES_ROOT", "DOWNLOAD_ROOT", "files_root")
)
# Whether THIS process runs the download worker loop. On for the dedicated worker # Whether THIS process runs the download worker loop. On for the dedicated worker
# container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled. # container, off for the API (which keeps the scheduler). Mirrors scheduler_enabled.
worker_enabled: bool = False worker_enabled: bool = False
+29 -10
View File
@@ -68,9 +68,11 @@ def site_key(url: str) -> str:
def source_key_for(db: Session, job: DownloadJob) -> str: def source_key_for(db: Session, job: DownloadJob) -> str:
"""The origin a job's file came from — the source folder it belongs in. """The origin a job's file came from — the source folder it belongs in.
youtube → YouTube; a clip → the clip's source (YouTube or Plex); an upload → Uploads; a URL → The one recorded when it was placed, else: youtube → YouTube; a clip → the clip's source
its site; an edit → wherever its parent's file came from (its parent job, or, once that is (YouTube or Plex); an upload → Uploads; a URL → its site; an edit → wherever its parent's file
deleted, the parent's asset).""" came from (its parent job, or, once that is deleted, the parent's asset)."""
if job.source_key:
return job.source_key
kind = job.source_kind kind = job.source_kind
if kind == "youtube": if kind == "youtube":
return SourceKey.YOUTUBE return SourceKey.YOUTUBE
@@ -122,9 +124,11 @@ def source_folder(db: Session, user_id: int, key: str) -> FileFolder:
def place(db: Session, job: DownloadJob) -> None: def place(db: Session, job: DownloadJob) -> None:
"""Put a new job's file in its source folder. Called wherever a job is created; the job must """Put a new job's file in its source folder, and record that origin on it. Called wherever a
carry its `user_id`, `source_kind` and source refs already.""" job is created; the job must carry its `user_id`, `source_kind` and source refs already."""
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id key = source_key_for(db, job)
job.source_key = key
job.folder_id = source_folder(db, job.user_id, key).id
def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None: def done_expiry(db: Session, folder_id: int | None, kept: bool) -> datetime | None:
@@ -151,12 +155,25 @@ class FolderError(Exception):
self.reason = reason self.reason = reason
def upload_target(db: Session, user_id: int, folder_id: int) -> FileFolder: def own_folder(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their """The user's folder `folder_id` that a file is about to be put in — held (FOR SHARE) until the
Uploads folder. A file put in a site's folder would claim an origin it does not have.""" caller's transaction ends, so a delete of the folder waits for the file and then takes it home
folder = db.get(FileFolder, folder_id) with the rest (`delete_own` locks it FOR UPDATE), never failing on its foreign key. `hold=False`
only looks (a check ahead of a long wait, which must not hold a lock through it). Another user's
folder answers like a missing one."""
stmt = select(FileFolder).where(FileFolder.id == folder_id)
if hold:
stmt = stmt.with_for_update(read=True)
folder = db.execute(stmt.execution_options(populate_existing=True)).scalar_one_or_none()
if folder is None or folder.user_id != user_id: if folder is None or folder.user_id != user_id:
raise FolderError("file_folder_gone") raise FolderError("file_folder_gone")
return folder
def upload_target(db: Session, user_id: int, folder_id: int, *, hold: bool = True) -> FileFolder:
"""The folder an upload asked to land in, if it may: one of the user's own folders, or their
Uploads folder. A file put in a site's folder would claim an origin it does not have."""
folder = own_folder(db, user_id, folder_id, hold=hold)
if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD: if folder.kind == FolderKind.SOURCE and folder.source_key != SourceKey.UPLOAD:
raise FolderError("file_folder_not_its_source") raise FolderError("file_folder_not_its_source")
return folder return folder
@@ -219,6 +236,8 @@ def delete_own(db: Session, folder: FileFolder) -> int:
folder it came from, its clock starting there now. Returns how many moved.""" folder it came from, its clock starting there now. Returns how many moved."""
if folder.kind != FolderKind.OWN: if folder.kind != FolderKind.OWN:
raise FolderError("file_folder_not_own") raise FolderError("file_folder_not_own")
# Wait out a file being put in it right now (`own_folder`), then take that one home too.
db.execute(select(FileFolder.id).where(FileFolder.id == folder.id).with_for_update())
jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all() jobs = db.execute(select(DownloadJob).where(DownloadJob.folder_id == folder.id)).scalars().all()
for job in jobs: for job in jobs:
job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id job.folder_id = source_folder(db, job.user_id, source_key_for(db, job)).id
+63 -19
View File
@@ -56,6 +56,32 @@ def _notify(db: Session, job: DownloadJob, asset: MediaAsset | None, ntype: str)
) )
def _relock_expired(
db: Session, job_id: int, asset_id: int | None, now: datetime
) -> DownloadJob | None:
"""Re-read an expired job under the lock order the user routes take (`own_job_for_update`: the
asset, then the job) and say whether it is still to go. Between the pass's SELECT and here a
user may have deleted or re-downloaded it (dropping it again would release its asset a second
time, freeing a file another job still holds) or kept or moved it (it is no longer expired)."""
if asset_id is not None:
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
job = db.execute(
select(DownloadJob)
.where(DownloadJob.id == job_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
still = (
job is not None
and job.asset_id == asset_id
and job.status == JobStatus.DONE
and not job.kept
and job.expires_at is not None
and job.expires_at <= now
)
return job if still else None
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None: def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the """Take a file out of its user's library: tell them, release the job's hold on the asset (the
file goes with the last holder), delete the row (its shares and links cascade).""" file goes with the last holder), delete the row (its shares and links cascade)."""
@@ -176,22 +202,22 @@ def run_download_gc(db: Session) -> dict:
db.commit() db.commit()
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1). # 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
dead = ( # Each file is re-checked under its locks and committed on its own: the release unlinks the
db.execute( # file at once, so it must never act on what the SELECT saw a while ago.
select(DownloadJob).where( dead = db.execute(
DownloadJob.status == JobStatus.DONE, select(DownloadJob.id, DownloadJob.asset_id).where(
DownloadJob.kept.is_(False), DownloadJob.status == JobStatus.DONE,
DownloadJob.expires_at.is_not(None), DownloadJob.kept.is_(False),
DownloadJob.expires_at <= now, DownloadJob.expires_at.is_not(None),
) DownloadJob.expires_at <= now,
) )
.scalars() ).all()
.all() db.rollback() # the pass-2 snapshot holds no locks; each file takes its own below
) for job_id, asset_id in dead:
for job in dead: job = _relock_expired(db, job_id, asset_id, now)
_drop_job(db, job, "download_expired") if job is not None:
expired += 1 _drop_job(db, job, "download_expired")
if dead: expired += 1
db.commit() db.commit()
# 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to # 3. LRU eviction over the total cache cap: the disk's safety net. Never an upload (no source to
@@ -221,15 +247,33 @@ def run_download_gc(db: Session) -> dict:
.scalars() .scalars()
.all() .all()
) )
for asset in candidates: candidate_ids = [a.id for a in candidates]
db.rollback()
for asset_id in candidate_ids:
if total <= cap: if total <= cap:
break break
total -= asset.size_bytes or 0 # Locked and re-read like an expiry: a holder may have kept it, or every holder
# deleted it (the file is then already gone and counted out) since the SELECT.
asset = db.execute(
select(MediaAsset)
.where(MediaAsset.id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
holders = ( holders = (
db.execute(select(DownloadJob).where(DownloadJob.asset_id == asset.id)) db.execute(
select(DownloadJob)
.where(DownloadJob.asset_id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
)
.scalars() .scalars()
.all() .all()
) )
if asset is None or asset.status != "ready" or any(j.kept for j in holders):
db.commit()
continue
total -= asset.size_bytes or 0
for job in holders: for job in holders:
_notify(db, job, asset, "download_evicted") _notify(db, job, asset, "download_evicted")
db.delete(job) db.delete(job)
@@ -237,7 +281,7 @@ def run_download_gc(db: Session) -> dict:
storage.delete_asset_files(settings.files_root, asset.rel_path) storage.delete_asset_files(settings.files_root, asset.rel_path)
db.delete(asset) db.delete(asset)
evicted += 1 evicted += 1
db.commit() db.commit()
# 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still- # 4. Reap orphaned errored rows (no file, no holder). get_or_create_asset resets a still-
# referenced errored row to pending on re-enqueue, so these only accumulate once every # referenced errored row to pending on re-enqueue, so these only accumulate once every
+1 -1
View File
@@ -49,7 +49,7 @@ def source_url(source_kind: str, source_ref: str) -> str:
def reference_url(job, asset) -> str | None: def reference_url(job, asset) -> str | None:
"""The clean "downloaded from" URL for a job — shared by the private Downloads page and the public """The clean "downloaded from" URL for a job — shared by the Files library and the public
watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's watch page so both surfaces always agree. None for edit derivatives (a clip's source is the user's
own earlier download, not a web page) and for a Plex clip (the library is not a public page); a own earlier download, not a web page) and for a Plex clip (the library is not a public page); a
YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else YouTube clip links its video at the clip's start; else the canonical page URL yt-dlp recorded; else
+20
View File
@@ -11,6 +11,7 @@ a Plex library using a metadata agent picks up rich info. Channel = "show", vide
date-based — the community-standard mapping for YouTube-in-Plex. date-based — the community-standard mapping for YouTube-in-Plex.
""" """
import os
import re import re
import shutil import shutil
import unicodedata import unicodedata
@@ -67,6 +68,25 @@ _FOLD_MAP = str.maketrans(
) )
def files_root_problem(root: str, system_root: str = "/") -> str | None:
"""Why `root` cannot be the files library's volume, or None when it can. A root that sits on the
container's own filesystem is the image's empty directory, not a mounted volume: the library
would answer 404 for every file, and what a download writes would go when the container does —
the state an install is in when its compose file still mounts the library somewhere else."""
path = Path(root)
if not path.is_dir():
return f"{root} does not exist"
try:
if path.stat().st_dev == Path(system_root).stat().st_dev:
return f"{root} is not a mounted volume (it is the container's own empty directory)"
# silent-ok: the failure IS the answer — returned as the reason, which the caller logs.
except OSError as e:
return f"{root} cannot be read ({e})"
if not os.access(path, os.W_OK):
return f"{root} is not writable"
return None
def _ascii_fold(text: str) -> str: def _ascii_fold(text: str) -> str:
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
+5
View File
@@ -28,6 +28,7 @@ log = logs.get_logger(__name__)
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
from app.config import settings # noqa: E402 — logging is configured first from app.config import settings # noqa: E402 — logging is configured first
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
from app.downloads import storage # noqa: E402 — logging is configured first
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
from app.routes import ( # noqa: E402 — logging is configured first from app.routes import ( # noqa: E402 — logging is configured first
@@ -64,6 +65,10 @@ from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 —
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
log.info("Siftlode starting up") log.info("Siftlode starting up")
problem = storage.files_root_problem(settings.files_root)
if problem:
# The rest of the app still serves; the files library cannot (the worker refuses to start).
log.critical("The files library is not reachable: %s. See docs/self-hosting.md.", problem)
# The stored copy of the log (log_events). The entrypoint has already migrated the schema. # The stored copy of the log (log_events). The entrypoint has already migrated the schema.
logsink.start() logsink.start()
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print # First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
+6 -1
View File
@@ -877,7 +877,7 @@ class MediaAsset(Base, TimestampMixin):
upload_date: Mapped[date | None] = mapped_column(Date) upload_date: Mapped[date | None] = mapped_column(Date)
thumbnail_url: Mapped[str | None] = mapped_column(Text) thumbnail_url: Mapped[str | None] = mapped_column(Text)
# yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from" # yt-dlp's canonical page URL for the source (webpage_url) — the clean "downloaded from"
# reference shown on the Downloads page. Null until the worker extracts it (older/queued rows # reference shown in the Files library. Null until the worker extracts it (older/queued rows
# fall back to a URL derived from source_kind+source_ref). # fall back to a URL derived from source_kind+source_ref).
source_webpage_url: Mapped[str | None] = mapped_column(Text) source_webpage_url: Mapped[str | None] = mapped_column(Text)
error: Mapped[str | None] = mapped_column(Text) error: Mapped[str | None] = mapped_column(Text)
@@ -1036,6 +1036,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
folder_id: Mapped[int | None] = mapped_column( folder_id: Mapped[int | None] = mapped_column(
ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True ForeignKey("file_folders.id", ondelete="RESTRICT"), index=True
) )
# The origin the file came from (the source folder's key), recorded where it is placed: an edit
# traces its origin through its parent, and that parent may be deleted or expire later — the
# file must still know which source folder it may go back to. NULL only for a file placed before
# the column existed outside a source folder; `folders.source_key_for` works those out.
source_key: Mapped[str | None] = mapped_column(String(260))
kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") kept: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") expiry_warned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
+5 -1
View File
@@ -399,6 +399,8 @@ def enqueue_clip(
# How long a finished job stays in the queue: the tray polls it from every page and must see a job # How long a finished job stays in the queue: the tray polls it from every page and must see a job
# turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's. # turn done (its "Done · Open" row and the done notice) — the finished file itself is the library's.
# A done job is timed from when it became done, never from its last write: a rename, a move or the
# GC's expiry warning touches an old file's row, and the tray would announce it as just finished.
QUEUE_RECENT = timedelta(minutes=10) QUEUE_RECENT = timedelta(minutes=10)
@@ -411,7 +413,9 @@ def list_downloads(user: User = Depends(current_user), db: Session = Depends(get
select(DownloadJob) select(DownloadJob)
.where( .where(
DownloadJob.user_id == user.id, DownloadJob.user_id == user.id,
DownloadJob.status.not_in(FINISHED) | (DownloadJob.updated_at >= recent), DownloadJob.status.not_in(FINISHED)
| ((DownloadJob.status == JobStatus.DONE) & (DownloadJob.ready_at >= recent))
| ((DownloadJob.status == JobStatus.CANCELED) & (DownloadJob.updated_at >= recent)),
) )
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()) .order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
.limit(MAX_LIST) .limit(MAX_LIST)
+29 -11
View File
@@ -13,6 +13,7 @@ import base64
import hashlib import hashlib
import os import os
import tempfile import tempfile
import threading
from collections.abc import AsyncIterator, Sequence from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -23,7 +24,7 @@ from urllib.parse import unquote, urlparse
from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import FileResponse, Response from fastapi.responses import FileResponse, Response
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select from sqlalchemy import func, or_, select, update
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -59,6 +60,9 @@ MAX_LIST = 500
# The longest a folder may keep a file, in days — ten years is "forever" in all but name. # The longest a folder may keep a file, in days — ten years is "forever" in all but name.
_MAX_RETENTION_DAYS = 3650 _MAX_RETENTION_DAYS = 3650
_HASH_CHUNK = 1 << 20 _HASH_CHUNK = 1 << 20
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
_HASHERS = 2
_HASHING = threading.BoundedSemaphore(_HASHERS)
# --- shared with routes.downloads ------------------------------------------------------- # --- shared with routes.downloads -------------------------------------------------------
@@ -235,7 +239,7 @@ def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
if not raw.strip(): if not raw.strip():
return None return None
try: try:
return folders.upload_target(db, user_id, int(raw)).id return folders.upload_target(db, user_id, int(raw), hold=False).id
except ValueError as e: except ValueError as e:
raise refused(409, "file_folder_gone") from e raise refused(409, "file_folder_gone") from e
except folders.FolderError as e: except folders.FolderError as e:
@@ -664,12 +668,9 @@ def update_file(
if "folder_id" in sent: if "folder_id" in sent:
if body.folder_id is None: if body.folder_id is None:
raise refused(400, "file_folder_gone") raise refused(400, "file_folder_gone")
folder = db.get(FileFolder, body.folder_id)
if folder is None or folder.user_id != user.id:
raise refused(409, "file_folder_gone")
before = job.folder_id before = job.folder_id
try: try:
folders.move(db, job, folder) folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
except folders.FolderError as e: except folders.FolderError as e:
raise refused(409, e.reason) from e raise refused(409, e.reason) from e
if job.folder_id != before: if job.folder_id != before:
@@ -842,13 +843,30 @@ def get_sha256(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db) job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict: ) -> dict:
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder. """The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool.""" A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
past that is told to come back), and none of them holds a database connection while it reads."""
asset, path = _ready_file(db, accessible_job(db, user, job_id)) asset, path = _ready_file(db, accessible_job(db, user, job_id))
if not asset.sha256: if asset.sha256:
return {"sha256": asset.sha256}
asset_id = asset.id
if not _HASHING.acquire(blocking=False):
raise refused(429, "file_hash_busy")
try:
release_read_connection(db)
h = hashlib.sha256() h = hashlib.sha256()
with path.open("rb") as f: with path.open("rb") as f:
while chunk := f.read(_HASH_CHUNK): while chunk := f.read(_HASH_CHUNK):
h.update(chunk) h.update(chunk)
asset.sha256 = h.hexdigest() except OSError as e:
db.commit() raise not_found("file") from e
return {"sha256": asset.sha256} finally:
_HASHING.release()
digest = h.hexdigest()
db.execute(
update(MediaAsset)
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
.values(sha256=digest)
)
db.commit()
return {"sha256": digest}
+7 -1
View File
@@ -37,7 +37,7 @@ from app import sysconfig
from app.config import settings from app.config import settings
from app.db import SessionLocal from app.db import SessionLocal
from app.downloads import edit as editmod from app.downloads import edit as editmod
from app.downloads import credentials, formats, quota, service from app.downloads import credentials, formats, quota, service, storage
from app.downloads import errors as dlerrors from app.downloads import errors as dlerrors
from app.downloads.jobdb import ( from app.downloads.jobdb import (
_edit_staging_dir, _edit_staging_dir,
@@ -710,6 +710,12 @@ def main() -> None:
# `log_events`, so on the deploy that first adds the table the worker may start before the API # `log_events`, so on the deploy that first adds the table the worker may start before the API
# has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks. # has migrated — those few seconds of batches fail and say so on stderr; nothing else breaks.
logsink.start() logsink.start()
problem = storage.files_root_problem(settings.files_root)
if problem:
# Refuse, loudly: downloading into a root that is not the volume loses every file with the
# container. The queue waits until the mount is fixed (see docs/self-hosting.md).
log.critical("The files library cannot be written: %s. The worker will not start.", problem)
raise SystemExit(1)
_recover_orphans() _recover_orphans()
global _target global _target
_target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer _target = max(1, settings.download_worker_concurrency) # used only if the DB can't answer
+133
View File
@@ -15,6 +15,7 @@ from app import sysconfig
from app.config import settings from app.config import settings
from app.downloads import folders, formats, service from app.downloads import folders, formats, service
from app.downloads import gc as gcmod from app.downloads import gc as gcmod
from app.db import SessionLocal
from app.downloads.status import JobStatus from app.downloads.status import JobStatus
from app.models import DownloadJob, FileFolder, MediaAsset, Notification from app.models import DownloadJob, FileFolder, MediaAsset, Notification
@@ -123,6 +124,68 @@ def test_an_edit_whose_parent_is_gone_follows_the_parents_asset(db, make_user):
assert folders.source_key_for(db, edit) == "site:vimeo.com" assert folders.source_key_for(db, edit) == "site:vimeo.com"
def test_an_edit_keeps_its_origin_after_its_parent_and_its_file_are_gone(db, make_user):
# The parent download expires, its file goes with its last holder, and both of the edit's
# pointers to it are cleared: the edit must still go back to the YouTube folder it came from.
uid = make_user().id
parent = _job(db, uid, _asset(db))
edit = _job(
db, uid, _asset(db, "edit", "e1"), kind="edit", source_job_id=parent.id,
source_asset_id=parent.asset_id,
) # fmt: skip
home = _folder(db, edit)
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.flush()
folders.move(db, edit, own)
edit.source_job_id = edit.source_asset_id = None # what ON DELETE SET NULL leaves
db.delete(parent)
db.commit()
folders.move(db, edit, home)
assert edit.folder_id == home.id
folders.move(db, edit, own)
folders.delete_own(db, own)
assert edit.folder_id == home.id
def test_a_folder_deleted_while_a_file_goes_in_takes_that_file_home_too(db, make_user):
# Before, the delete missed the file still on its way in and failed on the foreign key (a 500),
# or the file's insert did after the folder went.
import threading
uid = make_user().id
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.commit()
own_id = own.id
job = _job(db, uid, _asset(db))
home = job.folder_id
target = folders.own_folder(db, uid, own_id) # held from the check until this commits
outcome: list[object] = []
def delete_it():
other = SessionLocal()
try:
outcome.append(folders.delete_own(other, other.get(FileFolder, own_id)))
other.commit()
except Exception as e: # noqa: BLE001 — the assertion below names what went wrong
outcome.append(e)
finally:
other.close()
t = threading.Thread(target=delete_it)
t.start()
t.join(timeout=1)
assert t.is_alive() # the delete waits for the file, even before it is written
folders.move(db, job, target)
db.commit()
t.join(timeout=10)
assert outcome == [1]
db.expire_all()
assert db.get(DownloadJob, job.id).folder_id == home
assert db.get(FileFolder, own_id) is None
def test_folders_are_per_user(db, make_user): def test_folders_are_per_user(db, make_user):
a, b = make_user().id, make_user().id a, b = make_user().id, make_user().id
asset = _asset(db) asset = _asset(db)
@@ -201,6 +264,30 @@ def test_the_gc_warns_once_then_takes_the_file_out_of_the_library(db, make_user,
assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0 assert db.execute(select(func.count()).select_from(MediaAsset)).scalar() == 0
@pytest.mark.parametrize("restart", ["unkeep", "move", "retention"])
def test_a_file_whose_clock_starts_again_is_warned_again(db, make_user, _root, restart):
# The GC warns once per expiry; a file whose expiry was moved must be warned before the new one.
(_root / "a.mp4").write_bytes(b"x")
uid = make_user().id
job = _job(db, uid, _asset(db), expires_at=NOW + timedelta(hours=12))
_gc(db)
assert job.expiry_warned
if restart == "unkeep":
folders.set_kept(db, job, True)
folders.set_kept(db, job, False)
elif restart == "move":
home = _folder(db, job)
own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine")
db.add(own)
db.flush()
folders.move(db, job, own)
folders.move(db, job, home)
else:
folders.set_retention(db, _folder(db, job), 90)
db.commit()
assert not job.expiry_warned
def test_the_gc_never_takes_a_kept_file(db, make_user, _root): def test_the_gc_never_takes_a_kept_file(db, make_user, _root):
(_root / "a.mp4").write_bytes(b"x") (_root / "a.mp4").write_bytes(b"x")
uid = make_user().id uid = make_user().id
@@ -221,6 +308,52 @@ def test_one_users_expiry_leaves_another_users_copy(db, make_user, _root):
assert (_root / "a.mp4").exists() assert (_root / "a.mp4").exists()
def _meanwhile(monkeypatch, act):
"""Run `act` in another session (another request) between the GC's expiry SELECT and its
handling of each file it found — the window a user's click can land in."""
real = gcmod._relock_expired
def relock(db, job_id, asset_id, now):
other = SessionLocal()
try:
act(other, job_id)
other.commit()
finally:
other.close()
return real(db, job_id, asset_id, now)
monkeypatch.setattr(gcmod, "_relock_expired", relock)
def test_a_file_kept_while_the_gc_runs_stays(db, make_user, _root, monkeypatch):
(_root / "a.mp4").write_bytes(b"x")
job = _job(db, make_user().id, _asset(db), expires_at=NOW - timedelta(minutes=1))
_meanwhile(monkeypatch, lambda other, jid: setattr(other.get(DownloadJob, jid), "kept", True))
assert _gc(db)["expired"] == 0
db.expire_all()
assert db.get(DownloadJob, job.id) is not None
assert (_root / "a.mp4").exists()
def test_a_file_deleted_while_the_gc_runs_is_released_once(db, make_user, _root, monkeypatch):
# Released twice, the shared file would go from under the other user's copy.
(_root / "a.mp4").write_bytes(b"x")
a, b = make_user().id, make_user().id
asset = _asset(db)
_job(db, a, asset, expires_at=NOW - timedelta(minutes=1))
stays = _job(db, b, asset, expires_at=NOW + timedelta(days=20))
def user_deletes(other, jid):
service.delete(other, other.get(DownloadJob, jid))
_meanwhile(monkeypatch, user_deletes)
_gc(db)
db.expire_all()
assert db.get(DownloadJob, stays.id).asset_id == asset.id
assert db.get(MediaAsset, asset.id).ref_count == 1
assert (_root / "a.mp4").exists()
def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library( def test_the_cache_cap_never_evicts_a_kept_file_and_an_evicted_one_leaves_every_library(
db, make_user, _root db, make_user, _root
): ):
+55 -1
View File
@@ -421,6 +421,21 @@ def test_a_deleted_file_leaves_the_library_and_the_log_says_so(db, make_user, as
assert (row.actor_id, row.target_id) == (alice.id, str(job.id)) assert (row.actor_id, row.target_id) == (alice.id, str(job.id))
def test_a_file_shared_with_someone_is_theirs_to_watch_not_to_change(db, make_user, as_user, _root):
# The read routes let a share's recipient in; the write routes must not.
alice, friend = make_user(), make_user()
job = _file(db, alice.id, title="Mine")
db.add(DownloadShare(job_id=job.id, from_user_id=alice.id, to_user_id=friend.id))
db.commit()
c = as_user(friend)
assert c.patch(f"/api/files/{job.id}", json={"name": "Theirs"}).status_code == 404
assert c.patch(f"/api/files/{job.id}", json={"kept": True}).status_code == 404
assert c.delete(f"/api/files/{job.id}").status_code == 404
db.expire_all()
still = db.get(DownloadJob, job.id)
assert still is not None and still.display_name is None and not still.kept
# --- upload, the file, its hash ------------------------------------------------------------ # --- upload, the file, its hash ------------------------------------------------------------
@@ -532,6 +547,26 @@ def test_the_file_carries_its_hash_once_known_and_the_hash_is_taken_on_first_ask
assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404 assert as_user(make_user()).get(f"/api/files/{job.id}/sha256").status_code == 404
def test_only_a_few_files_are_hashed_at_once(db, make_user, as_user, _root):
# Each hash holds a thread of the pool every sync route shares for the whole read; a caller
# past the few that may run is told to come back, not queued behind them.
from app.routes import files as filesroute
alice = make_user()
job = _file(db, alice.id, rel="v.mp4")
(_root / "v.mp4").write_bytes(b"bytes")
held = [filesroute._HASHING.acquire(blocking=False) for _ in range(filesroute._HASHERS)]
try:
assert all(held)
busy = as_user(alice).get(f"/api/files/{job.id}/sha256")
finally:
for _ in held:
filesroute._HASHING.release()
assert busy.status_code == 429
assert busy.json()["detail"]["reason"] == "file_hash_busy"
assert as_user(alice).get(f"/api/files/{job.id}/sha256").status_code == 200
def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user): def test_a_new_file_at_an_assets_path_forgets_the_old_hash(db, make_user):
job = _file(db, make_user().id) job = _file(db, make_user().id)
asset = db.get(MediaAsset, job.asset_id) asset = db.get(MediaAsset, job.asset_id)
@@ -562,7 +597,7 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library(
db.execute( db.execute(
update(DownloadJob) update(DownloadJob)
.where(DownloadJob.id == job.id) .where(DownloadJob.id == job.id)
.values(updated_at=NOW - timedelta(hours=1)) .values(updated_at=NOW - timedelta(hours=1), ready_at=NOW - timedelta(hours=1))
) )
db.commit() db.commit()
rows = as_user(user).get("/api/downloads").json() rows = as_user(user).get("/api/downloads").json()
@@ -570,3 +605,22 @@ def test_the_queue_is_what_is_in_flight_and_what_just_finished_not_the_library(
assert {row["id"] for row in rows} == {fresh.id, waiting.id, stuck.id, paused.id, stopped.id} assert {row["id"] for row in rows} == {fresh.id, waiting.id, stuck.id, paused.id, stopped.id}
# The tray names where each came from, as the library does. # The tray names where each came from, as the library does.
assert {row["source_key"] for row in rows} == {"youtube"} assert {row["source_key"] for row in rows} == {"youtube"}
def test_an_old_file_written_to_does_not_come_back_as_just_finished(db, make_user, as_user):
# Renaming, keeping or moving touches the row; the tray would announce the file as ready again.
from sqlalchemy import update
user = make_user()
old = _file(db, user.id, title="Old")
db.execute(
update(DownloadJob)
.where(DownloadJob.id == old.id)
.values(updated_at=NOW - timedelta(days=30), ready_at=NOW - timedelta(days=30))
)
db.commit()
c = as_user(user)
assert (
c.patch(f"/api/files/{old.id}", json={"name": "Renamed", "kept": True}).status_code == 200
)
assert [row["id"] for row in c.get("/api/downloads").json()] == []
+68
View File
@@ -71,3 +71,71 @@ def test_existing_jobs_are_placed_and_keep_their_expiry(db, make_user):
} }
# alice: youtube, tiktok, upload, plex; bob: youtube. # alice: youtube, tiktok, upload, plex; bob: youtube.
assert folders == 5 assert folders == 5
def test_the_admins_retention_is_kept_and_the_downgrade_gives_the_asset_back_its_expiry(
db, make_user
):
alice, bob = make_user().id, make_user().id
cfg = Config("alembic.ini")
command.downgrade(cfg, "0101_plex_text_plain")
try:
with engine.begin() as conn:
conn.execute(
text(
"INSERT INTO system_config (key, value) VALUES ('download_retention_days', '7')"
" ON CONFLICT (key) DO UPDATE SET value = '7'"
)
)
yt = conn.execute(
text(
"INSERT INTO media_assets (source_kind, source_ref, format_sig, status,"
" ref_count, expires_at) VALUES ('youtube', 'abcdefghijk', 'sig', 'ready', 2,"
" '2030-01-02T00:00:00+00:00') RETURNING id"
)
).scalar_one()
for user in (alice, bob):
conn.execute(
text(
"INSERT INTO download_jobs (user_id, asset_id, source_kind, source_ref,"
" profile_snapshot, status) VALUES (:u, :a, 'youtube', 'abcdefghijk',"
" '{}', 'done')"
),
{"u": user, "a": yt},
)
command.upgrade(cfg, "head")
with engine.connect() as conn:
retention = conn.execute(
text(
"SELECT DISTINCT retention_days FROM file_folders WHERE source_key = 'youtube'"
)
).all()
def asset_expiry_after_downgrade(alice_at, bob_at):
with engine.begin() as conn:
for user, at in ((alice, alice_at), (bob, bob_at)):
conn.execute(
text(
"UPDATE download_jobs SET expires_at = CAST(:at AS timestamptz)"
" WHERE asset_id = :a AND user_id = :u"
),
{"at": at, "a": yt, "u": user},
)
command.downgrade(cfg, "0101_plex_text_plain")
with engine.connect() as conn:
got = conn.execute(
text("SELECT expires_at::date FROM media_assets WHERE id = :a"), {"a": yt}
).scalar()
command.upgrade(cfg, "head")
return str(got) if got else None
# The shared file goes when its last holder's copy would; one holder keeping it keeps it.
latest = asset_expiry_after_downgrade(
"2030-02-01T00:00:00+00:00", "2030-03-01T00:00:00+00:00"
)
kept_by_one = asset_expiry_after_downgrade("2030-02-01T00:00:00+00:00", None)
finally:
command.upgrade(cfg, "head")
assert retention == [(7,)]
assert latest == "2030-03-01"
assert kept_by_one is None
+5 -15
View File
@@ -1,5 +1,5 @@
"""Migration 0104's data step, DB-lane: a done file from before is dated by when it finished where """Migration 0104's data step, DB-lane: a done file from before is dated by when it was asked for
its "ready to watch" notice says so, else by when it was asked for; a job not done has no date.""" (where it already sat); a job not done has no date."""
from alembic import command from alembic import command
from alembic.config import Config from alembic.config import Config
@@ -8,7 +8,7 @@ from sqlalchemy import text
from app.db import engine from app.db import engine
def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user): def test_a_done_file_takes_its_request_time(db, make_user):
uid = make_user().id uid = make_user().id
cfg = Config("alembic.ini") cfg = Config("alembic.ini")
command.downgrade(cfg, "0103_files_sha256") command.downgrade(cfg, "0103_files_sha256")
@@ -25,17 +25,8 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user):
{"u": uid, "s": status, "c": created}, {"u": uid, "s": status, "c": created},
).scalar_one() ).scalar_one()
retried = job("done", "2026-10-01T10:00:00+00:00") done = job("done", "2026-10-02T10:00:00+00:00")
quiet = job("done", "2026-10-02T10:00:00+00:00")
failed = job("error", "2026-10-03T10:00:00+00:00") failed = job("error", "2026-10-03T10:00:00+00:00")
for at in ("2026-10-05T09:00:00+00:00", "2026-10-06T09:00:00+00:00"):
conn.execute(
text(
"INSERT INTO notifications (user_id, type, title, data, created_at) VALUES"
" (:u, 'download_done', 't', CAST(:d AS json), CAST(:c AS timestamptz))"
),
{"u": uid, "d": f'{{"job_id": {retried}}}', "c": at},
)
finally: finally:
command.upgrade(cfg, "head") command.upgrade(cfg, "head")
@@ -47,6 +38,5 @@ def test_a_done_file_takes_its_notice_time_else_its_request_time(db, make_user):
) )
).all() ).all()
) )
assert rows[retried] == "10-06 09" # its last finish assert rows[done] == "10-02 10"
assert rows[quiet] == "10-02 10" # no notice left: where it already sat
assert rows[failed] is None assert rows[failed] is None
+45
View File
@@ -0,0 +1,45 @@
"""Migration 0105's data step, DB-lane: a file in a source folder records that folder's key as its
origin; a file in an own folder is left to be worked out, as before."""
from alembic import command
from alembic.config import Config
from sqlalchemy import text
from app.db import engine
def test_a_file_in_a_source_folder_records_its_origin(db, make_user):
uid = make_user().id
cfg = Config("alembic.ini")
command.downgrade(cfg, "0104_job_ready_at")
try:
with engine.begin() as conn:
def folder(kind, key, name):
return conn.execute(
text(
"INSERT INTO file_folders (user_id, kind, source_key, name) VALUES"
" (:u, :k, :s, :n) RETURNING id"
),
{"u": uid, "k": kind, "s": key, "n": name},
).scalar_one()
def job(folder_id):
return conn.execute(
text(
"INSERT INTO download_jobs (user_id, source_kind, source_ref,"
" profile_snapshot, status, folder_id) VALUES (:u, 'url', 'x', '{}',"
" 'queued', :f) RETURNING id"
),
{"u": uid, "f": folder_id},
).scalar_one()
in_source = job(folder("source", "site:vimeo.com", None))
in_own = job(folder("own", None, "Mine"))
finally:
command.upgrade(cfg, "head")
with engine.connect() as conn:
rows = dict(conn.execute(text("SELECT id, source_key FROM download_jobs")).all())
assert rows[in_source] == "site:vimeo.com"
assert rows[in_own] is None
+22
View File
@@ -6,6 +6,7 @@ from datetime import date
from app.downloads.storage import ( from app.downloads.storage import (
MediaMeta, MediaMeta,
download_filename, download_filename,
files_root_problem,
rel_path, rel_path,
safe_abs_path, safe_abs_path,
sanitize, sanitize,
@@ -174,3 +175,24 @@ def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path
assert f"{stem}.nfo" in names, names assert f"{stem}.nfo" in names, names
assert f"{stem}.jpg" in names, names assert f"{stem}.jpg" in names, names
assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan
def test_a_files_root_on_the_containers_own_disk_is_refused(tmp_path):
# The image's empty /files, left unmounted by an old compose file: every file would 404 and
# every download would go with the container.
assert "not a mounted volume" in files_root_problem(
str(tmp_path), system_root=str(tmp_path.parent)
)
assert "does not exist" in files_root_problem(str(tmp_path / "gone"))
# On another filesystem than the container's own, and writable: the volume.
assert files_root_problem(str(tmp_path), system_root="/proc") is None
def test_the_old_name_of_the_files_root_still_finds_the_files(monkeypatch):
from app.config import Settings
monkeypatch.delenv("FILES_ROOT", raising=False)
monkeypatch.setenv("DOWNLOAD_ROOT", "/downloads")
assert Settings().files_root == "/downloads"
monkeypatch.setenv("FILES_ROOT", "/files")
assert Settings().files_root == "/files"
+15
View File
@@ -67,6 +67,7 @@ def test_worker_boots_with_the_stored_concurrency_and_log_level(monkeypatch, sto
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True) monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
monkeypatch.setattr(worker.logsink, "start", lambda: None) monkeypatch.setattr(worker.logsink, "start", lambda: None)
monkeypatch.setattr(worker, "_recover_orphans", lambda: None) monkeypatch.setattr(worker, "_recover_orphans", lambda: None)
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: None)
threading.Timer(0.5, worker._stop.set).start() threading.Timer(0.5, worker._stop.set).start()
worker.main() worker.main()
@@ -107,3 +108,17 @@ def test_an_unreadable_database_keeps_what_runs(monkeypatch, levels, loops):
assert worker._target == 2 assert worker._target == 2
assert sorted(threads) == [0, 1] # the loops still run on the boot value assert sorted(threads) == [0, 1] # the loops still run on the boot value
assert levels == [] assert levels == []
def test_the_worker_will_not_start_on_a_files_root_that_is_not_the_volume(monkeypatch, loops):
# Files downloaded into the container's own empty /files go with the container: refuse, loudly.
monkeypatch.setattr(worker.settings, "worker_enabled", True)
monkeypatch.setattr(worker.logs, "configure_logging", lambda _source: None)
monkeypatch.setattr(worker.signal, "signal", lambda *_a: None)
monkeypatch.setattr(worker, "_wait_for_schema", lambda: True)
monkeypatch.setattr(worker.logsink, "start", lambda: None)
monkeypatch.setattr(worker.storage, "files_root_problem", lambda _root: "not a mounted volume")
with pytest.raises(SystemExit) as exit_:
worker.main()
assert exit_.value.code == 1
assert loops == []
+2 -2
View File
@@ -72,7 +72,7 @@ services:
# The public instance owns the background scheduler (single writer). # The public instance owns the background scheduler (single writer).
SCHEDULER_ENABLED: "true" SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips (read-only root is fine — # Download center: the API serves finished files + builds filmstrips (read-only root is fine —
# it only writes to the /downloads mount + tmpfs). The yt-dlp job loop runs in `worker` below. # it only writes to the /files mount + tmpfs). The yt-dlp job loop runs in `worker` below.
FILES_ROOT: /files FILES_ROOT: /files
WORKER_ENABLED: "false" WORKER_ENABLED: "false"
depends_on: depends_on:
@@ -113,7 +113,7 @@ services:
restart: unless-stopped restart: unless-stopped
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
# DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
# $HOME caches + the staging dir); heavier limits since it does the video encoding. # $HOME caches + the staging dir); heavier limits since it does the video encoding.
worker: worker:
image: registry.example.com/you/siftlode:${APP_VERSION:-latest} image: registry.example.com/you/siftlode:${APP_VERSION:-latest}
+2 -2
View File
@@ -48,7 +48,7 @@ services:
# This instance owns the background scheduler (single writer). # This instance owns the background scheduler (single writer).
SCHEDULER_ENABLED: "true" SCHEDULER_ENABLED: "true"
# Download center: the API serves finished files + builds filmstrips; the worker (below) runs # Download center: the API serves finished files + builds filmstrips; the worker (below) runs
# the yt-dlp job loop. Read-only root is fine — it only writes to the /downloads mount + tmpfs. # the yt-dlp job loop. Read-only root is fine — it only writes to the /files mount + tmpfs.
FILES_ROOT: /files FILES_ROOT: /files
WORKER_ENABLED: "false" WORKER_ENABLED: "false"
depends_on: depends_on:
@@ -85,7 +85,7 @@ services:
restart: unless-stopped restart: unless-stopped
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
# DB (job queue) and the downloads mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to # DB (job queue) and the /files mount with the API. Not read-only (yt-dlp/deno/ffmpeg write to
# $HOME caches + the staging dir). # $HOME caches + the staging dir).
worker: worker:
image: git.b1fr0st.eu/peter/siftlode:${IMAGE_TAG:-latest} image: git.b1fr0st.eu/peter/siftlode:${IMAGE_TAG:-latest}
+1 -1
View File
@@ -63,7 +63,7 @@ services:
restart: unless-stopped restart: unless-stopped
# Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the # Download worker: same image, runs the yt-dlp / ffmpeg job loop instead of the API. Shares the
# DB (job queue) + the downloads mount with the API. # DB (job queue) + the /files mount with the API.
worker: worker:
build: build:
context: . context: .
+26 -10
View File
@@ -254,17 +254,33 @@ Then `pull` + `up -d` stays on that version until you change the line. Released
changed in them are listed in the app itself, under **Release Notes** in the account menu, and in changed in them are listed in the app itself, under **Release Notes** in the account menu, and in
[`CHANGELOG.md`](../CHANGELOG.md). [`CHANGELOG.md`](../CHANGELOG.md).
**The media mount was renamed** (the Files library): `DOWNLOAD_HOST_PATH` is now `FILES_HOST_PATH`, **The media mount was renamed** (the Files library): it is mounted at `/files` (`FILES_ROOT`), set
mounted at `/files`, and the default volume is `siftlode_files` instead of `siftlode_downloads`. If you from `FILES_HOST_PATH`, and the default volume is `siftlode_files`. Until you move, nothing breaks: a
set a host directory, rename the variable in `.env`. If you used the default volume, copy it over once compose file from before still mounts `/downloads` and sets `DOWNLOAD_ROOT`, and the app still reads
before you start the new version (Docker cannot rename a volume), then remove the old one when the that name. To move to the new layout, download the new
library looks right: [`docker-compose.selfhost.yml`](../docker-compose.selfhost.yml) over your old one, then:
```bash - **If you set a host directory**, rename `DOWNLOAD_HOST_PATH` to `FILES_HOST_PATH` in `.env`.
docker compose -f docker-compose.selfhost.yml down - **If you used the default volume**, copy it over once before you start the new version (Docker
docker volume create siftlode_files cannot rename a volume). Compose names a volume after the project, which is your folder's name
docker run --rm -v siftlode_downloads:/from -v siftlode_files:/to alpine cp -a /from/. /to/ unless you set one, so look the old one up rather than guessing it:
```
```bash
docker compose -f docker-compose.selfhost.yml down
docker volume ls --format '{{.Name}}' | grep 'siftlode_downloads$' # e.g. mysiftlode_siftlode_downloads
OLD=mysiftlode_siftlode_downloads # the name it printed
NEW="${OLD%siftlode_downloads}siftlode_files"
docker volume create "$NEW"
docker run --rm -v "$OLD":/from -v "$NEW":/to alpine cp -a /from/. /to/
docker compose -f docker-compose.selfhost.yml up -d
```
Open the Files library and check your files play; only then remove the old volume
(`docker volume rm "$OLD"`).
If the files library is not mounted where the app looks, the API says so in its log at start
(`The files library is not reachable`), and the download worker refuses to start rather than write
files into a container that throws them away.
**The registry keeps only the newest few releases, not every version the changelog lists.** A pin is **The registry keeps only the newest few releases, not every version the changelog lists.** A pin is
"decide when you take the next hop", not "stay here forever": once you are several releases behind, "decide when you take the next hop", not "stay here forever": once you are several releases behind,
+1
View File
@@ -237,6 +237,7 @@
"file_folder_name_taken": "You already have a folder with that name", "file_folder_name_taken": "You already have a folder with that name",
"file_folder_not_own": "Only a folder you made can be renamed or deleted", "file_folder_not_own": "Only a folder you made can be renamed or deleted",
"file_folder_gone": "That folder no longer exists", "file_folder_gone": "That folder no longer exists",
"file_hash_busy": "Too many files are being checked right now — try again in a moment",
"file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from", "file_folder_not_its_source": "A file can only go into one of your own folders or back to the folder it came from",
"file_no_preview": "There is no preview for this file" "file_no_preview": "There is no preview for this file"
} }
+1
View File
@@ -237,6 +237,7 @@
"file_folder_name_taken": "Már van ilyen nevű mappád", "file_folder_name_taken": "Már van ilyen nevű mappád",
"file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted", "file_folder_not_own": "Csak a saját mappádat nevezheted át vagy törölheted",
"file_folder_gone": "Ez a mappa már nem létezik", "file_folder_gone": "Ez a mappa már nem létezik",
"file_hash_busy": "Most túl sok fájl ellenőrzése fut — próbáld újra kicsit később",
"file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet", "file_folder_not_its_source": "Egy fájl csak a saját mappáidba vagy vissza az eredeti mappájába kerülhet",
"file_no_preview": "Ehhez a fájlhoz nincs előnézet" "file_no_preview": "Ehhez a fájlhoz nincs előnézet"
} }
+1 -1
View File
@@ -29,7 +29,7 @@ SECRET_KEY=$secretKey
TOKEN_ENCRYPTION_KEY=$fernet TOKEN_ENCRYPTION_KEY=$fernet
OAUTH_REDIRECT_URL=$url/auth/callback OAUTH_REDIRECT_URL=$url/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead # Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 <dir>). # of a Docker volume. Must be writable by uid 1000 (chown -R 1000:1000 <dir>).
# FILES_HOST_PATH=/mnt/media/youtube # FILES_HOST_PATH=/mnt/media/youtube
"@ | Set-Content -Path .env -Encoding ascii "@ | Set-Content -Path .env -Encoding ascii
+1 -1
View File
@@ -27,7 +27,7 @@ SECRET_KEY=$SECRET_KEY
TOKEN_ENCRYPTION_KEY=$TOKEN_ENCRYPTION_KEY TOKEN_ENCRYPTION_KEY=$TOKEN_ENCRYPTION_KEY
OAUTH_REDIRECT_URL=$URL/auth/callback OAUTH_REDIRECT_URL=$URL/auth/callback
# Optional: store Download Center media in a host folder (e.g. one your Plex server reads) instead # Optional: keep the Files library in a host folder (e.g. one your Plex server reads) instead
# of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 <dir>). # of a Docker volume. Must be writable by uid 1000 (sudo chown -R 1000:1000 <dir>).
# FILES_HOST_PATH=/mnt/media/youtube # FILES_HOST_PATH=/mnt/media/youtube
EOF EOF