fix(files): review round 2, the server — locks the GC and folders read under, the mount check, the old mount name

- GC: the expiry warning marks each file conditionally on the expiry it saw, one commit each (a file
  deleted meanwhile no longer fails the run; one re-timed keeps its fresh expiry unwarned); eviction
  counts the disk again when a candidate was freed meanwhile, so it never evicts for space already free.
- Folders: a retention change locks the folder and its files, and a finishing file reads the retention
  under a share lock, so neither misses the other; a folder delete locks the files it takes home (a
  file moved out meanwhile stays put, one deleted meanwhile no longer fails the delete).
- Upload: the folder is checked before the probe and held only once the job names it.
- Dedup enqueue locks the cached asset, so a release meanwhile yields a fresh one instead of a 500.
- Mount check reads the kernel's mount table instead of comparing devices (the vfs storage driver gave
  a mounted volume the device of /), and only inside a container; the API also logs a fresh, empty
  volume in the library's place.
- Compose files fall back to DOWNLOAD_HOST_PATH; the upgrade notes cover the from-source pull.
- Folder routes call owned_or_404 directly.
This commit is contained in:
2026-10-13 02:32:49 +02:00
parent 8b2260c9a8
commit 54a6bbf07f
15 changed files with 543 additions and 93 deletions
+86 -49
View File
@@ -22,7 +22,7 @@ import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import func, or_, select
from sqlalchemy import func, or_, select, update
from sqlalchemy.orm import Session
from app import logs, sysconfig
@@ -82,6 +82,59 @@ def _relock_expired(
return job if still else None
def _mark_warned(db: Session, job_id: int, expires_at: datetime) -> DownloadJob | None:
"""Mark one file warned if it still goes when the pass saw it go — not deleted, kept, moved or
re-timed since (each of those sets a fresh expiry, and its own warning to come) — and return it.
The UPDATE's row lock holds it so until the commit."""
hit = db.execute(
update(DownloadJob)
.where(
DownloadJob.id == job_id,
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
DownloadJob.expiry_warned.is_(False),
DownloadJob.expires_at == expires_at,
)
.values(expiry_warned=True)
.returning(DownloadJob.id)
.execution_options(synchronize_session=False)
).scalar_one_or_none()
return db.get(DownloadJob, job_id) if hit is not None else None
def _on_disk(db: Session) -> int:
"""The bytes the cache holds now (every asset whose file is on disk)."""
return (
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
).scalar()
or 0
)
def _relock_evictable(db: Session, asset_id: int) -> tuple[MediaAsset | None, list[DownloadJob]]:
"""Re-read an eviction candidate and its holders under their locks (the asset, then its jobs —
the order the user routes take): between the pass's SELECT and here a holder may have kept it,
or every holder deleted it."""
asset = db.execute(
select(MediaAsset)
.where(MediaAsset.id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
holders = (
db.execute(
select(DownloadJob)
.where(DownloadJob.asset_id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
)
.scalars()
.all()
)
return asset, list(holders)
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
file goes with the last holder), delete the row (its shares and links cascade)."""
@@ -176,29 +229,28 @@ def run_download_gc(db: Session) -> dict:
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
warned = expired = evicted = reaped = 0
# 1. Pre-expiry warnings — one per file, the first run inside the grace window.
# 1. Pre-expiry warnings — one per file, the first run inside the grace window. Each mark is
# conditional on the expiry the SELECT saw and committed on its own: a file deleted since is
# passed over (a write to its gone row would fail the whole run), and one kept, moved or re-timed
# since keeps its fresh expiry unwarned.
warn_before = now + timedelta(days=grace_days)
soon = (
db.execute(
select(DownloadJob).where(
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
DownloadJob.expiry_warned.is_(False),
DownloadJob.expires_at.is_not(None),
DownloadJob.expires_at > now,
DownloadJob.expires_at <= warn_before,
)
soon = db.execute(
select(DownloadJob.id, DownloadJob.expires_at).where(
DownloadJob.status == JobStatus.DONE,
DownloadJob.kept.is_(False),
DownloadJob.expiry_warned.is_(False),
DownloadJob.expires_at.is_not(None),
DownloadJob.expires_at > now,
DownloadJob.expires_at <= warn_before,
)
.scalars()
.all()
)
for job in soon:
_notify(
db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
)
job.expiry_warned = True
warned += 1
if soon:
).all()
db.rollback() # the pass-1 snapshot holds no locks; each mark takes its own below
for job_id, expires_at in soon:
job = _mark_warned(db, job_id, expires_at)
if job is not None:
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
_notify(db, job, asset, "download_expiring")
warned += 1
db.commit()
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
@@ -224,19 +276,14 @@ def run_download_gc(db: Session) -> dict:
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
cap = sysconfig.get_int(db, "download_total_max_bytes")
if cap > 0:
total = (
db.execute(
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
).scalar()
or 0
)
total = _on_disk(db)
if total > cap:
kept_assets = select(DownloadJob.asset_id).where(
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
)
candidates = (
candidate_ids = (
db.execute(
select(MediaAsset)
select(MediaAsset.id)
.where(
MediaAsset.status == "ready",
MediaAsset.source_kind != "upload",
@@ -247,30 +294,20 @@ def run_download_gc(db: Session) -> dict:
.scalars()
.all()
)
candidate_ids = [a.id for a in candidates]
db.rollback()
for asset_id in candidate_ids:
if total <= cap:
break
# Locked and re-read like an expiry: a holder may have kept it, or every holder
# deleted it (the file is then already gone and counted out) since the SELECT.
asset = db.execute(
select(MediaAsset)
.where(MediaAsset.id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
).scalar_one_or_none()
holders = (
db.execute(
select(DownloadJob)
.where(DownloadJob.asset_id == asset_id)
.with_for_update()
.execution_options(populate_existing=True)
)
.scalars()
.all()
)
if asset is None or asset.status != "ready" or any(j.kept for j in holders):
# deleted it since the SELECT.
asset, holders = _relock_evictable(db, asset_id)
if asset is None or asset.status != "ready":
# Its file went (or is being fetched afresh) since the sum: the disk is counted
# again, or the pass would evict another user's file for space already free.
db.commit()
total = _on_disk(db)
continue
if any(j.kept for j in holders):
db.commit()
continue
total -= asset.size_bytes or 0