fix(files): review round 2, the server — locks the GC and folders read under, the mount check, the old mount name
- GC: the expiry warning marks each file conditionally on the expiry it saw, one commit each (a file deleted meanwhile no longer fails the run; one re-timed keeps its fresh expiry unwarned); eviction counts the disk again when a candidate was freed meanwhile, so it never evicts for space already free. - Folders: a retention change locks the folder and its files, and a finishing file reads the retention under a share lock, so neither misses the other; a folder delete locks the files it takes home (a file moved out meanwhile stays put, one deleted meanwhile no longer fails the delete). - Upload: the folder is checked before the probe and held only once the job names it. - Dedup enqueue locks the cached asset, so a release meanwhile yields a fresh one instead of a 500. - Mount check reads the kernel's mount table instead of comparing devices (the vfs storage driver gave a mounted volume the device of /), and only inside a container; the API also logs a fresh, empty volume in the library's place. - Compose files fall back to DOWNLOAD_HOST_PATH; the upgrade notes cover the from-source pull. - Folder routes call owned_or_404 directly.
This commit is contained in:
+86
-49
@@ -22,7 +22,7 @@ import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
@@ -82,6 +82,59 @@ def _relock_expired(
|
||||
return job if still else None
|
||||
|
||||
|
||||
def _mark_warned(db: Session, job_id: int, expires_at: datetime) -> DownloadJob | None:
|
||||
"""Mark one file warned if it still goes when the pass saw it go — not deleted, kept, moved or
|
||||
re-timed since (each of those sets a fresh expiry, and its own warning to come) — and return it.
|
||||
The UPDATE's row lock holds it so until the commit."""
|
||||
hit = db.execute(
|
||||
update(DownloadJob)
|
||||
.where(
|
||||
DownloadJob.id == job_id,
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at == expires_at,
|
||||
)
|
||||
.values(expiry_warned=True)
|
||||
.returning(DownloadJob.id)
|
||||
.execution_options(synchronize_session=False)
|
||||
).scalar_one_or_none()
|
||||
return db.get(DownloadJob, job_id) if hit is not None else None
|
||||
|
||||
|
||||
def _on_disk(db: Session) -> int:
|
||||
"""The bytes the cache holds now (every asset whose file is on disk)."""
|
||||
return (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def _relock_evictable(db: Session, asset_id: int) -> tuple[MediaAsset | None, list[DownloadJob]]:
|
||||
"""Re-read an eviction candidate and its holders under their locks (the asset, then its jobs —
|
||||
the order the user routes take): between the pass's SELECT and here a holder may have kept it,
|
||||
or every holder deleted it."""
|
||||
asset = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
holders = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return asset, list(holders)
|
||||
|
||||
|
||||
def _drop_job(db: Session, job: DownloadJob, ntype: str) -> None:
|
||||
"""Take a file out of its user's library: tell them, release the job's hold on the asset (the
|
||||
file goes with the last holder), delete the row (its shares and links cascade)."""
|
||||
@@ -176,29 +229,28 @@ def run_download_gc(db: Session) -> dict:
|
||||
grace_days = sysconfig.get_int(db, "download_gc_grace_days")
|
||||
warned = expired = evicted = reaped = 0
|
||||
|
||||
# 1. Pre-expiry warnings — one per file, the first run inside the grace window.
|
||||
# 1. Pre-expiry warnings — one per file, the first run inside the grace window. Each mark is
|
||||
# conditional on the expiry the SELECT saw and committed on its own: a file deleted since is
|
||||
# passed over (a write to its gone row would fail the whole run), and one kept, moved or re-timed
|
||||
# since keeps its fresh expiry unwarned.
|
||||
warn_before = now + timedelta(days=grace_days)
|
||||
soon = (
|
||||
db.execute(
|
||||
select(DownloadJob).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at > now,
|
||||
DownloadJob.expires_at <= warn_before,
|
||||
)
|
||||
soon = db.execute(
|
||||
select(DownloadJob.id, DownloadJob.expires_at).where(
|
||||
DownloadJob.status == JobStatus.DONE,
|
||||
DownloadJob.kept.is_(False),
|
||||
DownloadJob.expiry_warned.is_(False),
|
||||
DownloadJob.expires_at.is_not(None),
|
||||
DownloadJob.expires_at > now,
|
||||
DownloadJob.expires_at <= warn_before,
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in soon:
|
||||
_notify(
|
||||
db, job, db.get(MediaAsset, job.asset_id) if job.asset_id else None, "download_expiring"
|
||||
)
|
||||
job.expiry_warned = True
|
||||
warned += 1
|
||||
if soon:
|
||||
).all()
|
||||
db.rollback() # the pass-1 snapshot holds no locks; each mark takes its own below
|
||||
for job_id, expires_at in soon:
|
||||
job = _mark_warned(db, job_id, expires_at)
|
||||
if job is not None:
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
_notify(db, job, asset, "download_expiring")
|
||||
warned += 1
|
||||
db.commit()
|
||||
|
||||
# 2. Expiry — a file past its folder's retention leaves the library (warned in pass 1).
|
||||
@@ -224,19 +276,14 @@ def run_download_gc(db: Session) -> dict:
|
||||
# fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
|
||||
cap = sysconfig.get_int(db, "download_total_max_bytes")
|
||||
if cap > 0:
|
||||
total = (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
total = _on_disk(db)
|
||||
if total > cap:
|
||||
kept_assets = select(DownloadJob.asset_id).where(
|
||||
DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
|
||||
)
|
||||
candidates = (
|
||||
candidate_ids = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
select(MediaAsset.id)
|
||||
.where(
|
||||
MediaAsset.status == "ready",
|
||||
MediaAsset.source_kind != "upload",
|
||||
@@ -247,30 +294,20 @@ def run_download_gc(db: Session) -> dict:
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
candidate_ids = [a.id for a in candidates]
|
||||
db.rollback()
|
||||
for asset_id in candidate_ids:
|
||||
if total <= cap:
|
||||
break
|
||||
# Locked and re-read like an expiry: a holder may have kept it, or every holder
|
||||
# deleted it (the file is then already gone and counted out) since the SELECT.
|
||||
asset = db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
holders = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.asset_id == asset_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if asset is None or asset.status != "ready" or any(j.kept for j in holders):
|
||||
# deleted it since the SELECT.
|
||||
asset, holders = _relock_evictable(db, asset_id)
|
||||
if asset is None or asset.status != "ready":
|
||||
# Its file went (or is being fetched afresh) since the sum: the disk is counted
|
||||
# again, or the pass would evict another user's file for space already free.
|
||||
db.commit()
|
||||
total = _on_disk(db)
|
||||
continue
|
||||
if any(j.kept for j in holders):
|
||||
db.commit()
|
||||
continue
|
||||
total -= asset.size_bytes or 0
|
||||
|
||||
Reference in New Issue
Block a user