fix(files): review round 3, the server — a fetched locked re-read, a finish that follows its file, an eviction that counts again

- set_retention and delete_own re-read the folder under its lock and use that row (an unfetched ORM
  result never refreshes the object); a folder gone meanwhile is refused (409), and the route audits
  the retention actually replaced
- a finishing download is stamped only while it is still in the folder (and keep) it was read with:
  a move or a folder delete landing meanwhile sends the read round again, so no file is done with no
  expiry
- the eviction pass counts the disk before each candidate and skips one asked for since it began
- docs: source builds get their own volume-copy snippet
- race tests assert the lock wait; one built on a committing delete never raced

backend/app/downloads/folders.py

@@ -211,21 +211,30 @@ def move(db: Session, job: DownloadJob, folder: FileFolder) -> None:
     restamp(db, job)
 
 
-def set_retention(db: Session, folder: FileFolder, days: int | None) -> None:
-    """Change how long `folder` keeps a file (None = until deleted). The done files already in it
-    move by the difference — each keeps the age it has — but none ends up closer than the GC's
-    warning window, so a shortened retention still warns before it takes anything."""
-    # Wait out a file being put in it or stamped from it right now (`own_folder`, `done_expiry`), so
-    # that file is among the ones restamped below. Re-read: another change may have landed first.
-    db.execute(
+def _hold_for_change(db: Session, folder: FileFolder) -> None:
+    """Lock `folder` FOR UPDATE and read it afresh into the same object: a change waits out a file
+    being put in it or stamped from it right now (`own_folder`, `done_expiry`), and works from what
+    another change that landed first left, not from what the caller read before the wait. The row is
+    consumed on purpose: an ORM result refreshes the object only as its rows are fetched."""
+    fresh = db.execute(
         select(FileFolder)
         .where(FileFolder.id == folder.id)
         .with_for_update()
         .execution_options(populate_existing=True)
-    )
+    ).scalar_one_or_none()
+    if fresh is None:
+        raise FolderError("file_folder_gone")
+
+
+def set_retention(db: Session, folder: FileFolder, days: int | None) -> int | None:
+    """Change how long `folder` keeps a file (None = until deleted). The done files already in it
+    move by the difference — each keeps the age it has — but none ends up closer than the GC's
+    warning window, so a shortened retention still warns before it takes anything. Returns the
+    retention it replaced (equal to `days` when nothing changed)."""
+    _hold_for_change(db, folder)  # so the file arriving now is among the ones restamped below
     old = folder.retention_days
     if old == days:
-        return
+        return old
     folder.retention_days = days
     now = datetime.now(timezone.utc)
     floor = now + timedelta(days=sysconfig.get_int(db, "download_gc_grace_days"))
@@ -248,6 +257,7 @@ def set_retention(db: Session, folder: FileFolder, days: int | None) -> None:
         else:
             job.expires_at = max(job.expires_at + timedelta(days=days - old), floor)
         job.expiry_warned = False
+    return old
 
 
 def delete_own(db: Session, folder: FileFolder) -> int:
@@ -255,8 +265,7 @@ def delete_own(db: Session, folder: FileFolder) -> int:
     folder it came from, its clock starting there now. Returns how many moved."""
     if folder.kind != FolderKind.OWN:
         raise FolderError("file_folder_not_own")
-    # Wait out a file being put in it right now (`own_folder`), then take that one home too.
-    db.execute(select(FileFolder.id).where(FileFolder.id == folder.id).with_for_update())
+    _hold_for_change(db, folder)  # so the file arriving now is taken home too
     # Locked as they are read: a file moved out meanwhile stays where it went, and one deleted
     # meanwhile is not written back (its gone row would fail the delete).
     jobs = (

backend/app/downloads/gc.py

@@ -276,8 +276,7 @@ def run_download_gc(db: Session) -> dict:
     #    fetch again) and never a file anyone kept; an evicted file leaves every holder's library.
     cap = sysconfig.get_int(db, "download_total_max_bytes")
     if cap > 0:
-        total = _on_disk(db)
-        if total > cap:
+        if _on_disk(db) > cap:
             kept_assets = select(DownloadJob.asset_id).where(
                 DownloadJob.kept.is_(True), DownloadJob.asset_id.is_not(None)
             )
@@ -296,21 +295,21 @@ def run_download_gc(db: Session) -> dict:
             )
             db.rollback()
             for asset_id in candidate_ids:
-                if total <= cap:
+                # Counted afresh before each: a file deleted since (this one, an upload, a candidate
+                # further down) has freed its space, and no other user's file goes for that space.
+                if _on_disk(db) <= cap:
                     break
-                # Locked and re-read like an expiry: a holder may have kept it, or every holder
-                # deleted it since the SELECT.
+                # Locked and re-read like an expiry: since the SELECT a holder may have kept it,
+                # every holder deleted it, or someone asked for it again (a fresh cache hit).
                 asset, holders = _relock_evictable(db, asset_id)
-                if asset is None or asset.status != "ready":
-                    # Its file went (or is being fetched afresh) since the sum: the disk is counted
-                    # again, or the pass would evict another user's file for space already free.
-                    db.commit()
-                    total = _on_disk(db)
-                    continue
-                if any(j.kept for j in holders):
+                if (
+                    asset is None
+                    or asset.status != "ready"
+                    or any(j.kept for j in holders)
+                    or (asset.last_access_at is not None and asset.last_access_at > now)
+                ):
                     db.commit()
                     continue
-                total -= asset.size_bytes or 0
                 for job in holders:
                     _notify(db, job, asset, "download_evicted")
                     db.delete(job)

backend/app/downloads/service.py

@@ -421,17 +421,7 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
         return False
     db.flush()  # asset-then-job lock order: push pending asset writes before we lock the job row
     if fields.get("status") == JobStatus.DONE:
-        fields = {**fields, "ready_at": datetime.now(timezone.utc)}
-        # Done is when a file's retention starts: its folder's, unless the user kept it.
-        held = db.execute(
-            select(DownloadJob.folder_id, DownloadJob.kept).where(DownloadJob.id == job_id)
-        ).one_or_none()
-        if held is not None:
-            fields = {
-                **fields,
-                "expires_at": folders.done_expiry(db, held.folder_id, held.kept),
-                "expiry_warned": False,
-            }
+        return _finalize_done(db, job_id, {**fields, "ready_at": datetime.now(timezone.utc)})
     # A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
     # rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
     # is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
@@ -445,6 +435,41 @@ def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
     return result.rowcount > 0
 
 
+def _finalize_done(db: Session, job_id: int, fields: dict) -> bool:
+    """`finalize_if_running` for a job that finished: done is when a file's retention starts, its
+    folder's unless the user kept it. The expiry is written only while the job is still where it was
+    read from — a move, a keep, or its folder's delete (which sends its files home) landing while the
+    folder's retention was read sends the read round again, never stamping a folder's retention on a
+    file now in another one."""
+    while True:
+        held = db.execute(
+            select(DownloadJob.folder_id, DownloadJob.kept, DownloadJob.status).where(
+                DownloadJob.id == job_id
+            )
+        ).one_or_none()
+        if held is None or held.status != JobStatus.RUNNING:
+            return False
+        stamped = {
+            **fields,
+            "expires_at": folders.done_expiry(db, held.folder_id, held.kept),
+            "expiry_warned": False,
+        }
+        hit = db.execute(
+            update(DownloadJob)
+            .where(
+                DownloadJob.id == job_id,
+                DownloadJob.status == JobStatus.RUNNING.value,
+                DownloadJob.folder_id.is_not_distinct_from(held.folder_id),
+                DownloadJob.kept == held.kept,
+            )
+            .values(**stamped)
+            .returning(DownloadJob.id)
+            .execution_options(synchronize_session=False)
+        ).scalar_one_or_none()
+        if hit is not None:
+            return True
+
+
 def release_asset(db: Session, job: DownloadJob) -> None:
     """Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
     asset anymore, delete the file + row immediately — a deleted download should free its disk,

backend/app/routes/files.py

@@ -421,18 +421,22 @@ def update_folder(
             target_type="file_folder",
             target_id=folder.id,
         )
-    if "retention_days" in sent and body.retention_days != folder.retention_days:
-        before = folder.retention_days
-        folders.set_retention(db, folder, body.retention_days)
-        audit.record(
-            db,
-            user.id,
-            AuditAction.FILE_FOLDER_RETENTION,
-            target_type="file_folder",
-            target_id=folder.id,
-            before={"retention_days": before},
-            after={"retention_days": body.retention_days},
-        )
+    if "retention_days" in sent:
+        # The value replaced is the one read under the folder's lock, not the one read above it.
+        try:
+            before = folders.set_retention(db, folder, body.retention_days)
+        except folders.FolderError as e:
+            raise refused(409, e.reason) from e
+        if before != body.retention_days:
+            audit.record(
+                db,
+                user.id,
+                AuditAction.FILE_FOLDER_RETENTION,
+                target_type="file_folder",
+                target_id=folder.id,
+                before={"retention_days": before},
+                after={"retention_days": body.retention_days},
+            )
     db.commit()
     count, size = _folder_totals(db, folder.id)
     return _serialize_folder(folder, count, size)