fix(files): review round 3, the server — a fetched locked re-read, a finish that follows its file, an eviction that counts again

- set_retention and delete_own re-read the folder under its lock and use that row (an unfetched ORM
  result never refreshes the object); a folder gone meanwhile is refused (409), and the route audits
  the retention actually replaced
- a finishing download is stamped only while it is still in the folder (and keep) it was read with:
  a move or a folder delete landing meanwhile sends the read round again, so no file is done with no
  expiry
- the eviction pass counts the disk before each candidate and skips one asked for since it began
- docs: source builds get their own volume-copy snippet
- race tests assert the lock wait; one built on a committing delete never raced

backend/tests/test_file_folders.py

@@ -501,10 +501,11 @@ def test_a_retention_changed_while_a_file_goes_in_times_that_file_too(db, make_u
         lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
     )
     t.join(timeout=1)
+    assert t.is_alive()  # the change waits for the file being put in
     folders.move(db, job, target)  # stamped with the 7 days it sees
     db.commit()
     t.join(timeout=10)
-    assert outcome == [None]
+    assert outcome == [7]  # the retention it replaced
     db.expire_all()
     assert db.get(DownloadJob, job.id).expires_at is None
 
@@ -527,13 +528,21 @@ def test_a_retention_changed_while_a_file_finishes_there_times_that_file_too(db,
         lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
     )
     t.join(timeout=1)
+    assert t.is_alive()  # the change waits for the file being stamped
     db.commit()
     t.join(timeout=10)
-    assert outcome == [None]
+    assert outcome == [7]  # the retention it replaced
     db.expire_all()
     assert db.get(DownloadJob, job.id).expires_at is None
 
 
+def _deleted_uncommitted(db, job):
+    """`service.delete` up to its commit: the file is gone in this session, its row still locked."""
+    service.release_asset(db, job)
+    db.delete(job)
+    db.flush()
+
+
 def test_a_file_moved_out_while_its_folder_is_deleted_stays_where_it_went(db, make_user):
     uid = make_user().id
     f = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="F")
@@ -566,10 +575,10 @@ def test_a_file_deleted_while_its_folder_is_deleted_does_not_fail_the_delete(db,
     job = _job(db, uid, _asset(db))
     folders.move(db, job, f)
     db.commit()
-    service.delete(db, db.get(DownloadJob, job.id))
-    db.flush()  # deleted, not yet committed
+    _deleted_uncommitted(db, db.get(DownloadJob, job.id))
     t, outcome = _in_thread(lambda other: folders.delete_own(other, other.get(FileFolder, f_id)))
     t.join(timeout=1)
+    assert t.is_alive()  # the delete waits for the file being deleted
     db.commit()
     t.join(timeout=10)
     assert outcome == [0]
@@ -577,6 +586,158 @@ def test_a_file_deleted_while_its_folder_is_deleted_does_not_fail_the_delete(db,
     assert db.get(FileFolder, f_id) is None
 
 
+def _own_with_a_file(db, uid, retention=30):
+    """A user's own folder keeping files `retention` days, holding one done file due in as many."""
+    own = FileFolder(
+        user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=retention
+    )
+    db.add(own)
+    db.flush()
+    job = _job(db, uid, _asset(db))
+    job.folder_id = own.id
+    job.expires_at = NOW + timedelta(days=retention)
+    db.commit()
+    return own.id, job.id
+
+
+def _elsewhere(act):
+    other = SessionLocal()
+    try:
+        act(other)
+        other.commit()
+    finally:
+        other.close()
+
+
+def test_a_retention_change_works_from_what_a_change_before_it_left(db, make_user):
+    # The locked re-read was never fetched, so the change shifted the files from the value read before
+    # its wait: 30 -> 10 -> 60 left them due in 40 days under a folder saying 60.
+    own_id, job_id = _own_with_a_file(db, make_user().id)
+    stale = db.get(FileFolder, own_id)  # read before the other change lands
+    _elsewhere(lambda other: folders.set_retention(other, other.get(FileFolder, own_id), 10))
+    assert folders.set_retention(db, stale, 60) == 10
+    db.commit()
+    db.expire_all()
+    due = db.get(DownloadJob, job_id).expires_at - NOW
+    assert timedelta(days=59) < due < timedelta(days=61)
+
+
+def test_a_retention_change_to_a_folder_deleted_meanwhile_is_refused(db, make_user):
+    own_id, _ = _own_with_a_file(db, make_user().id)
+    stale = db.get(FileFolder, own_id)
+    _elsewhere(lambda other: folders.delete_own(other, other.get(FileFolder, own_id)))
+    with pytest.raises(folders.FolderError, match="file_folder_gone"):
+        folders.set_retention(db, stale, 60)
+
+
+def test_a_file_moved_out_while_its_folder_retention_changes_keeps_its_new_folders(db, make_user):
+    uid = make_user().id
+    own_id, job_id = _own_with_a_file(db, uid)
+    g = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="G", retention_days=7)
+    db.add(g)
+    db.commit()
+    g_id = g.id
+    folders.move(db, db.get(DownloadJob, job_id), folders.own_folder(db, uid, g_id))
+    db.flush()  # the move is written, not yet committed
+    t, outcome = _in_thread(
+        lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
+    )
+    t.join(timeout=1)
+    assert t.is_alive()  # the change waits for the file it would time
+    db.commit()
+    t.join(timeout=10)
+    assert outcome == [30]
+    db.expire_all()
+    job = db.get(DownloadJob, job_id)
+    assert job.folder_id == g_id
+    assert job.expires_at is not None  # G's 7 days, not the forever of the folder it left
+
+
+def test_a_file_deleted_while_its_folder_retention_changes_does_not_fail_the_change(
+    db, make_user, _root
+):
+    (_root / "a.mp4").write_bytes(b"x")
+    own_id, job_id = _own_with_a_file(db, make_user().id)
+    _deleted_uncommitted(db, db.get(DownloadJob, job_id))
+    t, outcome = _in_thread(
+        lambda other: folders.set_retention(other, other.get(FileFolder, own_id), None)
+    )
+    t.join(timeout=1)
+    assert t.is_alive()  # the change waits for the file it would time
+    db.commit()
+    t.join(timeout=10)
+    assert outcome == [30]
+
+
+def test_a_file_finishing_while_its_folder_is_deleted_gets_its_new_folders_retention(db, make_user):
+    # The finish read the folder unlocked, then waited out the delete and found no folder: the file
+    # was done with no expiry at all, in a source folder that keeps files 30 days.
+    uid = make_user().id
+    own = FileFolder(user_id=uid, kind=folders.FolderKind.OWN, name="Mine", retention_days=7)
+    db.add(own)
+    db.commit()
+    own_id = own.id
+    job = _job(db, uid, _asset(db), status=JobStatus.RUNNING)
+    job_id = job.id
+    folders.move(db, job, own)
+    db.commit()
+    folders.delete_own(db, db.get(FileFolder, own_id))
+    db.flush()  # the delete, not yet committed
+    t, outcome = _in_thread(
+        lambda other: service.finalize_if_running(other, job_id, status=JobStatus.DONE)
+    )
+    t.join(timeout=1)
+    assert t.is_alive()  # the finish waits for the delete
+    db.commit()
+    t.join(timeout=10)
+    assert outcome == [True]
+    db.expire_all()
+    done = db.get(DownloadJob, job_id)
+    assert done.folder_id != own_id
+    assert done.expires_at is not None
+
+
+def _an_upload_over_the_cap_too(db, uid, _root):
+    (_root / "u.bin").write_bytes(b"x")
+    return _job(db, uid, _asset(db, "upload", "u1", rel="u.bin", size=10), kind="upload")
+
+
+def test_space_freed_elsewhere_while_the_cache_is_over_its_cap_evicts_no_more(
+    db, make_user, _root, monkeypatch
+):
+    # The pass counted the disk once: an upload deleted meanwhile still counted, so a second user's
+    # file went for space already free.
+    jold, jnew = _two_over_the_cap(db, make_user, _root)
+    up = _an_upload_over_the_cap_too(db, make_user().id, _root)
+
+    def upload_deleted(other, asset_id):
+        if asset_id == jold.asset_id:
+            service.delete(other, other.get(DownloadJob, up.id))
+
+    _meanwhile(monkeypatch, upload_deleted, step="_relock_evictable")
+    assert _gc(db)["evicted"] == 1
+    db.expire_all()
+    assert db.get(DownloadJob, jnew.id) is not None
+    assert (_root / "new.mp4").exists()
+
+
+def test_a_file_asked_for_again_while_the_cache_is_over_its_cap_stays(
+    db, make_user, _root, monkeypatch
+):
+    # A cache hit between the pass's SELECT and its lock: the file the user just asked for went.
+    jold, jnew = _two_over_the_cap(db, make_user, _root)
+
+    def asked_again(other, asset_id):
+        if asset_id == jold.asset_id:
+            other.get(MediaAsset, asset_id).last_access_at = datetime.now(timezone.utc)
+
+    _meanwhile(monkeypatch, asked_again, step="_relock_evictable")
+    _gc(db)
+    db.expire_all()
+    assert db.get(DownloadJob, jold.id) is not None
+    assert (_root / "old.mp4").exists()
+
+
 def test_a_file_asked_for_while_its_cached_copy_is_freed_gets_a_fresh_one(db, make_user):
     # The enqueue read the cached copy unlocked: freed meanwhile, its job pointed at a row that was gone
     # (a foreign-key failure, a 500).