fix(files): review round 1, the server — the tray's window, the GC's locks, a file's origin, the upgrade path

- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming,
  keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished.
- GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit
  one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a
  second release that frees a file another user still holds.
- A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit
  whose parent expired still goes back to its folder, not to Uploads.
- A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes
  that file home too, instead of failing on the foreign key.
- The hash route reads at most two files at once (429 past that) and holds no connection meanwhile.
- 0104's backfill dates old files by their request date only: no notice prod wrote names its job.
- The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the
  worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up.
- Stale comments about the /downloads mount and the Downloads page.
This commit is contained in:
2026-10-13 00:14:09 +02:00
parent 96509fe356
commit 815083bcc4
29 changed files with 593 additions and 92 deletions
+29 -11
View File
@@ -13,6 +13,7 @@ import base64
import hashlib
import os
import tempfile
import threading
from collections.abc import AsyncIterator, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -23,7 +24,7 @@ from urllib.parse import unquote, urlparse
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import FileResponse, Response
from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select
from sqlalchemy import func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -59,6 +60,9 @@ MAX_LIST = 500
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
_MAX_RETENTION_DAYS = 3650
_HASH_CHUNK = 1 << 20
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
_HASHERS = 2
_HASHING = threading.BoundedSemaphore(_HASHERS)
# --- shared with routes.downloads -------------------------------------------------------
@@ -235,7 +239,7 @@ def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
if not raw.strip():
return None
try:
return folders.upload_target(db, user_id, int(raw)).id
return folders.upload_target(db, user_id, int(raw), hold=False).id
except ValueError as e:
raise refused(409, "file_folder_gone") from e
except folders.FolderError as e:
@@ -664,12 +668,9 @@ def update_file(
if "folder_id" in sent:
if body.folder_id is None:
raise refused(400, "file_folder_gone")
folder = db.get(FileFolder, body.folder_id)
if folder is None or folder.user_id != user.id:
raise refused(409, "file_folder_gone")
before = job.folder_id
try:
folders.move(db, job, folder)
folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
except folders.FolderError as e:
raise refused(409, e.reason) from e
if job.folder_id != before:
@@ -842,13 +843,30 @@ def get_sha256(
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
) -> dict:
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool."""
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
past that is told to come back), and none of them holds a database connection while it reads."""
asset, path = _ready_file(db, accessible_job(db, user, job_id))
if not asset.sha256:
if asset.sha256:
return {"sha256": asset.sha256}
asset_id = asset.id
if not _HASHING.acquire(blocking=False):
raise refused(429, "file_hash_busy")
try:
release_read_connection(db)
h = hashlib.sha256()
with path.open("rb") as f:
while chunk := f.read(_HASH_CHUNK):
h.update(chunk)
asset.sha256 = h.hexdigest()
db.commit()
return {"sha256": asset.sha256}
except OSError as e:
raise not_found("file") from e
finally:
_HASHING.release()
digest = h.hexdigest()
db.execute(
update(MediaAsset)
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
.values(sha256=digest)
)
db.commit()
return {"sha256": digest}