fix(files): review round 1, the server — the tray's window, the GC's locks, a file's origin, the upgrade path
- The queue keeps a done job by when it became done (ready_at), not by its last write: renaming, keeping or moving an old file, or the GC's expiry warning, no longer brings it back as just finished. - GC expiry and eviction re-read each file under the asset→job locks the user routes take, and commit one file at a time: a delete, a re-download or a keep made while the GC runs is honoured, never a second release that frees a file another user still holds. - A file records the origin it was placed under (download_jobs.source_key, migration 0105): an edit whose parent expired still goes back to its folder, not to Uploads. - A folder a file is being put in is held (FOR SHARE) until it lands; a delete of it waits and takes that file home too, instead of failing on the foreign key. - The hash route reads at most two files at once (429 past that) and holds no connection meanwhile. - 0104's backfill dates old files by their request date only: no notice prod wrote names its job. - The old DOWNLOAD_ROOT still finds the files; a files root that is not a mounted volume stops the worker and is logged by the API; the volume-copy recipe looks the project-prefixed names up. - Stale comments about the /downloads mount and the Downloads page.
This commit is contained in:
+29
-11
@@ -13,6 +13,7 @@ import base64
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
@@ -23,7 +24,7 @@ from urllib.parse import unquote, urlparse
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -59,6 +60,9 @@ MAX_LIST = 500
|
||||
# The longest a folder may keep a file, in days — ten years is "forever" in all but name.
|
||||
_MAX_RETENTION_DAYS = 3650
|
||||
_HASH_CHUNK = 1 << 20
|
||||
# How many files the hash route reads at once: each holds a threadpool thread for the whole read.
|
||||
_HASHERS = 2
|
||||
_HASHING = threading.BoundedSemaphore(_HASHERS)
|
||||
|
||||
|
||||
# --- shared with routes.downloads -------------------------------------------------------
|
||||
@@ -235,7 +239,7 @@ def _upload_folder(db: Session, user_id: int, raw: str) -> int | None:
|
||||
if not raw.strip():
|
||||
return None
|
||||
try:
|
||||
return folders.upload_target(db, user_id, int(raw)).id
|
||||
return folders.upload_target(db, user_id, int(raw), hold=False).id
|
||||
except ValueError as e:
|
||||
raise refused(409, "file_folder_gone") from e
|
||||
except folders.FolderError as e:
|
||||
@@ -664,12 +668,9 @@ def update_file(
|
||||
if "folder_id" in sent:
|
||||
if body.folder_id is None:
|
||||
raise refused(400, "file_folder_gone")
|
||||
folder = db.get(FileFolder, body.folder_id)
|
||||
if folder is None or folder.user_id != user.id:
|
||||
raise refused(409, "file_folder_gone")
|
||||
before = job.folder_id
|
||||
try:
|
||||
folders.move(db, job, folder)
|
||||
folders.move(db, job, folders.own_folder(db, user.id, body.folder_id))
|
||||
except folders.FolderError as e:
|
||||
raise refused(409, e.reason) from e
|
||||
if job.folder_id != before:
|
||||
@@ -842,13 +843,30 @@ def get_sha256(
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The file's SHA-256 (hex), hashed now if it never was and kept on the asset for every holder.
|
||||
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool."""
|
||||
A sync handler: hashing a large file is blocking disk work, so it runs in the threadpool — and
|
||||
every other sync route shares that pool, so only `_HASHERS` files are hashed at a time (a caller
|
||||
past that is told to come back), and none of them holds a database connection while it reads."""
|
||||
asset, path = _ready_file(db, accessible_job(db, user, job_id))
|
||||
if not asset.sha256:
|
||||
if asset.sha256:
|
||||
return {"sha256": asset.sha256}
|
||||
asset_id = asset.id
|
||||
if not _HASHING.acquire(blocking=False):
|
||||
raise refused(429, "file_hash_busy")
|
||||
try:
|
||||
release_read_connection(db)
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
while chunk := f.read(_HASH_CHUNK):
|
||||
h.update(chunk)
|
||||
asset.sha256 = h.hexdigest()
|
||||
db.commit()
|
||||
return {"sha256": asset.sha256}
|
||||
except OSError as e:
|
||||
raise not_found("file") from e
|
||||
finally:
|
||||
_HASHING.release()
|
||||
digest = h.hexdigest()
|
||||
db.execute(
|
||||
update(MediaAsset)
|
||||
.where(MediaAsset.id == asset_id, MediaAsset.sha256.is_(None))
|
||||
.values(sha256=digest)
|
||||
)
|
||||
db.commit()
|
||||
return {"sha256": digest}
|
||||
|
||||
Reference in New Issue
Block a user