fix(downloads): a conversion's commit, a stopped upload, the fast join and a clip's copy hold up

- The fast join gives each ffconcat entry its own media-only format whitelist, so a library file
  can no longer be opened as a nested concat list (the list's whitelist was handed down to it).
- A file the guarded probe cannot read is a download-only file, not a guess from yt-dlp's info.
- A conversion's or an ingest's commit that raises is re-read in a fresh session before any file
  is removed: a commit that landed keeps the file its row now names.
- An original that cannot be removed after conversion is logged; the upload stays ready.
- An upload whose conversion is cancelled or deleted midway is freed once nothing holds it; a
  paused one is kept.
- A shared link says "preparing" only while its video's job is queued or running.
- The admin storage view counts every file on disk ("Files on disk"), not only ready ones.
- A clip's failure title, unknown failure and cancel dialog, and a clip's and an upload's delete
  dialog speak of what the job made; an upload's quota refusal explains that the original is kept.
- /watch announces "preparing" as an alert and moves focus to the new Retry after an action.
This commit is contained in:
2026-10-06 21:02:07 +02:00
parent d44d8d6dda
commit ed6facb703
19 changed files with 661 additions and 88 deletions
+5 -2
View File
@@ -338,13 +338,16 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
]
else:
listp = staging / "concat.txt"
lines = []
lines = ["ffconcat version 1.0"]
for s in segs:
lines.append(f"file '{_concat_quote(src)}'")
# ffmpeg hands the list's whitelist (which must name `concat` to read the list) down to
# the file it names, so a library file whose bytes are a concat list would be read as one
# and open the paths inside it. Each entry's own whitelist replaces the inherited one.
lines.append(f"option format_whitelist {INPUT_GUARD[1]}")
lines.append(f"inpoint {s['start_s']}")
lines.append(f"outpoint {s['end_s']}")
prep[listp] = "\n".join(lines) + "\n"
# The list is ours; the whitelist reaches the file it names too (ffmpeg hands it down).
guard = ["-format_whitelist", "concat," + INPUT_GUARD[1]]
cmd = head + ["-f", "concat", "-safe", "0", *guard, "-i", str(listp), "-c", "copy"]
+66 -6
View File
@@ -914,14 +914,59 @@ def _unplace(placed: Path | None) -> None:
placed.unlink(missing_ok=True)
def _conversion_landed(asset_id: int, new_rel: str) -> bool | None:
"""After the commit that makes a converted upload ready RAISED: did it land anyway? A commit can
reach the server and lose its answer (the connection drops after COMMIT). True/False by what the
row now says; None when the row cannot be read either."""
try:
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
return asset is not None and asset.status == "ready" and asset.rel_path == new_rel
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
# silent-ok: the caller re-raises the commit's own error, which `_fail_edit` reports.
return None
def _drop_original(job_id: int, src: Path) -> None:
"""Remove an upload's original once the converted file is committed in its place. Best effort:
the job is done and its file is ready, so a failure here must not fail it — the original is
named by no row now, and the GC's `.uploads` sweep takes it."""
try:
src.unlink(missing_ok=True)
except OSError as exc:
log.warning(
"upload job %s: original not removed, left to the GC sweep: %s",
job_id,
exc,
extra=logs.event("upload.original.unlink_failed", job_id=job_id),
)
def _free_unheld_upload(asset_id: int) -> bool:
"""Free an upload no job holds any more — cancelled or deleted while its conversion ran, when
`release_asset` had to leave it because the conversion still read its file. Once that has
stopped, free it as the release would have; left alone, its file would sit on disk counted by no
owner's footprint until the GC's upload pass. False when a job still holds it (paused)."""
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
if asset is None or asset.ref_count != 0:
return False
if asset.rel_path:
storage.delete_asset_files(settings.download_root, asset.rel_path)
db.delete(asset)
db.commit()
return True
def _convert_upload(job_id: int, asset_id: int) -> None:
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
under the new extension, and only then is the original removed — so a failed or stopped
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
A converted file placed but never committed is removed again on the way out; a stop in the
instant between the commit and the original's unlink is left to the GC's `.uploads` sweep."""
A converted file placed but never committed is removed again on the way out. When the commit's
own outcome is unknown neither file is removed: the row names one of them, and the GC's `.uploads`
sweep takes the one it does not — as it does an original whose unlink after the commit failed."""
with SessionLocal() as db:
asset = db.get(MediaAsset, asset_id)
old_rel = asset.rel_path if asset else None
@@ -965,12 +1010,25 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
finalized = service.finalize_if_running(
db, job_id, status=JobStatus.DONE, progress=100, phase=None
)
db.commit()
try:
db.commit()
except Exception:
landed = _conversion_landed(asset_id, new_rel)
if landed is False:
raise # rolled back: the original is still the asset's file — fail as any step
# Landed, or cannot tell: the converted file may be the asset's file now, so it is
# never removed blind (the sweep takes whichever file no row names).
placed = None
if landed is None:
raise
db.rollback()
placed = None
if finalized:
_notify_download_done(db, owner_id, title, job_id, kind="upload")
if new_rel != old_rel:
src.unlink(missing_ok=True)
_drop_original(job_id, src)
if not finalized:
_free_unheld_upload(asset_id) # finished after its job was cancelled or deleted
if finalized:
log.info(
"upload job %s converted: asset %s",
@@ -979,9 +1037,11 @@ def _convert_upload(job_id: int, asset_id: int) -> None:
extra=logs.event("upload.job.done", job_id=job_id, asset_id=asset_id, ext=ext),
)
except editmod.EditAborted:
# Paused/canceled mid-encode: the original is untouched; free the asset for a later retry.
# Paused/canceled mid-encode: the original is untouched. A paused job keeps it for its
# resume; one cancelled or deleted holds it no more, so it is freed now.
_unplace(placed)
_reset_asset_pending(asset_id)
if not _free_unheld_upload(asset_id):
_reset_asset_pending(asset_id)
log.info(
"upload job %s conversion aborted (pause/cancel)",
job_id,
+7 -7
View File
@@ -121,18 +121,18 @@ def resolve_media_facts(
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
dimensions and the duration still gap-fill from `info` in either case."""
mislabelling an audio download as a broken in-app 'video'. When ffprobe could NOT read the file
at all, no codec is reported: the probe runs with the media demuxer whitelist, so a file it
cannot open is one the player and the editor cannot open either — and taking `info`'s codecs
for it would make it an editable 'video' whose bytes are whatever the remote site served. The
cosmetic dimensions and the duration still gap-fill from `info` in either case."""
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
info_w = int(info["width"]) if info.get("width") else None
info_h = int(info["height"]) if info.get("height") else None
info_d = int(info["duration"]) if info.get("duration") else None
if data is None:
# ffprobe could not read the file — fall back to the extractor's info entirely.
return info_v, info_a, info_w, info_h, info_d
# Unreadable to the guarded probe: a download-only 'file' (no codecs → media_kind FILE).
return None, None, info_w, info_h, info_d
pv, pa, pw, ph, pd = _facts_from_probe(data)
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
# gap-fills the width/height/duration the probe couldn't read.
+19 -3
View File
@@ -83,9 +83,10 @@ def ingest_upload(
rel = storage.upload_rel_path(user_id, asset.id, filename)
storage.place_file(src_path, settings.download_root, rel)
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
asset_id = asset.id
try:
final = storage.abs_path(settings.download_root, rel)
actual_size = final.stat().st_size if final.exists() else size
@@ -126,6 +127,11 @@ def ingest_upload(
db.commit()
except Exception:
db.rollback()
if _row_landed(db, asset_id) is not False:
# The commit reached the server and lost its answer, or the row cannot be read: the file
# may be the asset's now, so it is never removed blind — the sweep takes it if no row
# names it.
raise
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
@@ -137,3 +143,13 @@ def ingest_upload(
raise
db.refresh(job)
return job
def _row_landed(db: Session, asset_id: int) -> bool | None:
"""After the ingest transaction raised: is the asset row there anyway? A commit can reach the
server and lose its answer (the connection drops after COMMIT). None when it cannot be read."""
try:
return db.get(MediaAsset, asset_id) is not None
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
# silent-ok: the caller re-raises the ingest's own error.
return None
+5 -3
View File
@@ -1144,7 +1144,9 @@ def admin_list_downloads(
@admin_router.get("/storage")
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
ready = db.execute(
# Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their
# conversion — the same disk the footprint and the GC cap count.
on_disk = db.execute(
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
).one()
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
@@ -1158,8 +1160,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d
for u in db.execute(select(User).where(User.id.in_([uid for uid, _ in per_user]))).scalars()
}
return {
"ready_files": ready[0],
"total_bytes": int(ready[1]),
"files_on_disk": on_disk[0],
"total_bytes": int(on_disk[1]),
"total_assets": total_assets,
"total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"),
"per_user": [
+9 -1
View File
@@ -16,6 +16,7 @@ from app.config import settings
from app.db import get_db
from app.downloads import links as linksmod
from app.downloads import storage
from app.downloads.status import JobStatus
from app.models import DownloadJob, DownloadLink, MediaAsset
from app.ratelimit import RateLimiter, client_key
from app.security import verify_password
@@ -46,9 +47,16 @@ def _link_or_404(db: Session, token: str) -> DownloadLink:
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
job = db.get(DownloadJob, link.job_id)
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
if asset is not None and asset.status in ("pending", "downloading"):
if (
job is not None
and asset is not None
and asset.status in ("pending", "downloading")
and job.status in (JobStatus.QUEUED, JobStatus.RUNNING)
):
# Being (re)made — a re-download, or a conversion — so the link works again once it is ready:
# its own reason, so the page can say "try again shortly" rather than "this link is gone".
# Only while the job is still going: a cancelled or paused rebuild leaves the asset pending
# with nothing working on it, and "try again in a few minutes" would never come true.
raise refused(404, "video_preparing")
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
raise refused(404, "video_gone")