fix(downloads): an upload's conversion is guarded, counted, named and cleaned up
- Every ffmpeg/ffprobe over a library file names the demuxers it may use
(edit.INPUT_GUARD; a downloaded thumbnail edit.THUMB_GUARD): a playlist-type
demuxer would read local paths named inside a user's upload.
- A re-encode ends on an even frame (edit.EVEN_FRAME): an odd 4:4:4/RGB source
failed its conversion; a clip stores the frame its file has.
- An upload counts against its owner's cap from the moment it lands (queued, or
kept after a failed conversion) — the footprint, the GC's cap and the admin
view read one ON_DISK predicate — and a conversion that outgrows the cap is
refused, keeping the original.
- The converted name keeps _a{id} (a dotted bare name once lost it and two
uploads shared a file); a converted file placed but not committed is removed,
and the orphan sweep enters .uploads; the LRU cap never evicts an upload.
- Ingest judges the extension the file is stored under; a failed conversion is
convert_failed / upload_missing, reports the "converting" phase, and notifies
as an upload. Job statuses in routes/downloads use JobStatus.
This commit is contained in:
@@ -194,6 +194,44 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
||||
# only patchily (the one safe H.264 is 8-bit 4:2:0).
|
||||
X264_ARGS = ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p"]
|
||||
AAC_ARGS = ["-c:a", "aac", "-b:a", "192k"]
|
||||
# The last step of every video filter chain a libx264 re-encode runs: 4:2:0 halves the chroma, so
|
||||
# libx264 refuses an odd width or height ("width not divisible by 2") — measured on an odd source
|
||||
# that is not 4:2:0 already (a 4:4:4 or RGB screen capture, 1365x767). A crop of a 4:2:0 source is
|
||||
# rounded to its chroma grid by ffmpeg itself; this holds every other path to the encoder to the same
|
||||
# even frame. Rounding down to even costs at most one pixel row/column.
|
||||
EVEN_FRAME = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
||||
|
||||
# The demuxers ffmpeg/ffprobe may use to open a library file — single-file containers only. Without
|
||||
# it ffmpeg picks the demuxer from the file's CONTENT and NAME, and an upload's name is the user's:
|
||||
# a playlist-type demuxer (HLS, DASH, concat) opens whatever local paths its entries name, so a
|
||||
# crafted upload could copy another user's clip or a Plex file into its own converted asset. A file
|
||||
# no demuxer here can open fails the probe, and an unprobeable upload is a download-only 'file'.
|
||||
MEDIA_DEMUXERS = (
|
||||
"mov", # mp4, m4a, mov, 3gp
|
||||
"matroska", # mkv, webm
|
||||
"mpegts",
|
||||
"mpeg",
|
||||
"avi",
|
||||
"asf",
|
||||
"flv",
|
||||
"ogg",
|
||||
"mp3",
|
||||
"aac",
|
||||
"ac3",
|
||||
"eac3",
|
||||
"flac",
|
||||
"wav",
|
||||
"w64",
|
||||
"aiff",
|
||||
"caf",
|
||||
"amr",
|
||||
)
|
||||
INPUT_GUARD = ["-format_whitelist", ",".join(MEDIA_DEMUXERS)]
|
||||
# A downloaded thumbnail is the remote site's bytes under the suffix yt-dlp gave it (`.png`, `.webp`,
|
||||
# `.jpg` — `pipeline._THUMB_SUFFIXES`): only the single-image demuxers those open, so a "thumbnail"
|
||||
# that is really a playlist cannot pull a local file into the poster. Anything else fails, and the
|
||||
# poster falls back to a cut frame (`ensure_poster`).
|
||||
THUMB_GUARD = ["-format_whitelist", "png_pipe,jpeg_pipe,webp_pipe"]
|
||||
|
||||
|
||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||
@@ -221,7 +259,7 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
cmd += ["-i", str(src)]
|
||||
cmd += [*INPUT_GUARD, "-i", str(src)]
|
||||
if end is not None:
|
||||
dur = end - start
|
||||
if dur > 0:
|
||||
@@ -229,10 +267,9 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
|
||||
if reencode:
|
||||
# Input -ss + re-encode is frame-accurate (decodes from the prior keyframe, discards up to
|
||||
# the exact cut). A crop adds the filter; an accurate trim re-encodes with no filter.
|
||||
if crop:
|
||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||
cmd += [*X264_ARGS, *AAC_ARGS]
|
||||
# the exact cut). A crop adds the filter; an accurate trim only evens the frame.
|
||||
cropf = f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}," if crop else ""
|
||||
cmd += ["-vf", cropf + EVEN_FRAME, *X264_ARGS, *AAC_ARGS]
|
||||
else:
|
||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||
|
||||
@@ -280,11 +317,14 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
parts, labels = [], []
|
||||
for i, s in enumerate(segs):
|
||||
st, en = s["start_s"], s["end_s"]
|
||||
parts.append(f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf}[v{i}]")
|
||||
parts.append(
|
||||
f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf},{EVEN_FRAME}[v{i}]"
|
||||
)
|
||||
parts.append(f"[0:a]atrim=start={st}:end={en},asetpts=PTS-STARTPTS[a{i}]")
|
||||
labels.append(f"[v{i}][a{i}]")
|
||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||
cmd = head + [
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-filter_complex",
|
||||
@@ -304,7 +344,9 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
lines.append(f"inpoint {s['start_s']}")
|
||||
lines.append(f"outpoint {s['end_s']}")
|
||||
prep[listp] = "\n".join(lines) + "\n"
|
||||
cmd = head + ["-f", "concat", "-safe", "0", "-i", str(listp), "-c", "copy"]
|
||||
# The list is ours; the whitelist reaches the file it names too (ffmpeg hands it down).
|
||||
guard = ["-format_whitelist", "concat," + INPUT_GUARD[1]]
|
||||
cmd = head + ["-f", "concat", "-safe", "0", *guard, "-i", str(listp), "-c", "copy"]
|
||||
|
||||
if out_ext in ("mp4", "m4a", "mov"):
|
||||
cmd += ["-movflags", "+faststart"]
|
||||
@@ -477,7 +519,7 @@ def run_ffprobe_json(
|
||||
package: the shared media probe (`probe.probe_json`: the stored facts and the phone-safe rule) goes
|
||||
through it, so the
|
||||
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
||||
args = ["ffprobe", "-v", "error"]
|
||||
args = ["ffprobe", "-v", "error", *INPUT_GUARD]
|
||||
if show_streams:
|
||||
args.append("-show_streams")
|
||||
if show_format:
|
||||
@@ -507,6 +549,7 @@ def probe_duration(path: Path) -> int:
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
*INPUT_GUARD,
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
@@ -564,6 +607,7 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -587,6 +631,7 @@ def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"-nostdin",
|
||||
"-ss",
|
||||
f"{max(0.0, at_s):.3f}",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -608,6 +653,7 @@ def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*THUMB_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
|
||||
Reference in New Issue
Block a user