feat(clip): crop a clip to a frame drawn on the picture, and give the clip panel the room it needs
An optional crop in clip mode: a Crop toggle puts a frame on the picture (free, 16:9, 9:16 or 1:1; the whole picture stays the default). The frame is sent in fractions of the box it was drawn in, with that box's shape; the worker fits the real picture into the box by its display aspect (sample aspect and rotation aware), drops the part over the bars, and crops first in the filter chain. A frame covering the whole box is no crop at all, and one off the picture is refused by name. The crop frame is a ui primitive (CropFrame) shared with the video editor, keyboard-operable. YouTube: the wide side panel shuts while clipping and comes back after; maximised, the clip panel docks in the stage. Plex: the picture sits above the clip panel so the frame lies on it.
This commit is contained in:
@@ -7,6 +7,11 @@ range then goes through ONE re-encode built here, so every clip has the same sha
|
||||
source: frame-exact at both ends, H.264 at most 1080p (an HDR source tonemapped to SDR — kept as
|
||||
10-bit PQ it played washed out), stereo AAC, in an .mp4 that plays on any phone.
|
||||
|
||||
A clip may also crop the picture to a frame the user drew over the playing video. The frame is
|
||||
given in fractions of the box it was drawn in, with that box's aspect: the YouTube iframe never
|
||||
tells its picture's size, so the frame there is drawn over the player's 16:9 box (a short's 9:16),
|
||||
and the cut maps it onto the real picture (`crop_filter`) — what lies over a letterbox bar is dropped.
|
||||
|
||||
The spec is stored on the job (`DownloadJob.edit_spec`) and is the clip's whole identity: its
|
||||
signature is the asset's cache key, so marking the same range again is an instant cache hit, and a
|
||||
different range or audio track is a different clip.
|
||||
@@ -43,6 +48,14 @@ _TONEMAP = (
|
||||
"zscale=t=bt709:m=bt709:r=tv,format=yuv420p"
|
||||
)
|
||||
_SCALE = f"scale=-2:'min({MAX_HEIGHT},ih)'"
|
||||
# A crop frame is at least this share of its box on each side (the player's frame holds the same
|
||||
# floor), and what is left of it on the real picture at least _MIN_VISIBLE of it.
|
||||
MIN_CROP = 0.1
|
||||
_MIN_VISIBLE = 0.05
|
||||
# The box aspects a frame may be drawn in: a 9:16 short to a 21:9 film, with room either side.
|
||||
_CROP_ASPECTS = (0.25, 4.0)
|
||||
# Float slack on the frame's far edges (x + w, y + h), which the client rounds to 4 decimals.
|
||||
_EDGE_SLACK = 1e-3
|
||||
|
||||
|
||||
class ClipError(Exception):
|
||||
@@ -54,10 +67,15 @@ class ClipError(Exception):
|
||||
|
||||
|
||||
def normalize(
|
||||
source: str, ref: str, start_s: float, end_s: float, audio_track: int | None = None
|
||||
source: str,
|
||||
ref: str,
|
||||
start_s: float,
|
||||
end_s: float,
|
||||
audio_track: int | None = None,
|
||||
crop: dict | None = None,
|
||||
) -> dict:
|
||||
"""The validated, canonical clip spec. Times are rounded to the millisecond so the same marks
|
||||
always give the same cache key. Raises ClipError."""
|
||||
"""The validated, canonical clip spec. Times are rounded to the millisecond and a crop to four
|
||||
decimals, so the same marks always give the same cache key. Raises ClipError."""
|
||||
if source not in SOURCES:
|
||||
raise ClipError("bad_source")
|
||||
pattern = _YOUTUBE_ID if source == "youtube" else _RATING_KEY
|
||||
@@ -75,9 +93,37 @@ def normalize(
|
||||
if not 0 <= audio_track <= _MAX_AUDIO_TRACK:
|
||||
raise ClipError("bad_audio")
|
||||
spec["audio_track"] = audio_track
|
||||
frame = _crop(crop)
|
||||
if frame is not None:
|
||||
spec["crop"] = frame
|
||||
return spec
|
||||
|
||||
|
||||
def _crop(raw: dict | None) -> dict | None:
|
||||
"""A crop frame, canonical — or None for no crop, which a frame over the whole box also is (the
|
||||
same clip as one cut without a frame, so the same cache key)."""
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
x, y, w, h, aspect = (float(raw[k]) for k in ("x", "y", "w", "h", "aspect"))
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise ClipError("bad_crop") from e
|
||||
if not all(math.isfinite(v) for v in (x, y, w, h, aspect)):
|
||||
raise ClipError("bad_crop")
|
||||
lo, hi = _CROP_ASPECTS
|
||||
if not lo <= aspect <= hi or x < 0 or y < 0:
|
||||
raise ClipError("bad_crop")
|
||||
if w < MIN_CROP - _EDGE_SLACK or h < MIN_CROP - _EDGE_SLACK:
|
||||
raise ClipError("bad_crop")
|
||||
if x + w > 1 + _EDGE_SLACK or y + h > 1 + _EDGE_SLACK:
|
||||
raise ClipError("bad_crop")
|
||||
frame = {"x": x, "y": y, "w": min(w, 1 - x), "h": min(h, 1 - y), "aspect": aspect}
|
||||
frame = {k: round(v, 4) for k, v in frame.items()}
|
||||
if (frame["x"], frame["y"], frame["w"], frame["h"]) == (0, 0, 1, 1):
|
||||
return None
|
||||
return frame
|
||||
|
||||
|
||||
def duration(spec: dict) -> float:
|
||||
return spec["end_s"] - spec["start_s"]
|
||||
|
||||
@@ -109,6 +155,76 @@ def is_hdr(probe: dict) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def display_aspect(probe: dict) -> float | None:
|
||||
"""The picture's shape as a player shows it, from an ffprobe `-show_streams` JSON: the first video
|
||||
stream's width × its sample aspect ratio over its height, turned a quarter when the stream says
|
||||
it is rotated (ffmpeg turns such a stream upright before any filter of ours, so a crop is cut
|
||||
from the upright picture). None when the stream has no size."""
|
||||
for s in probe.get("streams") or []:
|
||||
if s.get("codec_type") != "video":
|
||||
continue
|
||||
w, h = s.get("width"), s.get("height")
|
||||
if not isinstance(w, int) or not isinstance(h, int) or w <= 0 or h <= 0:
|
||||
return None
|
||||
dar = w * (_ratio(s.get("sample_aspect_ratio")) or 1.0) / h
|
||||
return 1 / dar if _quarter_turned(s) else dar
|
||||
return None
|
||||
|
||||
|
||||
def _ratio(v: object) -> float | None:
|
||||
"""'32:27' → 1.185…; '0:1', 'N/A', a missing or malformed value → None."""
|
||||
if not isinstance(v, str) or ":" not in v:
|
||||
return None
|
||||
num, _, den = v.partition(":")
|
||||
try:
|
||||
n, d = float(num), float(den)
|
||||
except ValueError: # silent-ok: a malformed ratio reads as none given; the caller takes 1:1
|
||||
return None
|
||||
return n / d if n > 0 and d > 0 else None
|
||||
|
||||
|
||||
def _quarter_turned(stream: dict) -> bool:
|
||||
rotation: object = (stream.get("tags") or {}).get("rotate")
|
||||
for sd in stream.get("side_data_list") or []:
|
||||
if "rotation" in sd:
|
||||
rotation = sd["rotation"]
|
||||
try:
|
||||
return round(float(str(rotation))) % 180 == 90
|
||||
except ValueError: # silent-ok: no (None) or a malformed rotation: the stream is upright
|
||||
return False
|
||||
|
||||
|
||||
def crop_filter(crop: dict, dar: float) -> str | None:
|
||||
"""The ffmpeg `crop` for a frame drawn in a box of `crop["aspect"]`, over a picture of display
|
||||
aspect `dar` fitted into that box as a player fits it (whole, centred, bars on the short sides).
|
||||
The frame's part over a bar is dropped: "" when what is left is the whole picture, None when too
|
||||
little of the frame lies on the picture at all.
|
||||
|
||||
In fractions of the input's own width and height, so a sample aspect ratio other than 1:1 (an
|
||||
anamorphic DVD) needs nothing of its own, and floored to even pixels: an even 4:2:0 frame, and a
|
||||
window that never runs past the picture's far edge."""
|
||||
box = crop["aspect"]
|
||||
pic_w, pic_h = (1.0, box / dar) if dar > box else (dar / box, 1.0)
|
||||
|
||||
def span(start: float, size: float, extent: float) -> tuple[float, float]:
|
||||
origin = (1 - extent) / 2
|
||||
a = min(1.0, max(0.0, (start - origin) / extent))
|
||||
b = min(1.0, max(0.0, (start + size - origin) / extent))
|
||||
return a, b - a
|
||||
|
||||
x, w = span(crop["x"], crop["w"], pic_w)
|
||||
y, h = span(crop["y"], crop["h"], pic_h)
|
||||
if w < _MIN_VISIBLE or h < _MIN_VISIBLE:
|
||||
return None
|
||||
if w > 1 - 1e-4 and h > 1 - 1e-4:
|
||||
return ""
|
||||
|
||||
def px(axis: str, f: float) -> str:
|
||||
return f"trunc({axis}*{f:.6f}/2)*2"
|
||||
|
||||
return f"crop={px('iw', w)}:{px('ih', h)}:{px('iw', x)}:{px('ih', y)}"
|
||||
|
||||
|
||||
def audio_track_count(probe: dict) -> int:
|
||||
return sum(1 for s in probe.get("streams") or [] if s.get("codec_type") == "audio")
|
||||
|
||||
@@ -122,11 +238,13 @@ def build_cmd(
|
||||
audio_track: int | None,
|
||||
hdr: bool,
|
||||
threads: int,
|
||||
crop: str = "",
|
||||
) -> list[str]:
|
||||
"""ffmpeg for one clip: an input seek (decodes from the keyframe before `start_s` and drops the
|
||||
frames up to it, so the clip starts on the marked frame), the first video stream, the chosen audio
|
||||
track (the first when none was chosen; none at all is fine), no subtitles, data or chapters."""
|
||||
vf = ",".join([*([_TONEMAP] if hdr else []), _SCALE, EVEN_FRAME])
|
||||
track (the first when none was chosen; none at all is fine), no subtitles, data or chapters.
|
||||
A `crop` (crop_filter) comes first: the tonemap and the scale then work on the frame it keeps."""
|
||||
vf = ",".join([*([crop] if crop else []), *([_TONEMAP] if hdr else []), _SCALE, EVEN_FRAME])
|
||||
audio = f"0:a:{audio_track}" if audio_track is not None else "0:a:0?"
|
||||
return [
|
||||
"ffmpeg",
|
||||
|
||||
@@ -985,6 +985,20 @@ def _clip_source(
|
||||
return src, spec["start_s"], None
|
||||
|
||||
|
||||
def _clip_crop(spec: dict, data: dict) -> str:
|
||||
"""The clip's crop filter, from its frame and the picture the file really holds ("" for none)."""
|
||||
frame = spec.get("crop")
|
||||
if not frame:
|
||||
return ""
|
||||
dar = clipmod.display_aspect(data)
|
||||
if dar is None:
|
||||
raise StatedFailure("The picture's size could not be read, so the crop cannot be placed.")
|
||||
crop = clipmod.crop_filter(frame, dar)
|
||||
if crop is None:
|
||||
raise StatedFailure("The crop frame lies outside the picture.")
|
||||
return crop
|
||||
|
||||
|
||||
def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
"""Cut an ad-hoc clip (downloads/clip): fetch or open its source, ONE re-encode to the clip
|
||||
shape, then into the user's library beside their edits."""
|
||||
@@ -1004,6 +1018,7 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
audio_track = spec.get("audio_track")
|
||||
if audio_track is not None and audio_track >= clipmod.audio_track_count(data):
|
||||
raise StatedFailure("The chosen audio track is not in the file.")
|
||||
crop = _clip_crop(spec, data)
|
||||
total_s = clipmod.duration(spec)
|
||||
out = staging / "clip.mp4"
|
||||
_set_job(job_id, phase="editing", progress=0, speed_bps=None, eta_s=None)
|
||||
@@ -1016,6 +1031,7 @@ def _process_clip(job_id: int, asset_id: int) -> None:
|
||||
audio_track=audio_track,
|
||||
hdr=clipmod.is_hdr(data),
|
||||
threads=editmod.encode_thread_cap(),
|
||||
crop=crop,
|
||||
),
|
||||
total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "editing"),
|
||||
|
||||
@@ -578,6 +578,14 @@ def enqueue_edit(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
class ClipCropIn(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
w: float
|
||||
h: float
|
||||
aspect: float
|
||||
|
||||
|
||||
class EnqueueClipIn(BaseModel):
|
||||
source: Literal["youtube", "plex"]
|
||||
# The YouTube video id or the Plex rating key.
|
||||
@@ -586,6 +594,8 @@ class EnqueueClipIn(BaseModel):
|
||||
end_s: float
|
||||
# Plex only: which of the file's audio tracks (0-based, in file order) — the one being heard.
|
||||
audio_track: int | None = None
|
||||
# A crop frame (downloads/clip): fractions of the box it was drawn in, and that box's aspect.
|
||||
crop: ClipCropIn | None = None
|
||||
display_name: str | None = None
|
||||
|
||||
|
||||
@@ -596,7 +606,10 @@ def enqueue_clip(
|
||||
"""Queue an ad-hoc clip: the marked range of a YouTube video or a Plex item, cut straight from
|
||||
the source into a new asset in this user's library (downloads/clip)."""
|
||||
try:
|
||||
spec = clipmod.normalize(body.source, body.ref, body.start_s, body.end_s, body.audio_track)
|
||||
crop = body.crop.model_dump() if body.crop else None
|
||||
spec = clipmod.normalize(
|
||||
body.source, body.ref, body.start_s, body.end_s, body.audio_track, crop
|
||||
)
|
||||
job = service.enqueue_clip(db, user.id, spec, display_text(body.display_name))
|
||||
except clipmod.ClipError as e:
|
||||
status = 404 if e.reason == "not_found" else 422
|
||||
|
||||
@@ -126,6 +126,99 @@ def test_hdr_is_read_from_the_first_video_stream(transfer, hdr):
|
||||
assert clipmod.is_hdr(data) is hdr
|
||||
|
||||
|
||||
# --- the crop ----------------------------------------------------------------------------------------
|
||||
|
||||
_FRAME = {"x": 0.5, "y": 0.25, "w": 0.25, "h": 0.5, "aspect": 16 / 9}
|
||||
|
||||
|
||||
def test_a_crop_frame_is_canonical_and_a_frame_over_the_whole_box_is_no_crop():
|
||||
a = clipmod.normalize("plex", "42", 0, 10, crop={**_FRAME, "x": 0.50004})
|
||||
b = clipmod.normalize("plex", "42", 0, 10, crop=_FRAME)
|
||||
assert a["crop"] == b["crop"] == {"x": 0.5, "y": 0.25, "w": 0.25, "h": 0.5, "aspect": 1.7778}
|
||||
assert clipmod.sig(a) == clipmod.sig(b)
|
||||
whole = clipmod.normalize(
|
||||
"plex", "42", 0, 10, crop={"x": 0, "y": 0, "w": 1, "h": 1, "aspect": 1}
|
||||
)
|
||||
assert whole == clipmod.normalize("plex", "42", 0, 10)
|
||||
# A far edge a hair past the box (the client's rounding) is the box's edge.
|
||||
assert clipmod.normalize("plex", "42", 0, 10, crop={**_FRAME, "w": 0.5004})["crop"]["w"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frame",
|
||||
[
|
||||
{**_FRAME, "w": 0.05},
|
||||
{**_FRAME, "h": 0.09},
|
||||
{**_FRAME, "x": -0.1},
|
||||
{**_FRAME, "x": 0.8},
|
||||
{**_FRAME, "y": 0.6},
|
||||
{**_FRAME, "aspect": 0},
|
||||
{**_FRAME, "aspect": 9},
|
||||
{**_FRAME, "x": float("nan")},
|
||||
{"x": 0.1, "y": 0.1, "w": 0.5},
|
||||
{**_FRAME, "h": "half"},
|
||||
],
|
||||
)
|
||||
def test_a_crop_frame_it_cannot_cut_is_refused(frame):
|
||||
with pytest.raises(clipmod.ClipError) as e:
|
||||
clipmod.normalize("plex", "42", 0, 10, crop=frame)
|
||||
assert e.value.reason == "bad_crop"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream", "dar"),
|
||||
[
|
||||
({"width": 1920, "height": 1080}, 16 / 9),
|
||||
# An anamorphic DVD: 720×480 stored, shown 16:9.
|
||||
({"width": 720, "height": 480, "sample_aspect_ratio": "32:27"}, 16 / 9),
|
||||
({"width": 640, "height": 480, "sample_aspect_ratio": "0:1"}, 4 / 3),
|
||||
# A phone video stored landscape, shown upright.
|
||||
({"width": 1280, "height": 720, "side_data_list": [{"rotation": -90}]}, 9 / 16),
|
||||
({"width": 1280, "height": 720, "tags": {"rotate": "270"}}, 9 / 16),
|
||||
({"width": 1280, "height": 720, "side_data_list": [{"rotation": 180}]}, 16 / 9),
|
||||
({"width": None, "height": 720}, None),
|
||||
],
|
||||
)
|
||||
def test_the_display_aspect_is_the_picture_as_a_player_shows_it(stream, dar):
|
||||
got = clipmod.display_aspect(
|
||||
{"streams": [{"codec_type": "audio"}, {"codec_type": "video", **stream}]}
|
||||
)
|
||||
assert got == pytest.approx(dar) if dar else got is None
|
||||
|
||||
|
||||
def _crop_args(f: str) -> list[float]:
|
||||
# crop=trunc(iw*W/2)*2:trunc(ih*H/2)*2:trunc(iw*X/2)*2:trunc(ih*Y/2)*2 → [W, H, X, Y]
|
||||
assert f.startswith("crop=")
|
||||
return [float(p.split("*")[1].split("/")[0]) for p in f[5:].split(":")]
|
||||
|
||||
|
||||
def test_a_frame_on_a_picture_that_fills_its_box_is_cut_as_drawn():
|
||||
assert _crop_args(clipmod.crop_filter(_FRAME, 16 / 9)) == pytest.approx([0.25, 0.5, 0.5, 0.25])
|
||||
|
||||
|
||||
def test_a_frame_drawn_in_a_16_9_box_over_a_4_3_picture_maps_onto_the_picture():
|
||||
# The 4:3 picture fills 3/4 of the box's width from 1/8 in: a frame over its right half's middle.
|
||||
frame = {"x": 0.5, "y": 0.25, "w": 0.1875, "h": 0.5, "aspect": 16 / 9}
|
||||
assert _crop_args(clipmod.crop_filter(frame, 4 / 3)) == pytest.approx([0.25, 0.5, 0.5, 0.25])
|
||||
|
||||
|
||||
def test_the_part_of_a_frame_over_a_bar_is_dropped_and_a_frame_on_a_bar_alone_is_no_cut():
|
||||
over = {"x": 0.0, "y": 0.0, "w": 0.25, "h": 0.5, "aspect": 16 / 9}
|
||||
w, h, x, y = _crop_args(clipmod.crop_filter(over, 4 / 3))
|
||||
assert (w, h, x, y) == pytest.approx([1 / 6, 0.5, 0, 0], abs=1e-6)
|
||||
assert clipmod.crop_filter({**over, "w": 0.13}, 4 / 3) is None
|
||||
# A letterboxed film: the frame over the whole box is the whole picture.
|
||||
whole = {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0, "aspect": 16 / 9}
|
||||
assert clipmod.crop_filter(whole, 2.39) == ""
|
||||
|
||||
|
||||
def test_the_crop_comes_first_so_the_tonemap_and_the_scale_work_on_what_it_keeps():
|
||||
f = clipmod.crop_filter(_FRAME, 16 / 9)
|
||||
vf = _cmd(hdr=True, crop=f)[_cmd(hdr=True, crop=f).index("-vf") + 1]
|
||||
assert vf.startswith(f + ",") and vf.index("tonemap") < vf.index("min(1080,ih)")
|
||||
assert "crop" not in _cmd()[_cmd().index("-vf") + 1]
|
||||
|
||||
|
||||
# --- the route ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -351,6 +444,45 @@ def test_a_plex_clip_is_cut_from_the_library_file_with_its_audio_track(
|
||||
assert asset.rel_path.startswith(".edits/") and asset.rel_path.endswith(".mp4")
|
||||
|
||||
|
||||
def test_a_cropped_plex_clip_cuts_the_frame_from_the_picture_and_is_its_own_clip(
|
||||
db, make_user, as_user, plex_episode, worker
|
||||
):
|
||||
worker["probe"] = {
|
||||
"streams": [
|
||||
{"codec_type": "video", "width": 720, "height": 480, "sample_aspect_ratio": "32:27"}
|
||||
]
|
||||
}
|
||||
c = as_user(make_user())
|
||||
plain = _clip(c, source="plex", ref="777").json()
|
||||
job = _clip(c, source="plex", ref="777", crop=_FRAME).json()
|
||||
assert job["asset"]["id"] != plain["asset"]["id"]
|
||||
assert job["edit_spec"]["crop"]["w"] == 0.25
|
||||
job, asset = worker["run"](job)
|
||||
assert (job.status, asset.status) == ("done", "ready")
|
||||
vf = worker["cmd"][worker["cmd"].index("-vf") + 1]
|
||||
# Stored to four decimals, the box's 16:9 is a hair off the file's: well under a pixel.
|
||||
assert _crop_args(vf.split(",")[0]) == pytest.approx([0.25, 0.5, 0.5, 0.25], abs=1e-4)
|
||||
|
||||
|
||||
def test_a_crop_frame_that_misses_the_picture_fails_and_says_so(
|
||||
db, make_user, as_user, plex_episode, worker
|
||||
):
|
||||
# A 4:3 file, a frame drawn over its pillar bar in a 16:9 box.
|
||||
worker["probe"] = {"streams": [{"codec_type": "video", "width": 640, "height": 480}]}
|
||||
frame = {"x": 0.0, "y": 0.0, "w": 0.12, "h": 0.5, "aspect": 16 / 9}
|
||||
job = _clip(as_user(make_user()), source="plex", ref="777", crop=frame).json()
|
||||
job, asset = worker["run"](job)
|
||||
assert (job.status, asset.status) == ("error", "error")
|
||||
assert "crop" in job.error
|
||||
assert "cmd" not in worker
|
||||
|
||||
|
||||
def test_a_crop_frame_the_route_cannot_cut_says_why(db, make_user, as_user, yt_video):
|
||||
r = _clip(as_user(make_user()), source="youtube", ref=yt_video, crop={**_FRAME, "w": 0.01})
|
||||
assert r.status_code == 422
|
||||
assert r.json()["detail"] == {"code": "clip", "reason": "bad_crop"}
|
||||
|
||||
|
||||
def test_a_plex_clip_of_a_track_the_file_does_not_have_fails_and_says_so(
|
||||
db, make_user, as_user, plex_episode, worker
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user