refactor(downloads): address /code-review on the download hotfix
- edit.py: extract _run_ffmpeg_best_effort (one run+timeout+swallow policy shared by
thumbnail_to_jpg / ensure_poster / ensure_storyboard) + a build_thumbnail_jpg_cmd
builder, matching the module's other ffmpeg commands.
- thumbnail_to_jpg: copy an already-JPEG thumbnail (Facebook, and Reddit's jpeg-under-
a-.png-name) instead of re-encoding it; only webp/png-proper are decoded via ffmpeg.
- errors.py: drop the redundant "account authentication" classify marker
("authentication is required" already covers the known Reddit message).
- formats.py: document that the opt-in EmbedThumbnail is unaffected by dropping the
convertor (it converts unsupported thumbs to PNG, not the crashing mjpeg2jpeg path).
- test_titles.py: guard the ReDoS regression with a thread + timeout (deterministic
"must not hang") instead of a wall-clock assertion.
This commit is contained in:
@@ -1,4 +1,9 @@
|
||||
"""normalize_title: de-shout / de-clutter video titles without mangling non-English text."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FuturesTimeout
|
||||
|
||||
import pytest
|
||||
|
||||
from app.titles import normalize_title
|
||||
|
||||
|
||||
@@ -29,20 +34,26 @@ def test_numeric_episode_marker_is_not_treated_as_a_hashtag():
|
||||
assert normalize_title("My show #3") == "My show #3"
|
||||
|
||||
|
||||
def test_midstring_hashtags_are_fast_and_kept_redos_regression():
|
||||
def test_midstring_hashtags_dont_hang_and_are_kept_redos_regression():
|
||||
# Facebook prepends an engagement prefix and appends a page name, leaving a long hashtag cluster
|
||||
# in the MIDDLE. That is not a trailing SEO cluster, so the tags stay — and, critically, this
|
||||
# exact shape used to trigger catastrophic backtracking (ReDoS) in `_TRAILING_HASHTAGS` that
|
||||
# pinned a download-worker thread at 100% CPU forever inside the yt-dlp progress hook.
|
||||
import time
|
||||
|
||||
raw = (
|
||||
"398K views · 6.1K reactions | The money should follow 🙄 #reel #reels #meme #memes "
|
||||
"#fun #funny #work #worklife #office #viral #trending #fyp #foryou #foryoupage | Memebonanza4U"
|
||||
)
|
||||
t = time.perf_counter()
|
||||
out = normalize_title(raw) or ""
|
||||
assert time.perf_counter() - t < 1.0 # linear now; the old regex ran for minutes / never returned
|
||||
# Run in a worker thread with a generous ceiling: the fixed regex returns in microseconds, so
|
||||
# only a genuine catastrophic-backtracking regression fails to complete — a hang is caught
|
||||
# deterministically rather than by asserting on wall-clock slack in the happy path.
|
||||
ex = ThreadPoolExecutor(max_workers=1)
|
||||
fut = ex.submit(normalize_title, raw)
|
||||
try:
|
||||
out = fut.result(timeout=5) or ""
|
||||
except FuturesTimeout:
|
||||
pytest.fail("normalize_title did not return within 5s — catastrophic backtracking regressed")
|
||||
finally:
|
||||
ex.shutdown(wait=False) # never join a (hypothetically) wedged thread
|
||||
assert "#foryoupage" in out and "#reel" in out # mid-string hashtags preserved (not trailing)
|
||||
assert "views" not in out # the engagement-count prefix was still stripped
|
||||
|
||||
|
||||
Reference in New Issue
Block a user