fix(downloads): shell-safe names, refresh display_name on re-download, GC orphan sweep
Follow-up to user prod testing of 0.51.0: - re-download now clears job.display_name so the card title + Content-Disposition filename are re-derived from the freshly-cleaned title (previously an auto-name from an older un-cleaned title — still carrying a "… views · … reactions |" prefix — survived the rebuild). - on-disk AND served names are now a strict shell-safe slug: only [A-Za-z0-9._-] survive; every other char (the "!" the user saw, plus ( ) [ ] & $ ' " ; | ? * …) collapses to "_". The id no longer uses [brackets] (glob metacharacters). Shared _slug helper for sanitize + display_filename. - GC gains a 5th pass: sweep on-disk files/dirs no ready asset owns (old accented channel dirs, a poster from a superseded naming scheme left after a re-download), skipping the dot system trees and anything newer than an hour so an in-flight download is never touched. - Tests: slug strips !/$/()[]; rel_path is bracket-free; orphan-sweep keeps owned/system/fresh files.
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
"""Orphan-file sweep: remove on-disk cruft no ready asset owns (old accented dirs, stale posters),
|
||||
while never touching system trees, owned files, or freshly-written downloads."""
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads.gc import _sweep_orphan_files
|
||||
|
||||
|
||||
class _Scalars:
|
||||
def __init__(self, rows):
|
||||
self._rows = rows
|
||||
|
||||
def scalars(self):
|
||||
return list(self._rows)
|
||||
|
||||
|
||||
class _StubDB:
|
||||
"""Minimal stand-in: `_sweep_orphan_files` only issues one select for ready rel_paths."""
|
||||
|
||||
def __init__(self, rel_paths):
|
||||
self._rel_paths = rel_paths
|
||||
|
||||
def execute(self, _stmt):
|
||||
return _Scalars(self._rel_paths)
|
||||
|
||||
|
||||
def _aged(path: Path, hours: float) -> None:
|
||||
old = time.time() - hours * 3600
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def test_sweep_removes_orphans_but_keeps_owned_system_and_fresh(tmp_path: Path):
|
||||
root = tmp_path
|
||||
# An owned asset (in the keep set) + its sidecars.
|
||||
kept_dir = root / "Chan" / "Season_2026"
|
||||
kept_dir.mkdir(parents=True)
|
||||
kept_media = kept_dir / "Chan_-_2026-01-01_-_Title_vid_a1.mp4"
|
||||
for p in (kept_media, kept_media.with_suffix(".nfo"), kept_media.with_suffix(".jpg")):
|
||||
p.write_bytes(b"x")
|
||||
_aged(p, 5) # old, but owned → must survive
|
||||
|
||||
# An orphan poster in a stale (e.g. old accented) channel dir — old, unowned → swept.
|
||||
orphan_dir = root / "OldChan" / "Season_2026"
|
||||
orphan_dir.mkdir(parents=True)
|
||||
orphan = orphan_dir / "OldChan_-_old_title_vid.jpg"
|
||||
orphan.write_bytes(b"x")
|
||||
_aged(orphan, 5)
|
||||
|
||||
# A system tree (storyboards) — never touched, even when old.
|
||||
sysfile = root / ".storyboards" / "asset-1.jpg"
|
||||
sysfile.parent.mkdir()
|
||||
sysfile.write_bytes(b"x")
|
||||
_aged(sysfile, 5)
|
||||
|
||||
# A brand-new unowned file (an in-flight download's just-placed output) — too young to sweep.
|
||||
fresh_dir = root / "FreshChan" / "Season_2026"
|
||||
fresh_dir.mkdir(parents=True)
|
||||
fresh = fresh_dir / "FreshChan_-_now_vid_a2.mp4"
|
||||
fresh.write_bytes(b"x") # mtime = now
|
||||
|
||||
db = _StubDB(["Chan/Season_2026/Chan_-_2026-01-01_-_Title_vid_a1.mp4"])
|
||||
removed = _sweep_orphan_files(db, str(root))
|
||||
|
||||
assert removed == 1 # only the aged orphan
|
||||
assert kept_media.exists() and kept_media.with_suffix(".jpg").exists()
|
||||
assert not orphan.exists() and not orphan_dir.exists() # orphan + its emptied dirs pruned
|
||||
assert not (root / "OldChan").exists()
|
||||
assert sysfile.exists() # system tree untouched
|
||||
assert fresh.exists() # too new to sweep
|
||||
@@ -47,6 +47,17 @@ def test_rel_path_is_ascii_everywhere():
|
||||
rel = storage.rel_path(meta, "mp4", "plex", asset_id=7)
|
||||
assert _is_ascii_slug(rel.replace("/", ""))
|
||||
assert "Hajdu_Balazs" in rel and "Aristom_Odon" in rel
|
||||
# Shell-unpleasant chars (incl. the id brackets) never appear in a generated path.
|
||||
for bad in "[]!?*&$()'\"; |":
|
||||
assert bad not in rel
|
||||
|
||||
|
||||
def test_rel_path_strips_exclamation_and_other_shell_chars():
|
||||
meta = storage.MediaMeta(
|
||||
video_id="abc123", title="Rovinjból! A vihar (2026) $$$", uploader="Chan", upload_date=None
|
||||
)
|
||||
rel = storage.rel_path(meta, "mp4", "plex", asset_id=1)
|
||||
assert "!" not in rel and "$" not in rel and "(" not in rel
|
||||
|
||||
|
||||
def test_engagement_prefix_stripped():
|
||||
|
||||
@@ -23,19 +23,19 @@ class TestSanitize:
|
||||
for bad in '/\\:*?"<>|🔥':
|
||||
assert bad not in out
|
||||
|
||||
def test_repeated_punctuation_collapses(self):
|
||||
assert sanitize("wow!!!___---end") == "wow!_-end"
|
||||
def test_unsafe_punctuation_is_stripped_to_a_single_separator(self):
|
||||
# Shell-unpleasant punctuation (! etc.) is removed; runs of separators collapse to one "_".
|
||||
assert sanitize("wow!!!___---end") == "wow_end"
|
||||
assert "!" not in sanitize("Rovinjból! A vihar")
|
||||
|
||||
def test_length_is_capped(self):
|
||||
assert len(sanitize("x" * 500, limit=10)) <= 10
|
||||
|
||||
def test_cap_is_by_bytes_not_characters(self):
|
||||
# Accented (2-byte UTF-8) letters must not blow past the byte limit — the FS component cap
|
||||
# is 255 BYTES, so a char-only cap would overflow for multi-byte titles.
|
||||
def test_cap_stays_within_the_byte_limit(self):
|
||||
# Names are folded to ASCII, so a char cap is also the byte cap — accented input can't
|
||||
# overflow the 255-BYTE FS component limit.
|
||||
out = sanitize("á" * 500, limit=10)
|
||||
assert len(out.encode("utf-8")) <= 10
|
||||
# And it never splits a character (would raise otherwise / leave a replacement char).
|
||||
out.encode("utf-8").decode("utf-8")
|
||||
assert out.isascii() and len(out.encode("utf-8")) <= 10
|
||||
|
||||
def test_empty_or_all_junk_falls_back_to_untitled(self):
|
||||
assert sanitize("") == "untitled"
|
||||
@@ -51,11 +51,11 @@ class TestRelPath:
|
||||
)
|
||||
|
||||
def test_flat_layout_is_a_single_file(self):
|
||||
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_[abc123].mp4"
|
||||
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_abc123.mp4"
|
||||
|
||||
def test_plex_layout_nests_channel_and_season(self):
|
||||
out = rel_path(self.meta, "mp4", layout="plex")
|
||||
assert out == "Some_Channel/Season_2024/Some_Channel_-_2024-05-02_-_Cool_Clip_[abc123].mp4"
|
||||
assert out == "Some_Channel/Season_2024/Some_Channel_-_2024-05-02_-_Cool_Clip_abc123.mp4"
|
||||
|
||||
def test_missing_date_uses_year_zero_and_placeholder_date(self):
|
||||
meta = MediaMeta(video_id="x", title="t", uploader="c", upload_date=None)
|
||||
@@ -67,10 +67,10 @@ class TestRelPath:
|
||||
a = rel_path(self.meta, "mp4", asset_id=5)
|
||||
b = rel_path(self.meta, "mp4", asset_id=6)
|
||||
assert a != b
|
||||
assert a.endswith("_[abc123]_a5.mp4") and b.endswith("_[abc123]_a6.mp4")
|
||||
assert a.endswith("_abc123_a5.mp4") and b.endswith("_abc123_a6.mp4")
|
||||
# Omitting it keeps the plain name (unchanged for callers/tests that don't pass one).
|
||||
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_[abc123].mp4"
|
||||
assert rel_path(self.meta, "mp4", layout="flat", asset_id=7) == "Cool_Clip_[abc123]_a7.mp4"
|
||||
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_abc123.mp4"
|
||||
assert rel_path(self.meta, "mp4", layout="flat", asset_id=7) == "Cool_Clip_abc123_a7.mp4"
|
||||
|
||||
def test_every_component_stays_within_the_255_byte_fs_limit(self):
|
||||
# A long accented channel AND title, plus the asset-id suffix, must not overflow any path
|
||||
|
||||
Reference in New Issue
Block a user