fix(downloads): shell-safe names, refresh display_name on re-download, GC orphan sweep

Follow-up to user prod testing of 0.51.0:
- re-download now clears job.display_name so the card title + Content-Disposition filename are
  re-derived from the freshly-cleaned title (previously an auto-name from an older un-cleaned
  title — still carrying a "… views · … reactions |" prefix — survived the rebuild).
- on-disk AND served names are now a strict shell-safe slug: only [A-Za-z0-9._-] survive; every
  other char (the "!" the user saw, plus ( ) [ ] & $ ' " ; | ? * …) collapses to "_". The id no
  longer uses [brackets] (glob metacharacters). Shared _slug helper for sanitize + display_filename.
- GC gains a 5th pass: sweep on-disk files/dirs no ready asset owns (old accented channel dirs, a
  poster from a superseded naming scheme left after a re-download), skipping the dot system trees
  and anything newer than an hour so an in-flight download is never touched.
- Tests: slug strips !/$/()[]; rel_path is bracket-free; orphan-sweep keeps owned/system/fresh files.
This commit is contained in:
2026-07-22 03:18:14 +02:00
parent e21c6f6b92
commit fa35925cc6
6 changed files with 196 additions and 60 deletions
+70
View File
@@ -0,0 +1,70 @@
"""Orphan-file sweep: remove on-disk cruft no ready asset owns (old accented dirs, stale posters),
while never touching system trees, owned files, or freshly-written downloads."""
import os
import time
from pathlib import Path
from app.downloads.gc import _sweep_orphan_files
class _Scalars:
def __init__(self, rows):
self._rows = rows
def scalars(self):
return list(self._rows)
class _StubDB:
"""Minimal stand-in: `_sweep_orphan_files` only issues one select for ready rel_paths."""
def __init__(self, rel_paths):
self._rel_paths = rel_paths
def execute(self, _stmt):
return _Scalars(self._rel_paths)
def _aged(path: Path, hours: float) -> None:
old = time.time() - hours * 3600
os.utime(path, (old, old))
def test_sweep_removes_orphans_but_keeps_owned_system_and_fresh(tmp_path: Path):
root = tmp_path
# An owned asset (in the keep set) + its sidecars.
kept_dir = root / "Chan" / "Season_2026"
kept_dir.mkdir(parents=True)
kept_media = kept_dir / "Chan_-_2026-01-01_-_Title_vid_a1.mp4"
for p in (kept_media, kept_media.with_suffix(".nfo"), kept_media.with_suffix(".jpg")):
p.write_bytes(b"x")
_aged(p, 5) # old, but owned → must survive
# An orphan poster in a stale (e.g. old accented) channel dir — old, unowned → swept.
orphan_dir = root / "OldChan" / "Season_2026"
orphan_dir.mkdir(parents=True)
orphan = orphan_dir / "OldChan_-_old_title_vid.jpg"
orphan.write_bytes(b"x")
_aged(orphan, 5)
# A system tree (storyboards) — never touched, even when old.
sysfile = root / ".storyboards" / "asset-1.jpg"
sysfile.parent.mkdir()
sysfile.write_bytes(b"x")
_aged(sysfile, 5)
# A brand-new unowned file (an in-flight download's just-placed output) — too young to sweep.
fresh_dir = root / "FreshChan" / "Season_2026"
fresh_dir.mkdir(parents=True)
fresh = fresh_dir / "FreshChan_-_now_vid_a2.mp4"
fresh.write_bytes(b"x") # mtime = now
db = _StubDB(["Chan/Season_2026/Chan_-_2026-01-01_-_Title_vid_a1.mp4"])
removed = _sweep_orphan_files(db, str(root))
assert removed == 1 # only the aged orphan
assert kept_media.exists() and kept_media.with_suffix(".jpg").exists()
assert not orphan.exists() and not orphan_dir.exists() # orphan + its emptied dirs pruned
assert not (root / "OldChan").exists()
assert sysfile.exists() # system tree untouched
assert fresh.exists() # too new to sweep
+11
View File
@@ -47,6 +47,17 @@ def test_rel_path_is_ascii_everywhere():
rel = storage.rel_path(meta, "mp4", "plex", asset_id=7)
assert _is_ascii_slug(rel.replace("/", ""))
assert "Hajdu_Balazs" in rel and "Aristom_Odon" in rel
# Shell-unpleasant chars (incl. the id brackets) never appear in a generated path.
for bad in "[]!?*&$()'\"; |":
assert bad not in rel
def test_rel_path_strips_exclamation_and_other_shell_chars():
meta = storage.MediaMeta(
video_id="abc123", title="Rovinjból! A vihar (2026) $$$", uploader="Chan", upload_date=None
)
rel = storage.rel_path(meta, "mp4", "plex", asset_id=1)
assert "!" not in rel and "$" not in rel and "(" not in rel
def test_engagement_prefix_stripped():
+13 -13
View File
@@ -23,19 +23,19 @@ class TestSanitize:
for bad in '/\\:*?"<>|🔥':
assert bad not in out
def test_repeated_punctuation_collapses(self):
assert sanitize("wow!!!___---end") == "wow!_-end"
def test_unsafe_punctuation_is_stripped_to_a_single_separator(self):
# Shell-unpleasant punctuation (! etc.) is removed; runs of separators collapse to one "_".
assert sanitize("wow!!!___---end") == "wow_end"
assert "!" not in sanitize("Rovinjból! A vihar")
def test_length_is_capped(self):
assert len(sanitize("x" * 500, limit=10)) <= 10
def test_cap_is_by_bytes_not_characters(self):
# Accented (2-byte UTF-8) letters must not blow past the byte limit — the FS component cap
# is 255 BYTES, so a char-only cap would overflow for multi-byte titles.
def test_cap_stays_within_the_byte_limit(self):
# Names are folded to ASCII, so a char cap is also the byte cap — accented input can't
# overflow the 255-BYTE FS component limit.
out = sanitize("á" * 500, limit=10)
assert len(out.encode("utf-8")) <= 10
# And it never splits a character (would raise otherwise / leave a replacement char).
out.encode("utf-8").decode("utf-8")
assert out.isascii() and len(out.encode("utf-8")) <= 10
def test_empty_or_all_junk_falls_back_to_untitled(self):
assert sanitize("") == "untitled"
@@ -51,11 +51,11 @@ class TestRelPath:
)
def test_flat_layout_is_a_single_file(self):
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_[abc123].mp4"
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_abc123.mp4"
def test_plex_layout_nests_channel_and_season(self):
out = rel_path(self.meta, "mp4", layout="plex")
assert out == "Some_Channel/Season_2024/Some_Channel_-_2024-05-02_-_Cool_Clip_[abc123].mp4"
assert out == "Some_Channel/Season_2024/Some_Channel_-_2024-05-02_-_Cool_Clip_abc123.mp4"
def test_missing_date_uses_year_zero_and_placeholder_date(self):
meta = MediaMeta(video_id="x", title="t", uploader="c", upload_date=None)
@@ -67,10 +67,10 @@ class TestRelPath:
a = rel_path(self.meta, "mp4", asset_id=5)
b = rel_path(self.meta, "mp4", asset_id=6)
assert a != b
assert a.endswith("_[abc123]_a5.mp4") and b.endswith("_[abc123]_a6.mp4")
assert a.endswith("_abc123_a5.mp4") and b.endswith("_abc123_a6.mp4")
# Omitting it keeps the plain name (unchanged for callers/tests that don't pass one).
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_[abc123].mp4"
assert rel_path(self.meta, "mp4", layout="flat", asset_id=7) == "Cool_Clip_[abc123]_a7.mp4"
assert rel_path(self.meta, "mp4", layout="flat") == "Cool_Clip_abc123.mp4"
assert rel_path(self.meta, "mp4", layout="flat", asset_id=7) == "Cool_Clip_abc123_a7.mp4"
def test_every_component_stays_within_the_255_byte_fs_limit(self):
# A long accented channel AND title, plus the asset-id suffix, must not overflow any path