peter
452ff042ff
Since 2026-10-28 the signed-in Facebook read gives no upload date, so every Facebook file since was named 0000-00-00 in Season_0 (31 on prod, none before). The signed-out read that already titles these downloads carries the date, so it now lends that too when the signed-in read has none (backfill.public_title_info is now public_page_info). On-disk names dropped every letter outside the Latin script, so a Hebrew, Cyrillic, Greek or CJK uploader went to one shared "untitled" folder (prod asset 503). anyascii now writes them in Latin letters (Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
106 lines
4.4 KiB
Python
106 lines
4.4 KiB
Python
"""ASCII-slug naming (no spaces, no accents) for on-disk paths and served filenames, and the
|
|
social engagement-prefix strip in title normalization."""
|
|
|
|
from app.downloads import storage
|
|
from app.titles import normalize_title
|
|
|
|
|
|
def _is_ascii_slug(s: str) -> bool:
|
|
return s.isascii() and " " not in s
|
|
|
|
|
|
def test_sanitize_folds_accents_to_ascii():
|
|
assert storage.sanitize("Barnabás Skriván") == "Barnabas_Skrivan"
|
|
assert storage.sanitize("Comedy Central Magyarország") == "Comedy_Central_Magyarorszag"
|
|
assert (
|
|
storage.sanitize("Marco Mobili Bútoráruház Veszprém") == "Marco_Mobili_Butoraruhaz_Veszprem"
|
|
)
|
|
# Hungarian double-acute (ő/ű) and German ß.
|
|
assert storage.sanitize("Előző Műsor") == "Elozo_Musor"
|
|
assert storage.sanitize("Straße") == "Strasse"
|
|
|
|
|
|
def test_sanitize_output_is_pure_ascii_without_spaces():
|
|
for name in ("Poénbomba", "Mágáné Zsuzsanna", "café déjà vu", "naïve œuvre"):
|
|
out = storage.sanitize(name)
|
|
assert _is_ascii_slug(out), out
|
|
|
|
|
|
def test_sanitize_writes_a_non_latin_name_in_latin_letters():
|
|
# Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
|
|
# folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
|
|
names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
|
|
slugs = [storage.sanitize(n) for n in names]
|
|
for name, slug in zip(names, slugs, strict=True):
|
|
assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
|
|
assert len(set(slugs)) == len(slugs), slugs # two names, two folders
|
|
# The measured spellings (anyascii 0.3.3, the locked one).
|
|
assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
|
|
|
|
|
|
def test_sanitize_falls_back_when_nothing_is_left():
|
|
# Emoji and symbols are dropped, and a name of nothing else still gets a folder.
|
|
assert storage.sanitize("🎉🎉") == "untitled"
|
|
|
|
|
|
def test_display_filename_uses_underscores_and_ascii():
|
|
out = storage.display_filename("Amikor a férfi éhesen ér haza")
|
|
assert out == "Amikor_a_ferfi_ehesen_er_haza"
|
|
assert _is_ascii_slug(out)
|
|
|
|
|
|
def test_download_filename_appends_single_extension():
|
|
from pathlib import Path
|
|
|
|
assert storage.download_filename("Szép Videó", "mp4", Path("x.mp4")) == "Szep_Video.mp4"
|
|
|
|
|
|
def test_rel_path_is_ascii_everywhere():
|
|
meta = storage.MediaMeta(
|
|
video_id="abc123", title="Áristom Ödön", uploader="Hajdú Balázs", upload_date=None
|
|
)
|
|
rel = storage.rel_path(meta, "mp4", "plex", asset_id=7)
|
|
assert _is_ascii_slug(rel.replace("/", ""))
|
|
assert "Hajdu_Balazs" in rel and "Aristom_Odon" in rel
|
|
# Shell-unpleasant chars (incl. the id brackets) never appear in a generated path.
|
|
for bad in "[]!?*&$()'\"; |":
|
|
assert bad not in rel
|
|
|
|
|
|
def test_rel_path_strips_exclamation_and_other_shell_chars():
|
|
meta = storage.MediaMeta(
|
|
video_id="abc123", title="Rovinjból! A vihar (2026) $$$", uploader="Chan", upload_date=None
|
|
)
|
|
rel = storage.rel_path(meta, "mp4", "plex", asset_id=1)
|
|
assert "!" not in rel and "$" not in rel and "(" not in rel
|
|
|
|
|
|
def test_engagement_prefix_stripped():
|
|
assert (
|
|
normalize_title("1.6M views · 66K reactions | Amikor a férfi éhesen ér haza")
|
|
== "Amikor a férfi éhesen ér haza"
|
|
)
|
|
assert normalize_title("13M views · 162K reactions |Valami cím") == "Valami cím"
|
|
assert normalize_title("500 views | Rövid") == "Rövid"
|
|
|
|
|
|
def test_engagement_prefix_does_not_touch_normal_titles():
|
|
# A pipe that isn't an engagement prefix must survive.
|
|
assert normalize_title("Vlog 12 | A nagy nap") == "Vlog 12 | A nagy nap"
|
|
assert normalize_title("How I built a boat") == "How I built a boat"
|
|
|
|
|
|
def test_engagement_only_title_does_not_resurrect_the_stats():
|
|
# A title that is ONLY the stats prefix must not fall back to the raw (stats-bearing) string.
|
|
out = normalize_title("1.6M views · 66K reactions |")
|
|
assert out == ""
|
|
assert "views" not in (out or "") and "reactions" not in (out or "")
|
|
|
|
|
|
def test_prefix_plus_only_hashtags_resurrects_neither_stats_nor_hashtags():
|
|
# stats prefix + trailing SEO hashtags → both are removed; the fallback must not bring back the
|
|
# hashtags that rule 2 strips (nor the stats prefix).
|
|
out = normalize_title("5 views | #foo #bar")
|
|
assert out == ""
|
|
assert "#foo" not in (out or "") and "views" not in (out or "")
|