fix(downloads): Facebook files get their upload date, non-Latin names their letters

Since 2026-10-28 the signed-in Facebook read gives no upload date, so
every Facebook file since was named 0000-00-00 in Season_0 (31 on prod,
none before). The signed-out read that already titles these downloads
carries the date, so it now lends that too when the signed-in read has
none (backfill.public_title_info is now public_page_info).

On-disk names dropped every letter outside the Latin script, so a Hebrew,
Cyrillic, Greek or CJK uploader went to one shared "untitled" folder
(prod asset 503). anyascii now writes them in Latin letters
(Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
This commit is contained in:
2026-11-14 01:15:17 +01:00
parent 25198a936f
commit 452ff042ff
8 changed files with 92 additions and 30 deletions
+41 -9
View File
@@ -22,6 +22,9 @@ _PUBLIC = {
"description": "sorry not helping\n\n#workhumor #officelife",
"uploader": "Work dot exe",
}
# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919:
# "20261113"), which the signed-in read no longer gives.
_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"}
def _fake_ydl(monkeypatch, outcome):
@@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
seen = _fake_ydl(monkeypatch, _PUBLIC)
fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "sorry not helping"
# One metadata-only read, and signed out: no cookie goes with it.
@@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url()
def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch):
# Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one,
# 31 since none) — the signed-in read lost it with the title. The signed-out read still has it.
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7
)
assert info["upload_date"] == "20261113"
def test_a_date_the_signed_in_read_gave_stays(monkeypatch):
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None, "upload_date": "20261001"},
_URL,
adapters.FacebookAdapter(),
7,
)
assert info["upload_date"] == "20261001"
def test_a_real_title_reads_nothing_more(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Funny Dark Humor Dad Joke!", "description": None}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == []
@@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": "Office jokes, part two"}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == []
@@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch):
_fake_ydl(monkeypatch, {"title": "Facebook", "description": None})
fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "Facebook"
@@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": None}
assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert seen == []
@@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
db.add(job)
db.commit()
read_page = backfill.public_title_info
read_page = backfill.public_page_info
def renamed_while_reading(*args):
with SessionLocal() as other:
@@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
other.commit()
return read_page(*args)
monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading)
monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading)
monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None)
assert backfill.run()["retitled"] == 1
@@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
db, make_user, monkeypatch, tmp_path
):
# The whole download path with yt-dlp and the media tools faked: the signed-in read gives only
# the placeholder, and the file and the download's name get the post's text.
seen = _fake_ydl(monkeypatch, _PUBLIC)
# the placeholder, and the file and the download's name get the post's text — and its date.
seen = _fake_ydl(monkeypatch, _PUBLIC_DATED)
monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
@@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
assert db.get(DownloadJob, job.id).status == "done"
assert db.get(MediaAsset, asset.id).title == "sorry not helping"
assert db.get(DownloadJob, job.id).display_name == "sorry not helping"
done = db.get(MediaAsset, asset.id)
assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13"
# The date names the file and its season, not 0000-00-00 / Season_0.
assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_")
+15 -3
View File
@@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces():
assert _is_ascii_slug(out), out
def test_sanitize_non_latin_falls_back_not_crashes():
# A title with no ASCII base collapses to the untitled fallback rather than an empty name.
assert storage.sanitize("日本語") == "untitled"
def test_sanitize_writes_a_non_latin_name_in_latin_letters():
# Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
# folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
slugs = [storage.sanitize(n) for n in names]
for name, slug in zip(names, slugs, strict=True):
assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
assert len(set(slugs)) == len(slugs), slugs # two names, two folders
# The measured spellings (anyascii 0.3.3, the locked one).
assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
def test_sanitize_falls_back_when_nothing_is_left():
# Emoji and symbols are dropped, and a name of nothing else still gets a folder.
assert storage.sanitize("🎉🎉") == "untitled"
def test_display_filename_uses_underscores_and_ascii():