From 452ff042ff0914b0ff664361f7b97e11f5671ced Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 14 Nov 2026 01:15:17 +0100 Subject: [PATCH 1/2] fix(downloads): Facebook files get their upload date, non-Latin names their letters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 2026-10-28 the signed-in Facebook read gives no upload date, so every Facebook file since was named 0000-00-00 in Season_0 (31 on prod, none before). The signed-out read that already titles these downloads carries the date, so it now lends that too when the signed-in read has none (backfill.public_title_info is now public_page_info). On-disk names dropped every letter outside the Latin script, so a Hebrew, Cyrillic, Greek or CJK uploader went to one shared "untitled" folder (prod asset 503). anyascii now writes them in Latin letters (Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged. --- backend/app/downloads/adapters.py | 2 +- backend/app/downloads/backfill.py | 27 +++++++++------ backend/app/downloads/pipeline.py | 7 ++-- backend/app/downloads/storage.py | 10 ++++-- backend/requirements.lock | 4 ++- backend/requirements.txt | 4 +++ backend/tests/test_facebook_titles.py | 50 ++++++++++++++++++++++----- backend/tests/test_naming.py | 18 ++++++++-- 8 files changed, 92 insertions(+), 30 deletions(-) diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 18a6ec63..28d25cd6 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -227,7 +227,7 @@ class FacebookAdapter(SourceAdapter): # signed-out read titles the page with that same text behind the engagement counts, cut short. # Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives # "Facebook" and NO description, while a signed-out read of the same reel still carries the text — - # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`). + # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`). _PLACEHOLDER_TITLES = frozenset({"video", "facebook"}) def is_placeholder_title(self, title: str | None) -> bool: diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py index ca0cc804..73ca7438 100644 --- a/backend/app/downloads/backfill.py +++ b/backend/app/downloads/backfill.py @@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann URL may have expired — those just get a locally-generated poster instead). A Facebook download titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its page, and so is each download name that only repeated the placeholder. That page read -(`public_title_info`) also titles every new download, from the worker. +(`public_page_info`) also titles and dates every new download, from the worker. Run once per environment: docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())" @@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in return None -def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict: - """`info` to title a download from: itself, unless its title is a source's placeholder with - nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out, - metadata-only read of the page lends its text — the description, else its title — as the - description the adapter takes a placeholder's title from. Nothing better there: `info` as it was.""" +def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict: + """`info` to title and date a download from: itself, unless its title is a source's placeholder + with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one + signed-out, metadata-only read of the page lends its text — the description, else its title — as + the description the adapter takes a placeholder's title from, and its upload date when `info` has + none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since + was named 0000-00-00 in Season_0). Nothing better there: `info` as it was.""" if not adapter.is_placeholder_title(adapter.title(info)): return info - public = extract_meta(url, adapter, asset_id) - text = public and (public.get("description") or public.get("title")) - return {**info, "description": text} if text else info + public = extract_meta(url, adapter, asset_id) or {} + lent: dict = {} + text = public.get("description") or public.get("title") + if text: + lent["description"] = text + if not info.get("upload_date") and public.get("upload_date"): + lent["upload_date"] = public["upload_date"] + return {**info, **lent} if lent else info def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool: @@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s names that only repeated it. A name its owner gave (anything else) stays.""" if not adapter.is_placeholder_title(a.title): return False - info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id) + info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id) named = normalize_title(adapter.title(info)) if not named or adapter.is_placeholder_title(named): return False diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index d4466c45..d9d418fc 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -591,15 +591,16 @@ def _download( # too. Same probe the upload ingest uses. vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info) # A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the - # page's text from one signed-out read; the download itself stays the signed-in one. The page + # page's text (and a missing upload date) from one signed-out read; the download itself stays + # the signed-in one. The page # yt-dlp read, not the link queued: a group post's link is rewritten to its video, and # signed out the post itself is the login wall. - titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id) + titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id) meta = storage.MediaMeta( video_id=info.get("id") or source_ref, title=normalize_title(adapter.title(titled)) or source_ref, uploader=info.get("uploader") or info.get("channel") or "Unknown Channel", - upload_date=_parse_upload_date(info.get("upload_date")), + upload_date=_parse_upload_date(titled.get("upload_date")), uniqueid_type=adapter.nfo_uniqueid_type(), duration_s=duration_s, description=info.get("description"), diff --git a/backend/app/downloads/storage.py b/backend/app/downloads/storage.py index 892a3ed8..d51849cb 100644 --- a/backend/app/downloads/storage.py +++ b/backend/app/downloads/storage.py @@ -20,6 +20,8 @@ from datetime import date from pathlib import Path from xml.sax.saxutils import escape +from anyascii import anyascii + from app import logs from app.downloads import mediakind @@ -121,12 +123,14 @@ def files_root_problem( def _ascii_fold(text: str) -> str: """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop - the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have - no ASCII base).""" + the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters + (`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew, + Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still + not ASCII after that is stripped.""" text = text.translate(_FOLD_MAP) text = unicodedata.normalize("NFKD", text) text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn") - return text.encode("ascii", "ignore").decode("ascii") + return anyascii(text).encode("ascii", "ignore").decode("ascii") @dataclass diff --git a/backend/requirements.lock b/backend/requirements.lock index b69add52..14cac8f5 100644 --- a/backend/requirements.lock +++ b/backend/requirements.lock @@ -7,7 +7,8 @@ # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403); # certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322); -# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284). +# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284); +# anyascii 0.3.3 added (no dependencies of its own, so the rest of the lock is unchanged). APScheduler==3.11.3 Authlib==1.9.1 Mako==1.4.1 @@ -18,6 +19,7 @@ SQLAlchemy==2.0.52 alembic==1.19.1 annotated-doc==0.0.5 annotated-types==0.8.0 +anyascii==0.3.3 anyio==4.14.2 argon2-cffi-bindings==25.1.0 argon2-cffi==25.1.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index 1f94f58f..466e0066 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,6 +13,10 @@ py3langid>=0.3,<1.0 itsdangerous>=2.1,<3.0 cryptography>=50.0.0,<51 argon2-cffi>=23.1,<26 +# On-disk names are ASCII (storage._ascii_fold): a Hebrew, Cyrillic, Greek or CJK uploader name is +# written in Latin letters instead of dropped whole into one shared "untitled" folder. Pure Python, +# no dependencies of its own. +anyascii>=0.3.3,<0.4 # Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile) # does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so diff --git a/backend/tests/test_facebook_titles.py b/backend/tests/test_facebook_titles.py index 4bebe1c6..f361454c 100644 --- a/backend/tests/test_facebook_titles.py +++ b/backend/tests/test_facebook_titles.py @@ -22,6 +22,9 @@ _PUBLIC = { "description": "sorry not helping\n\n#workhumor #officelife", "uploader": "Work dot exe", } +# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919: +# "20261113"), which the signed-in read no longer gives. +_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"} def _fake_ydl(monkeypatch, outcome): @@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo seen = _fake_ydl(monkeypatch, _PUBLIC) fb = adapters.FacebookAdapter() - info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) + info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7) assert fb.title(info) == "sorry not helping" # One metadata-only read, and signed out: no cookie goes with it. @@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url() +def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch): + # Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one, + # 31 since none) — the signed-in read lost it with the title. The signed-out read still has it. + _fake_ydl(monkeypatch, _PUBLIC_DATED) + + info = backfill.public_page_info( + {"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7 + ) + + assert info["upload_date"] == "20261113" + + +def test_a_date_the_signed_in_read_gave_stays(monkeypatch): + _fake_ydl(monkeypatch, _PUBLIC_DATED) + + info = backfill.public_page_info( + {"title": "Facebook", "description": None, "upload_date": "20261001"}, + _URL, + adapters.FacebookAdapter(), + 7, + ) + + assert info["upload_date"] == "20261001" + + def test_a_real_title_reads_nothing_more(monkeypatch): seen = _fake_ydl(monkeypatch, _PUBLIC) info = {"title": "Funny Dark Humor Dad Joke!", "description": None} - assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info + assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert seen == [] @@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa seen = _fake_ydl(monkeypatch, _PUBLIC) info = {"title": "Video", "description": "Office jokes, part two"} - assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info + assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert seen == [] @@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch): _fake_ydl(monkeypatch, {"title": "Facebook", "description": None}) fb = adapters.FacebookAdapter() - info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) + info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7) assert fb.title(info) == "Facebook" @@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch): seen = _fake_ydl(monkeypatch, _PUBLIC) info = {"title": "Video", "description": None} - assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info + assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info assert seen == [] @@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch): db.add(job) db.commit() - read_page = backfill.public_title_info + read_page = backfill.public_page_info def renamed_while_reading(*args): with SessionLocal() as other: @@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch): other.commit() return read_page(*args) - monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading) + monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading) monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None) assert backfill.run()["retitled"] == 1 @@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page( db, make_user, monkeypatch, tmp_path ): # The whole download path with yt-dlp and the media tools faked: the signed-in read gives only - # the placeholder, and the file and the download's name get the post's text. - seen = _fake_ydl(monkeypatch, _PUBLIC) + # the placeholder, and the file and the download's name get the post's text — and its date. + seen = _fake_ydl(monkeypatch, _PUBLIC_DATED) monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced) @@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page( assert db.get(DownloadJob, job.id).status == "done" assert db.get(MediaAsset, asset.id).title == "sorry not helping" assert db.get(DownloadJob, job.id).display_name == "sorry not helping" + done = db.get(MediaAsset, asset.id) + assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13" + # The date names the file and its season, not 0000-00-00 / Season_0. + assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_") diff --git a/backend/tests/test_naming.py b/backend/tests/test_naming.py index 0d383446..47a27eaa 100644 --- a/backend/tests/test_naming.py +++ b/backend/tests/test_naming.py @@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces(): assert _is_ascii_slug(out), out -def test_sanitize_non_latin_falls_back_not_crashes(): - # A title with no ASCII base collapses to the untitled fallback rather than an empty name. - assert storage.sanitize("日本語") == "untitled" +def test_sanitize_writes_a_non_latin_name_in_latin_letters(): + # Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the + # folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them. + names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي") + slugs = [storage.sanitize(n) for n in names] + for name, slug in zip(names, slugs, strict=True): + assert _is_ascii_slug(slug) and slug != "untitled", (name, slug) + assert len(set(slugs)) == len(slugs), slugs # two names, two folders + # The measured spellings (anyascii 0.3.3, the locked one). + assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"] + + +def test_sanitize_falls_back_when_nothing_is_left(): + # Emoji and symbols are dropped, and a name of nothing else still gets a folder. + assert storage.sanitize("🎉🎉") == "untitled" def test_display_filename_uses_underscores_and_ascii(): From 87ab33b03c71918aefa2d80ed8c6a4f8faf44462 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 14 Nov 2026 01:21:11 +0100 Subject: [PATCH 2/2] release: 0.119.2 --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- frontend/src/content/release-notes/en.json | 8 ++++++++ frontend/src/content/release-notes/hu.json | 8 ++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c7adb2..94a3b74f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.119.2 — 2026-11-14 + +### Fixed + +- A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming "untitled". + ## 0.119.1 — 2026-11-14 ### Fixed diff --git a/VERSION b/VERSION index 368ee87c..6c17fd52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.119.1 +0.119.2 diff --git a/frontend/src/content/release-notes/en.json b/frontend/src/content/release-notes/en.json index dce00bab..96840d2d 100644 --- a/frontend/src/content/release-notes/en.json +++ b/frontend/src/content/release-notes/en.json @@ -1,4 +1,12 @@ [ + { + "version": "0.119.2", + "date": "2026-11-14", + "new": [], + "fixed": [ + "A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming \"untitled\"." + ] + }, { "version": "0.119.1", "date": "2026-11-14", diff --git a/frontend/src/content/release-notes/hu.json b/frontend/src/content/release-notes/hu.json index a161a11d..ccdf3bca 100644 --- a/frontend/src/content/release-notes/hu.json +++ b/frontend/src/content/release-notes/hu.json @@ -1,4 +1,12 @@ [ + { + "version": "0.119.2", + "date": "2026-11-14", + "new": [], + "fixed": [ + "A Facebook-letöltések újra megkapják a feltöltés dátumát, így a fájl és az évad neve azt mutatja, nem 0000-00-00-t. A más írású, például héber vagy cirill csatornanév a fájlnevekben latin betűkkel jelenik meg, nem „untitled” lesz." + ] + }, { "version": "0.119.1", "date": "2026-11-14",