release: 0.119.2 (bug/fb-date-translit)

2026-11-14 01:34:13 +01:00
parent 25198a936f 87ab33b03c
commit c7c314ace2
12 changed files with 115 additions and 31 deletions
@@ -7,6 +7,12 @@ edit the release notes and re-run the generator.

 Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).

+## 0.119.2 — 2026-11-14
+
+### Fixed
+
+- A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming "untitled".
+
 ## 0.119.1 — 2026-11-14

 ### Fixed
@@ -1 +1 @@
-0.119.1
+0.119.2
@@ -227,7 +227,7 @@ class FacebookAdapter(SourceAdapter):
    # signed-out read titles the page with that same text behind the engagement counts, cut short.
    # Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives
    # "Facebook" and NO description, while a signed-out read of the same reel still carries the text —
-    # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`).
+    # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`).
    _PLACEHOLDER_TITLES = frozenset({"video", "facebook"})

    def is_placeholder_title(self, title: str | None) -> bool:
@@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann
 URL may have expired — those just get a locally-generated poster instead). A Facebook download
 titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
 page, and so is each download name that only repeated the placeholder. That page read
-(`public_title_info`) also titles every new download, from the worker.
+(`public_page_info`) also titles and dates every new download, from the worker.

 Run once per environment:
    docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
@@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in
    return None


-def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
-    """`info` to title a download from: itself, unless its title is a source's placeholder with
-    nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out,
-    metadata-only read of the page lends its text — the description, else its title — as the
-    description the adapter takes a placeholder's title from. Nothing better there: `info` as it was."""
+def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
+    """`info` to title and date a download from: itself, unless its title is a source's placeholder
+    with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one
+    signed-out, metadata-only read of the page lends its text — the description, else its title — as
+    the description the adapter takes a placeholder's title from, and its upload date when `info` has
+    none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since
+    was named 0000-00-00 in Season_0). Nothing better there: `info` as it was."""
    if not adapter.is_placeholder_title(adapter.title(info)):
        return info
-    public = extract_meta(url, adapter, asset_id)
-    text = public and (public.get("description") or public.get("title"))
-    return {**info, "description": text} if text else info
+    public = extract_meta(url, adapter, asset_id) or {}
+    lent: dict = {}
+    text = public.get("description") or public.get("title")
+    if text:
+        lent["description"] = text
+    if not info.get("upload_date") and public.get("upload_date"):
+        lent["upload_date"] = public["upload_date"]
+    return {**info, **lent} if lent else info


 def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
@@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s
    names that only repeated it. A name its owner gave (anything else) stays."""
    if not adapter.is_placeholder_title(a.title):
        return False
-    info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id)
+    info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id)
    named = normalize_title(adapter.title(info))
    if not named or adapter.is_placeholder_title(named):
        return False
@@ -591,15 +591,16 @@ def _download(
        # too. Same probe the upload ingest uses.
        vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
        # A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the
-        # page's text from one signed-out read; the download itself stays the signed-in one. The page
+        # page's text (and a missing upload date) from one signed-out read; the download itself stays
+        # the signed-in one. The page
        # yt-dlp read, not the link queued: a group post's link is rewritten to its video, and
        # signed out the post itself is the login wall.
-        titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id)
+        titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id)
        meta = storage.MediaMeta(
            video_id=info.get("id") or source_ref,
            title=normalize_title(adapter.title(titled)) or source_ref,
            uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
-            upload_date=_parse_upload_date(info.get("upload_date")),
+            upload_date=_parse_upload_date(titled.get("upload_date")),
            uniqueid_type=adapter.nfo_uniqueid_type(),
            duration_s=duration_s,
            description=info.get("description"),
@@ -20,6 +20,8 @@ from datetime import date
 from pathlib import Path
 from xml.sax.saxutils import escape

+from anyascii import anyascii
+
 from app import logs
 from app.downloads import mediakind

@@ -121,12 +123,14 @@ def files_root_problem(

 def _ascii_fold(text: str) -> str:
    """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
-    the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
-    no ASCII base)."""
+    the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters
+    (`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew,
+    Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still
+    not ASCII after that is stripped."""
    text = text.translate(_FOLD_MAP)
    text = unicodedata.normalize("NFKD", text)
    text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
-    return text.encode("ascii", "ignore").decode("ascii")
+    return anyascii(text).encode("ascii", "ignore").decode("ascii")


@dataclass
@@ -7,7 +7,8 @@
 # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
 # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403);
 # certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322);
-# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284).
+# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284);
+# anyascii 0.3.3 added (no dependencies of its own, so the rest of the lock is unchanged).
 APScheduler==3.11.3
 Authlib==1.9.1
 Mako==1.4.1
@@ -18,6 +19,7 @@ SQLAlchemy==2.0.52
 alembic==1.19.1
 annotated-doc==0.0.5
 annotated-types==0.8.0
+anyascii==0.3.3
 anyio==4.14.2
 argon2-cffi-bindings==25.1.0
 argon2-cffi==25.1.0
@@ -13,6 +13,10 @@ py3langid>=0.3,<1.0
 itsdangerous>=2.1,<3.0
 cryptography>=50.0.0,<51
 argon2-cffi>=23.1,<26
+# On-disk names are ASCII (storage._ascii_fold): a Hebrew, Cyrillic, Greek or CJK uploader name is
+# written in Latin letters instead of dropped whole into one shared "untitled" folder. Pure Python,
+# no dependencies of its own.
+anyascii>=0.3.3,<0.4
 # Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
 # does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
 # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
@@ -22,6 +22,9 @@ _PUBLIC = {
    "description": "sorry not helping\n\n#workhumor #officelife",
    "uploader": "Work dot exe",
 }
+# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919:
+# "20261113"), which the signed-in read no longer gives.
+_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"}


 def _fake_ydl(monkeypatch, outcome):
@@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
    seen = _fake_ydl(monkeypatch, _PUBLIC)
    fb = adapters.FacebookAdapter()

-    info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
+    info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)

    assert fb.title(info) == "sorry not helping"
    # One metadata-only read, and signed out: no cookie goes with it.
@@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
    assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url()


+def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch):
+    # Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one,
+    # 31 since none) — the signed-in read lost it with the title. The signed-out read still has it.
+    _fake_ydl(monkeypatch, _PUBLIC_DATED)
+
+    info = backfill.public_page_info(
+        {"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7
+    )
+
+    assert info["upload_date"] == "20261113"
+
+
+def test_a_date_the_signed_in_read_gave_stays(monkeypatch):
+    _fake_ydl(monkeypatch, _PUBLIC_DATED)
+
+    info = backfill.public_page_info(
+        {"title": "Facebook", "description": None, "upload_date": "20261001"},
+        _URL,
+        adapters.FacebookAdapter(),
+        7,
+    )
+
+    assert info["upload_date"] == "20261001"
+
+
 def test_a_real_title_reads_nothing_more(monkeypatch):
    seen = _fake_ydl(monkeypatch, _PUBLIC)
    info = {"title": "Funny Dark Humor Dad Joke!", "description": None}

-    assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
+    assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
    assert seen == []


@@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa
    seen = _fake_ydl(monkeypatch, _PUBLIC)
    info = {"title": "Video", "description": "Office jokes, part two"}

-    assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
+    assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
    assert seen == []


@@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch):
    _fake_ydl(monkeypatch, {"title": "Facebook", "description": None})
    fb = adapters.FacebookAdapter()

-    info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
+    info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)

    assert fb.title(info) == "Facebook"

@@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch):
    seen = _fake_ydl(monkeypatch, _PUBLIC)
    info = {"title": "Video", "description": None}

-    assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info
+    assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info
    assert seen == []


@@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
    db.add(job)
    db.commit()

-    read_page = backfill.public_title_info
+    read_page = backfill.public_page_info

    def renamed_while_reading(*args):
        with SessionLocal() as other:
@@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
            other.commit()
        return read_page(*args)

-    monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading)
+    monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading)
    monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None)

    assert backfill.run()["retitled"] == 1
@@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
    db, make_user, monkeypatch, tmp_path
 ):
    # The whole download path with yt-dlp and the media tools faked: the signed-in read gives only
-    # the placeholder, and the file and the download's name get the post's text.
-    seen = _fake_ydl(monkeypatch, _PUBLIC)
+    # the placeholder, and the file and the download's name get the post's text — and its date.
+    seen = _fake_ydl(monkeypatch, _PUBLIC_DATED)
    monkeypatch.setattr(settings, "files_root", str(tmp_path))
    monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
    monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
@@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
    assert db.get(DownloadJob, job.id).status == "done"
    assert db.get(MediaAsset, asset.id).title == "sorry not helping"
    assert db.get(DownloadJob, job.id).display_name == "sorry not helping"
+    done = db.get(MediaAsset, asset.id)
+    assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13"
+    # The date names the file and its season, not 0000-00-00 / Season_0.
+    assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_")
@@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces():
        assert _is_ascii_slug(out), out


-def test_sanitize_non_latin_falls_back_not_crashes():
-    # A title with no ASCII base collapses to the untitled fallback rather than an empty name.
-    assert storage.sanitize("日本語") == "untitled"
+def test_sanitize_writes_a_non_latin_name_in_latin_letters():
+    # Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
+    # folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
+    names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
+    slugs = [storage.sanitize(n) for n in names]
+    for name, slug in zip(names, slugs, strict=True):
+        assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
+    assert len(set(slugs)) == len(slugs), slugs  # two names, two folders
+    # The measured spellings (anyascii 0.3.3, the locked one).
+    assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
+
+
+def test_sanitize_falls_back_when_nothing_is_left():
+    # Emoji and symbols are dropped, and a name of nothing else still gets a folder.
+    assert storage.sanitize("🎉🎉") == "untitled"


 def test_display_filename_uses_underscores_and_ascii():
@@ -1,4 +1,12 @@
 [
+  {
+    "version": "0.119.2",
+    "date": "2026-11-14",
+    "new": [],
+    "fixed": [
+      "A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming \"untitled\"."
+    ]
+  },
  {
    "version": "0.119.1",
    "date": "2026-11-14",
@@ -1,4 +1,12 @@
 [
+  {
+    "version": "0.119.2",
+    "date": "2026-11-14",
+    "new": [],
+    "fixed": [
+      "A Facebook-letöltések újra megkapják a feltöltés dátumát, így a fájl és az évad neve azt mutatja, nem 0000-00-00-t. A más írású, például héber vagy cirill csatornanév a fájlnevekben latin betűkkel jelenik meg, nem „untitled” lesz."
+    ]
+  },
  {
    "version": "0.119.1",
    "date": "2026-11-14",
@@ -1 +1 @@
 .119.1
 .119.2