Compare commits

..
3 Commits
Author SHA1 Message Date
peter 3bb71a4bc3 release: 0.119.2 (bug/fb-date-translit) 2026-11-14 01:34:13 +01:00
peter 87ab33b03c release: 0.119.2 2026-11-14 01:21:11 +01:00
peter 452ff042ff fix(downloads): Facebook files get their upload date, non-Latin names their letters
Since 2026-10-28 the signed-in Facebook read gives no upload date, so
every Facebook file since was named 0000-00-00 in Season_0 (31 on prod,
none before). The signed-out read that already titles these downloads
carries the date, so it now lends that too when the signed-in read has
none (backfill.public_title_info is now public_page_info).

On-disk names dropped every letter outside the Latin script, so a Hebrew,
Cyrillic, Greek or CJK uploader went to one shared "untitled" folder
(prod asset 503). anyascii now writes them in Latin letters
(Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
2026-11-14 01:15:17 +01:00
12 changed files with 115 additions and 31 deletions
+6
View File
@@ -7,6 +7,12 @@ edit the release notes and re-run the generator.
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
## 0.119.2 — 2026-11-14
### Fixed
- A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming "untitled".
## 0.119.1 — 2026-11-14 ## 0.119.1 — 2026-11-14
### Fixed ### Fixed
+1 -1
View File
@@ -1 +1 @@
0.119.1 0.119.2
+1 -1
View File
@@ -227,7 +227,7 @@ class FacebookAdapter(SourceAdapter):
# signed-out read titles the page with that same text behind the engagement counts, cut short. # signed-out read titles the page with that same text behind the engagement counts, cut short.
# Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives # Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives
# "Facebook" and NO description, while a signed-out read of the same reel still carries the text — # "Facebook" and NO description, while a signed-out read of the same reel still carries the text —
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`). # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`).
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"}) _PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
def is_placeholder_title(self, title: str | None) -> bool: def is_placeholder_title(self, title: str | None) -> bool:
+17 -10
View File
@@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann
URL may have expired — those just get a locally-generated poster instead). A Facebook download URL may have expired — those just get a locally-generated poster instead). A Facebook download
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
page, and so is each download name that only repeated the placeholder. That page read page, and so is each download name that only repeated the placeholder. That page read
(`public_title_info`) also titles every new download, from the worker. (`public_page_info`) also titles and dates every new download, from the worker.
Run once per environment: Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())" docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
@@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in
return None return None
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict: def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
"""`info` to title a download from: itself, unless its title is a source's placeholder with """`info` to title and date a download from: itself, unless its title is a source's placeholder
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out, with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one
metadata-only read of the page lends its text — the description, else its title — as the signed-out, metadata-only read of the page lends its text — the description, else its title — as
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was.""" the description the adapter takes a placeholder's title from, and its upload date when `info` has
none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since
was named 0000-00-00 in Season_0). Nothing better there: `info` as it was."""
if not adapter.is_placeholder_title(adapter.title(info)): if not adapter.is_placeholder_title(adapter.title(info)):
return info return info
public = extract_meta(url, adapter, asset_id) public = extract_meta(url, adapter, asset_id) or {}
text = public and (public.get("description") or public.get("title")) lent: dict = {}
return {**info, "description": text} if text else info text = public.get("description") or public.get("title")
if text:
lent["description"] = text
if not info.get("upload_date") and public.get("upload_date"):
lent["upload_date"] = public["upload_date"]
return {**info, **lent} if lent else info
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool: def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
@@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s
names that only repeated it. A name its owner gave (anything else) stays.""" names that only repeated it. A name its owner gave (anything else) stays."""
if not adapter.is_placeholder_title(a.title): if not adapter.is_placeholder_title(a.title):
return False return False
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id) info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id)
named = normalize_title(adapter.title(info)) named = normalize_title(adapter.title(info))
if not named or adapter.is_placeholder_title(named): if not named or adapter.is_placeholder_title(named):
return False return False
+4 -3
View File
@@ -591,15 +591,16 @@ def _download(
# too. Same probe the upload ingest uses. # too. Same probe the upload ingest uses.
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info) vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
# A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the # A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the
# page's text from one signed-out read; the download itself stays the signed-in one. The page # page's text (and a missing upload date) from one signed-out read; the download itself stays
# the signed-in one. The page
# yt-dlp read, not the link queued: a group post's link is rewritten to its video, and # yt-dlp read, not the link queued: a group post's link is rewritten to its video, and
# signed out the post itself is the login wall. # signed out the post itself is the login wall.
titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id) titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id)
meta = storage.MediaMeta( meta = storage.MediaMeta(
video_id=info.get("id") or source_ref, video_id=info.get("id") or source_ref,
title=normalize_title(adapter.title(titled)) or source_ref, title=normalize_title(adapter.title(titled)) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel", uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")), upload_date=_parse_upload_date(titled.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(), uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=duration_s, duration_s=duration_s,
description=info.get("description"), description=info.get("description"),
+7 -3
View File
@@ -20,6 +20,8 @@ from datetime import date
from pathlib import Path from pathlib import Path
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
from anyascii import anyascii
from app import logs from app import logs
from app.downloads import mediakind from app.downloads import mediakind
@@ -121,12 +123,14 @@ def files_root_problem(
def _ascii_fold(text: str) -> str: def _ascii_fold(text: str) -> str:
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters
no ASCII base).""" (`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew,
Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still
not ASCII after that is stripped."""
text = text.translate(_FOLD_MAP) text = text.translate(_FOLD_MAP)
text = unicodedata.normalize("NFKD", text) text = unicodedata.normalize("NFKD", text)
text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn") text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
return text.encode("ascii", "ignore").decode("ascii") return anyascii(text).encode("ascii", "ignore").decode("ascii")
@dataclass @dataclass
+3 -1
View File
@@ -7,7 +7,8 @@
# `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403); # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403);
# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322); # certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322);
# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284). # Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284);
# anyascii 0.3.3 added (no dependencies of its own, so the rest of the lock is unchanged).
APScheduler==3.11.3 APScheduler==3.11.3
Authlib==1.9.1 Authlib==1.9.1
Mako==1.4.1 Mako==1.4.1
@@ -18,6 +19,7 @@ SQLAlchemy==2.0.52
alembic==1.19.1 alembic==1.19.1
annotated-doc==0.0.5 annotated-doc==0.0.5
annotated-types==0.8.0 annotated-types==0.8.0
anyascii==0.3.3
anyio==4.14.2 anyio==4.14.2
argon2-cffi-bindings==25.1.0 argon2-cffi-bindings==25.1.0
argon2-cffi==25.1.0 argon2-cffi==25.1.0
+4
View File
@@ -13,6 +13,10 @@ py3langid>=0.3,<1.0
itsdangerous>=2.1,<3.0 itsdangerous>=2.1,<3.0
cryptography>=50.0.0,<51 cryptography>=50.0.0,<51
argon2-cffi>=23.1,<26 argon2-cffi>=23.1,<26
# On-disk names are ASCII (storage._ascii_fold): a Hebrew, Cyrillic, Greek or CJK uploader name is
# written in Latin letters instead of dropped whole into one shared "untitled" folder. Pure Python,
# no dependencies of its own.
anyascii>=0.3.3,<0.4
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile) # Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver # does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
+41 -9
View File
@@ -22,6 +22,9 @@ _PUBLIC = {
"description": "sorry not helping\n\n#workhumor #officelife", "description": "sorry not helping\n\n#workhumor #officelife",
"uploader": "Work dot exe", "uploader": "Work dot exe",
} }
# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919:
# "20261113"), which the signed-in read no longer gives.
_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"}
def _fake_ydl(monkeypatch, outcome): def _fake_ydl(monkeypatch, outcome):
@@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
fb = adapters.FacebookAdapter() fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "sorry not helping" assert fb.title(info) == "sorry not helping"
# One metadata-only read, and signed out: no cookie goes with it. # One metadata-only read, and signed out: no cookie goes with it.
@@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url() assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url()
def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch):
# Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one,
# 31 since none) — the signed-in read lost it with the title. The signed-out read still has it.
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7
)
assert info["upload_date"] == "20261113"
def test_a_date_the_signed_in_read_gave_stays(monkeypatch):
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None, "upload_date": "20261001"},
_URL,
adapters.FacebookAdapter(),
7,
)
assert info["upload_date"] == "20261001"
def test_a_real_title_reads_nothing_more(monkeypatch): def test_a_real_title_reads_nothing_more(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Funny Dark Humor Dad Joke!", "description": None} info = {"title": "Funny Dark Humor Dad Joke!", "description": None}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": "Office jokes, part two"} info = {"title": "Video", "description": "Office jokes, part two"}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch):
_fake_ydl(monkeypatch, {"title": "Facebook", "description": None}) _fake_ydl(monkeypatch, {"title": "Facebook", "description": None})
fb = adapters.FacebookAdapter() fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "Facebook" assert fb.title(info) == "Facebook"
@@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": None} info = {"title": "Video", "description": None}
assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
db.add(job) db.add(job)
db.commit() db.commit()
read_page = backfill.public_title_info read_page = backfill.public_page_info
def renamed_while_reading(*args): def renamed_while_reading(*args):
with SessionLocal() as other: with SessionLocal() as other:
@@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
other.commit() other.commit()
return read_page(*args) return read_page(*args)
monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading) monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading)
monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None) monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None)
assert backfill.run()["retitled"] == 1 assert backfill.run()["retitled"] == 1
@@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
db, make_user, monkeypatch, tmp_path db, make_user, monkeypatch, tmp_path
): ):
# The whole download path with yt-dlp and the media tools faked: the signed-in read gives only # The whole download path with yt-dlp and the media tools faked: the signed-in read gives only
# the placeholder, and the file and the download's name get the post's text. # the placeholder, and the file and the download's name get the post's text — and its date.
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC_DATED)
monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced) monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
@@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
assert db.get(DownloadJob, job.id).status == "done" assert db.get(DownloadJob, job.id).status == "done"
assert db.get(MediaAsset, asset.id).title == "sorry not helping" assert db.get(MediaAsset, asset.id).title == "sorry not helping"
assert db.get(DownloadJob, job.id).display_name == "sorry not helping" assert db.get(DownloadJob, job.id).display_name == "sorry not helping"
done = db.get(MediaAsset, asset.id)
assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13"
# The date names the file and its season, not 0000-00-00 / Season_0.
assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_")
+15 -3
View File
@@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces():
assert _is_ascii_slug(out), out assert _is_ascii_slug(out), out
def test_sanitize_non_latin_falls_back_not_crashes(): def test_sanitize_writes_a_non_latin_name_in_latin_letters():
# A title with no ASCII base collapses to the untitled fallback rather than an empty name. # Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
assert storage.sanitize("日本語") == "untitled" # folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
slugs = [storage.sanitize(n) for n in names]
for name, slug in zip(names, slugs, strict=True):
assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
assert len(set(slugs)) == len(slugs), slugs # two names, two folders
# The measured spellings (anyascii 0.3.3, the locked one).
assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
def test_sanitize_falls_back_when_nothing_is_left():
# Emoji and symbols are dropped, and a name of nothing else still gets a folder.
assert storage.sanitize("🎉🎉") == "untitled"
def test_display_filename_uses_underscores_and_ascii(): def test_display_filename_uses_underscores_and_ascii():
@@ -1,4 +1,12 @@
[ [
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming \"untitled\"."
]
},
{ {
"version": "0.119.1", "version": "0.119.1",
"date": "2026-11-14", "date": "2026-11-14",
@@ -1,4 +1,12 @@
[ [
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook-letöltések újra megkapják a feltöltés dátumát, így a fájl és az évad neve azt mutatja, nem 0000-00-00-t. A más írású, például héber vagy cirill csatornanév a fájlnevekben latin betűkkel jelenik meg, nem „untitled” lesz."
]
},
{ {
"version": "0.119.1", "version": "0.119.1",
"date": "2026-11-14", "date": "2026-11-14",