Compare commits

..
7 Commits
Author SHA1 Message Date
peter 3bb71a4bc3 release: 0.119.2 (bug/fb-date-translit) 2026-11-14 01:34:13 +01:00
peter 87ab33b03c release: 0.119.2 2026-11-14 01:21:11 +01:00
peter 452ff042ff fix(downloads): Facebook files get their upload date, non-Latin names their letters
Since 2026-10-28 the signed-in Facebook read gives no upload date, so
every Facebook file since was named 0000-00-00 in Season_0 (31 on prod,
none before). The signed-out read that already titles these downloads
carries the date, so it now lends that too when the signed-in read has
none (backfill.public_title_info is now public_page_info).

On-disk names dropped every letter outside the Latin script, so a Hebrew,
Cyrillic, Greek or CJK uploader went to one shared "untitled" folder
(prod asset 503). anyascii now writes them in Latin letters
(Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
2026-11-14 01:15:17 +01:00
peter 25198a936f release: 0.119.1 (bug/fb-share-group-post) 2026-11-14 00:49:24 +01:00
peter 9d979a5202 release: 0.119.1 2026-11-14 00:36:34 +01:00
peter fb3aca4c9c fix(downloads): a shared or feed-opened Facebook group post downloads its own video
A Facebook share link (/share/v/…) redirects to a group post, but only the
permalink form went through the post-page read, so yt-dlp followed the
share link itself and downloaded an ad from the signed-in group feed
(prod job 471). The link of a post opened from the group's feed
(/groups/<g>/?multi_permalinks=<id>) was not recognised at all, and yt-dlp
answers it with "Unsupported URL" (prod job 472).

With a cookie, a share link is now followed in the same request that reads
the post page, and a feed link is read as the post it names. Without one,
the feed link goes to yt-dlp in the permalink form it knows. A post page's
"no single video" verdict now names the post for a share link too.
2026-11-14 00:29:59 +01:00
peter 4f78877201 test: the app's clock never goes back in the test suite
The docker VM the suite runs in steps its wall clock back now and then (measured: 2 steps in 20 s, up to 0.5 s), so rows a test made one after another could read out of order (a files list sorted newest first, a scheduler's last runs). Every app module that took datetime from the datetime module reads a clock that never repeats or goes back; the scheduler test's own fixture for this goes.
2026-11-13 03:02:00 +01:00
17 changed files with 313 additions and 63 deletions
+12
View File
@@ -7,6 +7,18 @@ edit the release notes and re-run the generator.
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
## 0.119.2 — 2026-11-14
### Fixed
- A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming "untitled".
## 0.119.1 — 2026-11-14
### Fixed
- A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with "Site not supported".
## 0.119.0 — 2026-11-13
### New
+1 -1
View File
@@ -1 +1 @@
0.119.0
0.119.2
+45 -6
View File
@@ -12,7 +12,7 @@ Resolution works both before extraction (from source_kind / URL host) and after
"""
import re
from urllib.parse import urlparse
from urllib.parse import parse_qs, urlparse
from yt_dlp.networking.impersonate import ImpersonateTarget
@@ -183,10 +183,23 @@ class FacebookAdapter(SourceAdapter):
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
post_id = facebook_group_post_id(url)
if post_id is None or not ydl_opts.get("cookiefile"):
if not ydl_opts.get("cookiefile"):
# yt-dlp knows the permalink form and answers the feed's form with "Unsupported URL".
if post_id is not None and not _FB_GROUP_POST_PATH.match(urlparse(url).path):
return _group_permalink(url, post_id)
return url
# A share link is a post only once followed: handed to yt-dlp, it was followed to the group
# page and an ad from the signed-in feed came down (prod job 471). One request does both — the
# redirect lands on the post page, which is then read as any post link's.
if post_id is None and not _is_share_link(url):
return url
with egress.open_ydl(ydl_opts) as ydl:
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
response = ydl.urlopen(url)
if post_id is None:
post_id = facebook_group_post_id(response.url)
if post_id is None:
return url # a shared reel or video: yt-dlp follows the link as it always has
raw = response.read(self._POST_PAGE_MAX_BYTES)
page = raw.decode("utf-8", "replace")
# Measured: a refused or expired cookie gets the login page (~450 KB, a login
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
@@ -214,7 +227,7 @@ class FacebookAdapter(SourceAdapter):
# signed-out read titles the page with that same text behind the engagement counts, cut short.
# Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives
# "Facebook" and NO description, while a signed-out read of the same reel still carries the text —
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`).
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`).
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
def is_placeholder_title(self, title: str | None) -> bool:
@@ -246,6 +259,7 @@ class FacebookAdapter(SourceAdapter):
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
_FB_GROUP_PATH = re.compile(r"^/groups/[^/]+/?$")
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
@@ -256,12 +270,37 @@ def _quoted(post_id: str) -> str:
def facebook_group_post_id(url: str) -> str | None:
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
"""The post id of a Facebook group post link, else None: `/groups/<group>/permalink|posts/<id>/`,
or a post opened from the group's feed, `/groups/<group>/?multi_permalinks=<id>` (one id only —
with several, which one was meant is not ours to guess)."""
parsed = urlparse(url)
if not _host_is(parsed.netloc.lower(), "facebook.com"):
return None
match = _FB_GROUP_POST_PATH.match(parsed.path)
return match.group(1) if match else None
if match:
return match.group(1)
if not _FB_GROUP_PATH.match(parsed.path):
return None
ids = parse_qs(parsed.query).get("multi_permalinks", [])
return ids[0] if len(ids) == 1 and ids[0].isdigit() else None
def _group_permalink(url: str, post_id: str) -> str:
"""The permalink form of a group post link — the one form yt-dlp itself accepts."""
group = urlparse(url).path.split("/")[2]
return f"https://www.facebook.com/groups/{group}/permalink/{post_id}/"
def facebook_post_link(url: str) -> bool:
"""A link that `resolve_url` may read as a group post: a post link, or a share link (which may
lead to one) — what the failure catalog accepts for the post page's own verdicts."""
return facebook_group_post_id(url) is not None or _is_share_link(url)
def _is_share_link(url: str) -> bool:
# facebook.com/share/<code>/ and /share/v|r|p/<code>/ — a redirect to the shared object.
parsed = urlparse(url)
return _host_is(parsed.netloc.lower(), "facebook.com") and parsed.path.startswith("/share/")
def facebook_post_video_id(page: str, post_id: str) -> str | None:
+17 -10
View File
@@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann
URL may have expired — those just get a locally-generated poster instead). A Facebook download
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
page, and so is each download name that only repeated the placeholder. That page read
(`public_title_info`) also titles every new download, from the worker.
(`public_page_info`) also titles and dates every new download, from the worker.
Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
@@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in
return None
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
"""`info` to title a download from: itself, unless its title is a source's placeholder with
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out,
metadata-only read of the page lends its text — the description, else its title — as the
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was."""
def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
"""`info` to title and date a download from: itself, unless its title is a source's placeholder
with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one
signed-out, metadata-only read of the page lends its text — the description, else its title — as
the description the adapter takes a placeholder's title from, and its upload date when `info` has
none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since
was named 0000-00-00 in Season_0). Nothing better there: `info` as it was."""
if not adapter.is_placeholder_title(adapter.title(info)):
return info
public = extract_meta(url, adapter, asset_id)
text = public and (public.get("description") or public.get("title"))
return {**info, "description": text} if text else info
public = extract_meta(url, adapter, asset_id) or {}
lent: dict = {}
text = public.get("description") or public.get("title")
if text:
lent["description"] = text
if not info.get("upload_date") and public.get("upload_date"):
lent["upload_date"] = public["upload_date"]
return {**info, **lent} if lent else info
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
@@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s
names that only repeated it. A name its owner gave (anything else) stays."""
if not adapter.is_placeholder_title(a.title):
return False
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id)
info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id)
named = normalize_title(adapter.title(info))
if not named or adapter.is_placeholder_title(named):
return False
+4 -2
View File
@@ -32,6 +32,7 @@ from app.downloads.adapters import (
POST_NOT_SHOWN_MESSAGE,
POST_WITHOUT_VIDEO_MESSAGE,
facebook_group_post_id,
facebook_post_link,
)
from app.downloads.credentials import host_key_for_url
@@ -227,13 +228,14 @@ _CATALOG: tuple[_Entry, ...] = (
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
# that video, not the post — the category says it. Only for group post links (the adapter's own
# test of one): a reel with no formats is a different failure.
# test of one): a reel with no formats is a different failure. The adapter's own text also comes
# from a share link that led to a post, so its test takes those too.
_Entry(
host="facebook",
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
code="format_unavailable",
detail="facebook.group_post_video",
link=facebook_group_post_id,
link=facebook_post_link,
),
_Entry(
host="facebook",
+4 -3
View File
@@ -591,15 +591,16 @@ def _download(
# too. Same probe the upload ingest uses.
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
# A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the
# page's text from one signed-out read; the download itself stays the signed-in one. The page
# page's text (and a missing upload date) from one signed-out read; the download itself stays
# the signed-in one. The page
# yt-dlp read, not the link queued: a group post's link is rewritten to its video, and
# signed out the post itself is the login wall.
titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id)
titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id)
meta = storage.MediaMeta(
video_id=info.get("id") or source_ref,
title=normalize_title(adapter.title(titled)) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")),
upload_date=_parse_upload_date(titled.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=duration_s,
description=info.get("description"),
+7 -3
View File
@@ -20,6 +20,8 @@ from datetime import date
from pathlib import Path
from xml.sax.saxutils import escape
from anyascii import anyascii
from app import logs
from app.downloads import mediakind
@@ -121,12 +123,14 @@ def files_root_problem(
def _ascii_fold(text: str) -> str:
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
no ASCII base)."""
the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters
(`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew,
Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still
not ASCII after that is stripped."""
text = text.translate(_FOLD_MAP)
text = unicodedata.normalize("NFKD", text)
text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
return text.encode("ascii", "ignore").decode("ascii")
return anyascii(text).encode("ascii", "ignore").decode("ascii")
@dataclass
+3 -1
View File
@@ -7,7 +7,8 @@
# `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403);
# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322);
# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284).
# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284);
# anyascii 0.3.3 added (no dependencies of its own, so the rest of the lock is unchanged).
APScheduler==3.11.3
Authlib==1.9.1
Mako==1.4.1
@@ -18,6 +19,7 @@ SQLAlchemy==2.0.52
alembic==1.19.1
annotated-doc==0.0.5
annotated-types==0.8.0
anyascii==0.3.3
anyio==4.14.2
argon2-cffi-bindings==25.1.0
argon2-cffi==25.1.0
+4
View File
@@ -13,6 +13,10 @@ py3langid>=0.3,<1.0
itsdangerous>=2.1,<3.0
cryptography>=50.0.0,<51
argon2-cffi>=23.1,<26
# On-disk names are ASCII (storage._ascii_fold): a Hebrew, Cyrillic, Greek or CJK uploader name is
# written in Latin letters instead of dropped whole into one shared "untitled" folder. Pure Python,
# no dependencies of its own.
anyascii>=0.3.3,<0.4
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
+36
View File
@@ -13,6 +13,7 @@ that the migration chain applies cleanly. Each test then gets a clean database v
tests that COMMIT (the whole point of this lane — transaction-ownership behaviour) don't leak.
"""
import datetime as _dt
import os
import shutil
from urllib.parse import quote
@@ -317,6 +318,41 @@ def _reset_rate_limiters():
obj.reset()
class _NeverBack(_dt.datetime):
"""The app's clock in tests: the wall clock, but never repeating and never going back — each
reading at least a microsecond after the last one in this process."""
last: _dt.datetime | None = None
clock: type[_dt.datetime] = (
_dt.datetime
) # what it reads; a probe can put a stepping one under it
@classmethod
def now(cls, tz=None):
t = cls.clock.now(tz)
last = cls.last
if last is not None and (last.tzinfo is None) == (t.tzinfo is None) and t <= last:
t = last + _dt.timedelta(microseconds=1)
cls.last = t
return t
@pytest.fixture(autouse=True)
def _clock_never_goes_back(monkeypatch):
"""Rows a test makes one after another are stamped in that order. The wall clock of the docker
VM the suite runs in steps BACK now and then (measured: 2 steps in 20 s, up to 0.5 s), and a
test that makes three files and reads them newest first then got them out of order. Every loaded
`app.*` module that took `datetime` from the datetime module reads this clock instead (a sweep,
as the limiter reset below does, so a new module is covered the day it lands). A database
default (`server_default=func.now()`) is Postgres's clock and is not covered: a test that orders
such rows made within a second of each other gives them distinct times itself."""
import sys
for name, mod in list(sys.modules.items()):
if (name == "app" or name.startswith("app.")) and vars(mod).get("datetime") is _dt.datetime:
monkeypatch.setattr(mod, "datetime", _NeverBack)
@pytest.fixture(autouse=True)
def _reset_log_erasures():
"""`logs._erased` is module-level and remembers an erased account for an hour of monotonic time
+6
View File
@@ -218,6 +218,12 @@ def test_our_post_without_a_video_verdict_names_the_group_link():
"format_unavailable",
"facebook.group_post_video",
)
# The share link a post was reached by (prod job 471's kind) names the post too.
share = "https://www.facebook.com/share/v/19bCAHKyrd/"
assert dlerrors.diagnose(msg, url=share, cookie_attached=True) == (
"format_unavailable",
"facebook.group_post_video",
)
assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None)
@@ -27,6 +27,18 @@ _GROUP_LINK = f"https://www.facebook.com/groups/983554479435596/permalink/{_POST
_POST_VIDEO = "1408015447857753"
_FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video
_CHROME = ImpersonateTarget.from_str("chrome")
# The two other ways a group post reaches us (prod jobs 471 and 472, the same post): Facebook's "Share"
# link, which redirects to the post's permalink (measured), and the link of a post opened from the
# group's feed, which yt-dlp does not know at all ("Unsupported URL").
_SHARE_LINK = "https://www.facebook.com/share/v/19bCAHKyrd/"
_LANDED = (
f"https://www.facebook.com/groups/it.humor.and.memes/permalink/{_POST}/"
"?rdid=y9HAMntF7aCoAKZE&share_url=https%3A%2F%2Fwww.facebook.com%2Fshare%2Fv%2F19bCAHKyrd%2F"
)
_MULTI_LINK = (
f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST}"
"&hoisted_section_header_type=recently_seen&__cft__[0]=AZgK&__tn__=%2CO%2CP-R"
)
def _video_node(video_id: str) -> str:
@@ -86,6 +98,9 @@ def ydl(monkeypatch):
body = web["page"].encode()
class Response:
# Where the request ended up after its redirects (yt-dlp's `Response.url`).
url = web.get("landed") or web["opened"][-1]
def read(self, amt=None):
web["reads"].append(amt)
return body
@@ -228,12 +243,79 @@ def test_without_a_cookie_the_group_post_link_goes_to_ytdlp_unchanged(ydl, tmp_p
assert web["extracted"] == [_GROUP_LINK]
def test_a_share_link_to_a_group_post_downloads_the_posts_own_video(ydl, tmp_path):
# Prod job 471: handed the share link, yt-dlp followed it to the group page and downloaded an
# ad from the signed-in feed. Followed here first, it is the group post it leads to.
_, outcomes, web = ydl
web["page"], web["landed"] = _page(_POST_VIDEO), _LANDED
outcomes += [{"id": _POST_VIDEO}]
assert _run(tmp_path, "/cookies.txt", _SHARE_LINK) == {"id": _POST_VIDEO}
assert web["opened"] == [_SHARE_LINK] # one request: the redirect and the post page at once
assert web["reads"] == [adapters.FacebookAdapter._POST_PAGE_MAX_BYTES]
assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"]
def test_a_share_link_to_a_reel_goes_to_ytdlp_unchanged(ydl, tmp_path):
_, outcomes, web = ydl
web["landed"] = _URL
outcomes += [{"id": "806379462330494"}]
_run(tmp_path, "/cookies.txt", _SHARE_LINK)
assert web["reads"] == [] # not a post: its page is not read
assert web["extracted"] == [_SHARE_LINK] # the path that already downloads shared reels
def test_a_share_link_that_fails_as_a_group_post_says_why(ydl, tmp_path):
_, _, web = ydl
web["page"], web["landed"] = _page(), _LANDED
with pytest.raises(RuntimeError) as exc:
_run(tmp_path, "/cookies.txt", _SHARE_LINK)
assert web["extracted"] == []
# The job's own link is the share link: the verdict still names the post.
assert dlerrors.diagnose(str(exc.value), url=_SHARE_LINK, cookie_attached=True) == (
"format_unavailable",
"facebook.group_post_video",
)
def test_a_feed_opened_group_post_downloads_the_posts_own_video(ydl, tmp_path):
# Prod job 472: yt-dlp answers this link with "Unsupported URL".
_, outcomes, web = ydl
web["page"] = _page(_POST_VIDEO)
outcomes += [{"id": _POST_VIDEO}]
assert _run(tmp_path, "/cookies.txt", _MULTI_LINK) == {"id": _POST_VIDEO}
assert web["opened"] == [_MULTI_LINK]
assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"]
def test_without_a_cookie_a_feed_opened_group_post_goes_as_its_permalink(ydl, tmp_path):
# Signed out there is no page to read, but yt-dlp knows the permalink form and not this one.
_, outcomes, web = ydl
outcomes += [{"id": _POST_VIDEO}]
_run(tmp_path, None, _MULTI_LINK)
assert web["opened"] == []
assert web["extracted"] == [
f"https://www.facebook.com/groups/983554479435596/permalink/{_POST}/"
]
def test_the_group_post_link_forms_are_recognised():
for url in (
_GROUP_LINK,
f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/",
f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?",
f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}",
_MULTI_LINK,
f"https://www.facebook.com/groups/983554479435596?multi_permalinks={_POST}",
):
assert adapters.facebook_group_post_id(url) == _POST, url
for url in (
@@ -241,6 +323,10 @@ def test_the_group_post_link_forms_are_recognised():
f"https://www.facebook.com/watch/?v={_POST_VIDEO}",
f"https://facebook.com.evil/groups/x/permalink/{_POST}/",
"https://www.facebook.com/groups/983554479435596/",
# More than one post in the link: which one was meant is not ours to guess.
f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST},{_POST_VIDEO}",
f"https://www.facebook.com/groups/983554479435596/members/?multi_permalinks={_POST}",
_SHARE_LINK, # a share link is a post only once it is followed (`resolve_url`)
):
assert adapters.facebook_group_post_id(url) is None, url
assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK
+41 -9
View File
@@ -22,6 +22,9 @@ _PUBLIC = {
"description": "sorry not helping\n\n#workhumor #officelife",
"uploader": "Work dot exe",
}
# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919:
# "20261113"), which the signed-in read no longer gives.
_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"}
def _fake_ydl(monkeypatch, outcome):
@@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
seen = _fake_ydl(monkeypatch, _PUBLIC)
fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "sorry not helping"
# One metadata-only read, and signed out: no cookie goes with it.
@@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url()
def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch):
# Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one,
# 31 since none) — the signed-in read lost it with the title. The signed-out read still has it.
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7
)
assert info["upload_date"] == "20261113"
def test_a_date_the_signed_in_read_gave_stays(monkeypatch):
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None, "upload_date": "20261001"},
_URL,
adapters.FacebookAdapter(),
7,
)
assert info["upload_date"] == "20261001"
def test_a_real_title_reads_nothing_more(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Funny Dark Humor Dad Joke!", "description": None}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == []
@@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": "Office jokes, part two"}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == []
@@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch):
_fake_ydl(monkeypatch, {"title": "Facebook", "description": None})
fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7)
info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "Facebook"
@@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": None}
assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert seen == []
@@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
db.add(job)
db.commit()
read_page = backfill.public_title_info
read_page = backfill.public_page_info
def renamed_while_reading(*args):
with SessionLocal() as other:
@@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
other.commit()
return read_page(*args)
monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading)
monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading)
monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None)
assert backfill.run()["retitled"] == 1
@@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
db, make_user, monkeypatch, tmp_path
):
# The whole download path with yt-dlp and the media tools faked: the signed-in read gives only
# the placeholder, and the file and the download's name get the post's text.
seen = _fake_ydl(monkeypatch, _PUBLIC)
# the placeholder, and the file and the download's name get the post's text — and its date.
seen = _fake_ydl(monkeypatch, _PUBLIC_DATED)
monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
@@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
assert db.get(DownloadJob, job.id).status == "done"
assert db.get(MediaAsset, asset.id).title == "sorry not helping"
assert db.get(DownloadJob, job.id).display_name == "sorry not helping"
done = db.get(MediaAsset, asset.id)
assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13"
# The date names the file and its season, not 0000-00-00 / Season_0.
assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_")
+15 -3
View File
@@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces():
assert _is_ascii_slug(out), out
def test_sanitize_non_latin_falls_back_not_crashes():
# A title with no ASCII base collapses to the untitled fallback rather than an empty name.
assert storage.sanitize("日本語") == "untitled"
def test_sanitize_writes_a_non_latin_name_in_latin_letters():
# Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
# folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
slugs = [storage.sanitize(n) for n in names]
for name, slug in zip(names, slugs, strict=True):
assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
assert len(set(slugs)) == len(slugs), slugs # two names, two folders
# The measured spellings (anyascii 0.3.3, the locked one).
assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
def test_sanitize_falls_back_when_nothing_is_left():
# Emoji and symbols are dropped, and a name of nothing else still gets a folder.
assert storage.sanitize("🎉🎉") == "untitled"
def test_display_filename_uses_underscores_and_ascii():
-25
View File
@@ -22,31 +22,6 @@ from app.runstatus import RunStatus, RunTrigger
_JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs
class _StrictlyLater(datetime):
"""A clock that never repeats or goes back: each reading at least a microsecond after the last."""
last: datetime | None = None
clock: type[datetime] = datetime
@classmethod
def now(cls, tz=None):
t = cls.clock.now(tz)
if cls.last is not None and t <= cls.last:
t = cls.last + timedelta(microseconds=1)
cls.last = t
return t
@pytest.fixture(autouse=True)
def _runs_start_in_order(monkeypatch):
"""Runs started in a row start in that order. These tests open several runs within microseconds
and read them back newest first, by start time; the wall clock of a container on a VM can step
back between two readings (seen: the third of three runs sorted before the second)."""
_StrictlyLater.last = None
_StrictlyLater.clock = jobruns.datetime
monkeypatch.setattr(jobruns, "datetime", _StrictlyLater)
def _release():
scheduler._record(_JOB, running=False)
@@ -1,4 +1,20 @@
[
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming \"untitled\"."
]
},
{
"version": "0.119.1",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with \"Site not supported\"."
]
},
{
"version": "0.119.0",
"date": "2026-11-13",
@@ -1,4 +1,20 @@
[
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook-letöltések újra megkapják a feltöltés dátumát, így a fájl és az évad neve azt mutatja, nem 0000-00-00-t. A más írású, például héber vagy cirill csatornanév a fájlnevekben latin betűkkel jelenik meg, nem „untitled” lesz."
]
},
{
"version": "0.119.1",
"date": "2026-11-14",
"new": [],
"fixed": [
"Egy Facebook-csoport posztja a Megosztás linkjével vagy a csoport hírfolyamából megnyitva is a poszt saját videóját tölti le. Eddig a Megosztás linkje a hírfolyam egy másik videóját is hozhatta, a hírfolyam linkje pedig „Nem támogatott oldal” hibát adott."
]
},
{
"version": "0.119.0",
"date": "2026-11-13",