Compare commits

..
7 Commits
Author SHA1 Message Date
peter 3bb71a4bc3 release: 0.119.2 (bug/fb-date-translit) 2026-11-14 01:34:13 +01:00
peter 87ab33b03c release: 0.119.2 2026-11-14 01:21:11 +01:00
peter 452ff042ff fix(downloads): Facebook files get their upload date, non-Latin names their letters
Since 2026-10-28 the signed-in Facebook read gives no upload date, so
every Facebook file since was named 0000-00-00 in Season_0 (31 on prod,
none before). The signed-out read that already titles these downloads
carries the date, so it now lends that too when the signed-in read has
none (backfill.public_title_info is now public_page_info).

On-disk names dropped every letter outside the Latin script, so a Hebrew,
Cyrillic, Greek or CJK uploader went to one shared "untitled" folder
(prod asset 503). anyascii now writes them in Latin letters
(Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
2026-11-14 01:15:17 +01:00
peter 25198a936f release: 0.119.1 (bug/fb-share-group-post) 2026-11-14 00:49:24 +01:00
peter 9d979a5202 release: 0.119.1 2026-11-14 00:36:34 +01:00
peter fb3aca4c9c fix(downloads): a shared or feed-opened Facebook group post downloads its own video
A Facebook share link (/share/v/…) redirects to a group post, but only the
permalink form went through the post-page read, so yt-dlp followed the
share link itself and downloaded an ad from the signed-in group feed
(prod job 471). The link of a post opened from the group's feed
(/groups/<g>/?multi_permalinks=<id>) was not recognised at all, and yt-dlp
answers it with "Unsupported URL" (prod job 472).

With a cookie, a share link is now followed in the same request that reads
the post page, and a feed link is read as the post it names. Without one,
the feed link goes to yt-dlp in the permalink form it knows. A post page's
"no single video" verdict now names the post for a share link too.
2026-11-14 00:29:59 +01:00
peter 4f78877201 test: the app's clock never goes back in the test suite
The docker VM the suite runs in steps its wall clock back now and then (measured: 2 steps in 20 s, up to 0.5 s), so rows a test made one after another could read out of order (a files list sorted newest first, a scheduler's last runs). Every app module that took datetime from the datetime module reads a clock that never repeats or goes back; the scheduler test's own fixture for this goes.
2026-11-13 03:02:00 +01:00
17 changed files with 313 additions and 63 deletions
+12
View File
@@ -7,6 +7,18 @@ edit the release notes and re-run the generator.
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
## 0.119.2 — 2026-11-14
### Fixed
- A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming "untitled".
## 0.119.1 — 2026-11-14
### Fixed
- A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with "Site not supported".
## 0.119.0 — 2026-11-13 ## 0.119.0 — 2026-11-13
### New ### New
+1 -1
View File
@@ -1 +1 @@
0.119.0 0.119.2
+45 -6
View File
@@ -12,7 +12,7 @@ Resolution works both before extraction (from source_kind / URL host) and after
""" """
import re import re
from urllib.parse import urlparse from urllib.parse import parse_qs, urlparse
from yt_dlp.networking.impersonate import ImpersonateTarget from yt_dlp.networking.impersonate import ImpersonateTarget
@@ -183,10 +183,23 @@ class FacebookAdapter(SourceAdapter):
# in the page scripts that carry the post id; its watch link downloads (plain first, then the # in the page scripts that carry the post id; its watch link downloads (plain first, then the
# Chrome retry above). Needs the cookie: signed out, the page is the login wall. # Chrome retry above). Needs the cookie: signed out, the page is the login wall.
post_id = facebook_group_post_id(url) post_id = facebook_group_post_id(url)
if post_id is None or not ydl_opts.get("cookiefile"): if not ydl_opts.get("cookiefile"):
# yt-dlp knows the permalink form and answers the feed's form with "Unsupported URL".
if post_id is not None and not _FB_GROUP_POST_PATH.match(urlparse(url).path):
return _group_permalink(url, post_id)
return url
# A share link is a post only once followed: handed to yt-dlp, it was followed to the group
# page and an ad from the signed-in feed came down (prod job 471). One request does both — the
# redirect lands on the post page, which is then read as any post link's.
if post_id is None and not _is_share_link(url):
return url return url
with egress.open_ydl(ydl_opts) as ydl: with egress.open_ydl(ydl_opts) as ydl:
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES) response = ydl.urlopen(url)
if post_id is None:
post_id = facebook_group_post_id(response.url)
if post_id is None:
return url # a shared reel or video: yt-dlp follows the link as it always has
raw = response.read(self._POST_PAGE_MAX_BYTES)
page = raw.decode("utf-8", "replace") page = raw.decode("utf-8", "replace")
# Measured: a refused or expired cookie gets the login page (~450 KB, a login # Measured: a refused or expired cookie gets the login page (~450 KB, a login
# form, no post id at all), which is not a post without a video — the user's cookie is the cause. # form, no post id at all), which is not a post without a video — the user's cookie is the cause.
@@ -214,7 +227,7 @@ class FacebookAdapter(SourceAdapter):
# signed-out read titles the page with that same text behind the engagement counts, cut short. # signed-out read titles the page with that same text behind the engagement counts, cut short.
# Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives # Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives
# "Facebook" and NO description, while a signed-out read of the same reel still carries the text — # "Facebook" and NO description, while a signed-out read of the same reel still carries the text —
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`). # so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`).
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"}) _PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
def is_placeholder_title(self, title: str | None) -> bool: def is_placeholder_title(self, title: str | None) -> bool:
@@ -246,6 +259,7 @@ class FacebookAdapter(SourceAdapter):
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$") _FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
_FB_GROUP_PATH = re.compile(r"^/groups/[^/]+/?$")
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S) _SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"') _VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
@@ -256,12 +270,37 @@ def _quoted(post_id: str) -> str:
def facebook_group_post_id(url: str) -> str | None: def facebook_group_post_id(url: str) -> str | None:
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None.""" """The post id of a Facebook group post link, else None: `/groups/<group>/permalink|posts/<id>/`,
or a post opened from the group's feed, `/groups/<group>/?multi_permalinks=<id>` (one id only —
with several, which one was meant is not ours to guess)."""
parsed = urlparse(url) parsed = urlparse(url)
if not _host_is(parsed.netloc.lower(), "facebook.com"): if not _host_is(parsed.netloc.lower(), "facebook.com"):
return None return None
match = _FB_GROUP_POST_PATH.match(parsed.path) match = _FB_GROUP_POST_PATH.match(parsed.path)
return match.group(1) if match else None if match:
return match.group(1)
if not _FB_GROUP_PATH.match(parsed.path):
return None
ids = parse_qs(parsed.query).get("multi_permalinks", [])
return ids[0] if len(ids) == 1 and ids[0].isdigit() else None
def _group_permalink(url: str, post_id: str) -> str:
"""The permalink form of a group post link — the one form yt-dlp itself accepts."""
group = urlparse(url).path.split("/")[2]
return f"https://www.facebook.com/groups/{group}/permalink/{post_id}/"
def facebook_post_link(url: str) -> bool:
"""A link that `resolve_url` may read as a group post: a post link, or a share link (which may
lead to one) — what the failure catalog accepts for the post page's own verdicts."""
return facebook_group_post_id(url) is not None or _is_share_link(url)
def _is_share_link(url: str) -> bool:
# facebook.com/share/<code>/ and /share/v|r|p/<code>/ — a redirect to the shared object.
parsed = urlparse(url)
return _host_is(parsed.netloc.lower(), "facebook.com") and parsed.path.startswith("/share/")
def facebook_post_video_id(page: str, post_id: str) -> str | None: def facebook_post_video_id(page: str, post_id: str) -> str | None:
+17 -10
View File
@@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann
URL may have expired — those just get a locally-generated poster instead). A Facebook download URL may have expired — those just get a locally-generated poster instead). A Facebook download
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
page, and so is each download name that only repeated the placeholder. That page read page, and so is each download name that only repeated the placeholder. That page read
(`public_title_info`) also titles every new download, from the worker. (`public_page_info`) also titles and dates every new download, from the worker.
Run once per environment: Run once per environment:
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())" docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
@@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in
return None return None
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict: def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
"""`info` to title a download from: itself, unless its title is a source's placeholder with """`info` to title and date a download from: itself, unless its title is a source's placeholder
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out, with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one
metadata-only read of the page lends its text — the description, else its title — as the signed-out, metadata-only read of the page lends its text — the description, else its title — as
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was.""" the description the adapter takes a placeholder's title from, and its upload date when `info` has
none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since
was named 0000-00-00 in Season_0). Nothing better there: `info` as it was."""
if not adapter.is_placeholder_title(adapter.title(info)): if not adapter.is_placeholder_title(adapter.title(info)):
return info return info
public = extract_meta(url, adapter, asset_id) public = extract_meta(url, adapter, asset_id) or {}
text = public and (public.get("description") or public.get("title")) lent: dict = {}
return {**info, "description": text} if text else info text = public.get("description") or public.get("title")
if text:
lent["description"] = text
if not info.get("upload_date") and public.get("upload_date"):
lent["upload_date"] = public["upload_date"]
return {**info, **lent} if lent else info
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool: def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
@@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s
names that only repeated it. A name its owner gave (anything else) stays.""" names that only repeated it. A name its owner gave (anything else) stays."""
if not adapter.is_placeholder_title(a.title): if not adapter.is_placeholder_title(a.title):
return False return False
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id) info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id)
named = normalize_title(adapter.title(info)) named = normalize_title(adapter.title(info))
if not named or adapter.is_placeholder_title(named): if not named or adapter.is_placeholder_title(named):
return False return False
+4 -2
View File
@@ -32,6 +32,7 @@ from app.downloads.adapters import (
POST_NOT_SHOWN_MESSAGE, POST_NOT_SHOWN_MESSAGE,
POST_WITHOUT_VIDEO_MESSAGE, POST_WITHOUT_VIDEO_MESSAGE,
facebook_group_post_id, facebook_group_post_id,
facebook_post_link,
) )
from app.downloads.credentials import host_key_for_url from app.downloads.credentials import host_key_for_url
@@ -227,13 +228,14 @@ _CATALOG: tuple[_Entry, ...] = (
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared # answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about # clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
# that video, not the post — the category says it. Only for group post links (the adapter's own # that video, not the post — the category says it. Only for group post links (the adapter's own
# test of one): a reel with no formats is a different failure. # test of one): a reel with no formats is a different failure. The adapter's own text also comes
# from a share link that led to a post, so its test takes those too.
_Entry( _Entry(
host="facebook", host="facebook",
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),), markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
code="format_unavailable", code="format_unavailable",
detail="facebook.group_post_video", detail="facebook.group_post_video",
link=facebook_group_post_id, link=facebook_post_link,
), ),
_Entry( _Entry(
host="facebook", host="facebook",
+4 -3
View File
@@ -591,15 +591,16 @@ def _download(
# too. Same probe the upload ingest uses. # too. Same probe the upload ingest uses.
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info) vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
# A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the # A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the
# page's text from one signed-out read; the download itself stays the signed-in one. The page # page's text (and a missing upload date) from one signed-out read; the download itself stays
# the signed-in one. The page
# yt-dlp read, not the link queued: a group post's link is rewritten to its video, and # yt-dlp read, not the link queued: a group post's link is rewritten to its video, and
# signed out the post itself is the login wall. # signed out the post itself is the login wall.
titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id) titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id)
meta = storage.MediaMeta( meta = storage.MediaMeta(
video_id=info.get("id") or source_ref, video_id=info.get("id") or source_ref,
title=normalize_title(adapter.title(titled)) or source_ref, title=normalize_title(adapter.title(titled)) or source_ref,
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel", uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
upload_date=_parse_upload_date(info.get("upload_date")), upload_date=_parse_upload_date(titled.get("upload_date")),
uniqueid_type=adapter.nfo_uniqueid_type(), uniqueid_type=adapter.nfo_uniqueid_type(),
duration_s=duration_s, duration_s=duration_s,
description=info.get("description"), description=info.get("description"),
+7 -3
View File
@@ -20,6 +20,8 @@ from datetime import date
from pathlib import Path from pathlib import Path
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
from anyascii import anyascii
from app import logs from app import logs
from app.downloads import mediakind from app.downloads import mediakind
@@ -121,12 +123,14 @@ def files_root_problem(
def _ascii_fold(text: str) -> str: def _ascii_fold(text: str) -> str:
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop """Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters
no ASCII base).""" (`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew,
Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still
not ASCII after that is stripped."""
text = text.translate(_FOLD_MAP) text = text.translate(_FOLD_MAP)
text = unicodedata.normalize("NFKD", text) text = unicodedata.normalize("NFKD", text)
text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn") text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
return text.encode("ascii", "ignore").decode("ascii") return anyascii(text).encode("ascii", "ignore").decode("ascii")
@dataclass @dataclass
+3 -1
View File
@@ -7,7 +7,8 @@
# `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19 # `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403); # (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403);
# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322); # certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322);
# Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284). # Authlib 1.7.2 -> 1.9.1 (CVE-2026-96760, CVE-2026-114616); joserfc 1.7.4 -> 1.7.6 (CVE-2026-119283, CVE-2026-119284);
# anyascii 0.3.3 added (no dependencies of its own, so the rest of the lock is unchanged).
APScheduler==3.11.3 APScheduler==3.11.3
Authlib==1.9.1 Authlib==1.9.1
Mako==1.4.1 Mako==1.4.1
@@ -18,6 +19,7 @@ SQLAlchemy==2.0.52
alembic==1.19.1 alembic==1.19.1
annotated-doc==0.0.5 annotated-doc==0.0.5
annotated-types==0.8.0 annotated-types==0.8.0
anyascii==0.3.3
anyio==4.14.2 anyio==4.14.2
argon2-cffi-bindings==25.1.0 argon2-cffi-bindings==25.1.0
argon2-cffi==25.1.0 argon2-cffi==25.1.0
+4
View File
@@ -13,6 +13,10 @@ py3langid>=0.3,<1.0
itsdangerous>=2.1,<3.0 itsdangerous>=2.1,<3.0
cryptography>=50.0.0,<51 cryptography>=50.0.0,<51
argon2-cffi>=23.1,<26 argon2-cffi>=23.1,<26
# On-disk names are ASCII (storage._ascii_fold): a Hebrew, Cyrillic, Greek or CJK uploader name is
# written in Latin letters instead of dropped whole into one shared "untitled" folder. Pure Python,
# no dependencies of its own.
anyascii>=0.3.3,<0.4
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile) # Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver # does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
# scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so # scripts Deno runs for YouTube's n-signature). yt-dlp is release-often (YouTube changes), so
+36
View File
@@ -13,6 +13,7 @@ that the migration chain applies cleanly. Each test then gets a clean database v
tests that COMMIT (the whole point of this lane — transaction-ownership behaviour) don't leak. tests that COMMIT (the whole point of this lane — transaction-ownership behaviour) don't leak.
""" """
import datetime as _dt
import os import os
import shutil import shutil
from urllib.parse import quote from urllib.parse import quote
@@ -317,6 +318,41 @@ def _reset_rate_limiters():
obj.reset() obj.reset()
class _NeverBack(_dt.datetime):
"""The app's clock in tests: the wall clock, but never repeating and never going back — each
reading at least a microsecond after the last one in this process."""
last: _dt.datetime | None = None
clock: type[_dt.datetime] = (
_dt.datetime
) # what it reads; a probe can put a stepping one under it
@classmethod
def now(cls, tz=None):
t = cls.clock.now(tz)
last = cls.last
if last is not None and (last.tzinfo is None) == (t.tzinfo is None) and t <= last:
t = last + _dt.timedelta(microseconds=1)
cls.last = t
return t
@pytest.fixture(autouse=True)
def _clock_never_goes_back(monkeypatch):
"""Rows a test makes one after another are stamped in that order. The wall clock of the docker
VM the suite runs in steps BACK now and then (measured: 2 steps in 20 s, up to 0.5 s), and a
test that makes three files and reads them newest first then got them out of order. Every loaded
`app.*` module that took `datetime` from the datetime module reads this clock instead (a sweep,
as the limiter reset below does, so a new module is covered the day it lands). A database
default (`server_default=func.now()`) is Postgres's clock and is not covered: a test that orders
such rows made within a second of each other gives them distinct times itself."""
import sys
for name, mod in list(sys.modules.items()):
if (name == "app" or name.startswith("app.")) and vars(mod).get("datetime") is _dt.datetime:
monkeypatch.setattr(mod, "datetime", _NeverBack)
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def _reset_log_erasures(): def _reset_log_erasures():
"""`logs._erased` is module-level and remembers an erased account for an hour of monotonic time """`logs._erased` is module-level and remembers an erased account for an hour of monotonic time
+6
View File
@@ -218,6 +218,12 @@ def test_our_post_without_a_video_verdict_names_the_group_link():
"format_unavailable", "format_unavailable",
"facebook.group_post_video", "facebook.group_post_video",
) )
# The share link a post was reached by (prod job 471's kind) names the post too.
share = "https://www.facebook.com/share/v/19bCAHKyrd/"
assert dlerrors.diagnose(msg, url=share, cookie_attached=True) == (
"format_unavailable",
"facebook.group_post_video",
)
assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None) assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None)
@@ -27,6 +27,18 @@ _GROUP_LINK = f"https://www.facebook.com/groups/983554479435596/permalink/{_POST
_POST_VIDEO = "1408015447857753" _POST_VIDEO = "1408015447857753"
_FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video _FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video
_CHROME = ImpersonateTarget.from_str("chrome") _CHROME = ImpersonateTarget.from_str("chrome")
# The two other ways a group post reaches us (prod jobs 471 and 472, the same post): Facebook's "Share"
# link, which redirects to the post's permalink (measured), and the link of a post opened from the
# group's feed, which yt-dlp does not know at all ("Unsupported URL").
_SHARE_LINK = "https://www.facebook.com/share/v/19bCAHKyrd/"
_LANDED = (
f"https://www.facebook.com/groups/it.humor.and.memes/permalink/{_POST}/"
"?rdid=y9HAMntF7aCoAKZE&share_url=https%3A%2F%2Fwww.facebook.com%2Fshare%2Fv%2F19bCAHKyrd%2F"
)
_MULTI_LINK = (
f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST}"
"&hoisted_section_header_type=recently_seen&__cft__[0]=AZgK&__tn__=%2CO%2CP-R"
)
def _video_node(video_id: str) -> str: def _video_node(video_id: str) -> str:
@@ -86,6 +98,9 @@ def ydl(monkeypatch):
body = web["page"].encode() body = web["page"].encode()
class Response: class Response:
# Where the request ended up after its redirects (yt-dlp's `Response.url`).
url = web.get("landed") or web["opened"][-1]
def read(self, amt=None): def read(self, amt=None):
web["reads"].append(amt) web["reads"].append(amt)
return body return body
@@ -228,12 +243,79 @@ def test_without_a_cookie_the_group_post_link_goes_to_ytdlp_unchanged(ydl, tmp_p
assert web["extracted"] == [_GROUP_LINK] assert web["extracted"] == [_GROUP_LINK]
def test_a_share_link_to_a_group_post_downloads_the_posts_own_video(ydl, tmp_path):
# Prod job 471: handed the share link, yt-dlp followed it to the group page and downloaded an
# ad from the signed-in feed. Followed here first, it is the group post it leads to.
_, outcomes, web = ydl
web["page"], web["landed"] = _page(_POST_VIDEO), _LANDED
outcomes += [{"id": _POST_VIDEO}]
assert _run(tmp_path, "/cookies.txt", _SHARE_LINK) == {"id": _POST_VIDEO}
assert web["opened"] == [_SHARE_LINK] # one request: the redirect and the post page at once
assert web["reads"] == [adapters.FacebookAdapter._POST_PAGE_MAX_BYTES]
assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"]
def test_a_share_link_to_a_reel_goes_to_ytdlp_unchanged(ydl, tmp_path):
_, outcomes, web = ydl
web["landed"] = _URL
outcomes += [{"id": "806379462330494"}]
_run(tmp_path, "/cookies.txt", _SHARE_LINK)
assert web["reads"] == [] # not a post: its page is not read
assert web["extracted"] == [_SHARE_LINK] # the path that already downloads shared reels
def test_a_share_link_that_fails_as_a_group_post_says_why(ydl, tmp_path):
_, _, web = ydl
web["page"], web["landed"] = _page(), _LANDED
with pytest.raises(RuntimeError) as exc:
_run(tmp_path, "/cookies.txt", _SHARE_LINK)
assert web["extracted"] == []
# The job's own link is the share link: the verdict still names the post.
assert dlerrors.diagnose(str(exc.value), url=_SHARE_LINK, cookie_attached=True) == (
"format_unavailable",
"facebook.group_post_video",
)
def test_a_feed_opened_group_post_downloads_the_posts_own_video(ydl, tmp_path):
# Prod job 472: yt-dlp answers this link with "Unsupported URL".
_, outcomes, web = ydl
web["page"] = _page(_POST_VIDEO)
outcomes += [{"id": _POST_VIDEO}]
assert _run(tmp_path, "/cookies.txt", _MULTI_LINK) == {"id": _POST_VIDEO}
assert web["opened"] == [_MULTI_LINK]
assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"]
def test_without_a_cookie_a_feed_opened_group_post_goes_as_its_permalink(ydl, tmp_path):
# Signed out there is no page to read, but yt-dlp knows the permalink form and not this one.
_, outcomes, web = ydl
outcomes += [{"id": _POST_VIDEO}]
_run(tmp_path, None, _MULTI_LINK)
assert web["opened"] == []
assert web["extracted"] == [
f"https://www.facebook.com/groups/983554479435596/permalink/{_POST}/"
]
def test_the_group_post_link_forms_are_recognised(): def test_the_group_post_link_forms_are_recognised():
for url in ( for url in (
_GROUP_LINK, _GROUP_LINK,
f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/", f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/",
f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?", f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?",
f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}", f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}",
_MULTI_LINK,
f"https://www.facebook.com/groups/983554479435596?multi_permalinks={_POST}",
): ):
assert adapters.facebook_group_post_id(url) == _POST, url assert adapters.facebook_group_post_id(url) == _POST, url
for url in ( for url in (
@@ -241,6 +323,10 @@ def test_the_group_post_link_forms_are_recognised():
f"https://www.facebook.com/watch/?v={_POST_VIDEO}", f"https://www.facebook.com/watch/?v={_POST_VIDEO}",
f"https://facebook.com.evil/groups/x/permalink/{_POST}/", f"https://facebook.com.evil/groups/x/permalink/{_POST}/",
"https://www.facebook.com/groups/983554479435596/", "https://www.facebook.com/groups/983554479435596/",
# More than one post in the link: which one was meant is not ours to guess.
f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST},{_POST_VIDEO}",
f"https://www.facebook.com/groups/983554479435596/members/?multi_permalinks={_POST}",
_SHARE_LINK, # a share link is a post only once it is followed (`resolve_url`)
): ):
assert adapters.facebook_group_post_id(url) is None, url assert adapters.facebook_group_post_id(url) is None, url
assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK
+41 -9
View File
@@ -22,6 +22,9 @@ _PUBLIC = {
"description": "sorry not helping\n\n#workhumor #officelife", "description": "sorry not helping\n\n#workhumor #officelife",
"uploader": "Work dot exe", "uploader": "Work dot exe",
} }
# A signed-out read carries the upload date too (measured 2026-11-14 on watch?v=1140126121708919:
# "20261113"), which the signed-in read no longer gives.
_PUBLIC_DATED = {**_PUBLIC, "upload_date": "20261113"}
def _fake_ydl(monkeypatch, outcome): def _fake_ydl(monkeypatch, outcome):
@@ -54,7 +57,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
fb = adapters.FacebookAdapter() fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "sorry not helping" assert fb.title(info) == "sorry not helping"
# One metadata-only read, and signed out: no cookie goes with it. # One metadata-only read, and signed out: no cookie goes with it.
@@ -64,11 +67,36 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo
assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url() assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url()
def test_the_signed_out_read_also_lends_the_upload_date(monkeypatch):
# Measured on prod: since 2026-10-28 no Facebook download has an upload date (37 before had one,
# 31 since none) — the signed-in read lost it with the title. The signed-out read still has it.
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None}, _URL, adapters.FacebookAdapter(), 7
)
assert info["upload_date"] == "20261113"
def test_a_date_the_signed_in_read_gave_stays(monkeypatch):
_fake_ydl(monkeypatch, _PUBLIC_DATED)
info = backfill.public_page_info(
{"title": "Facebook", "description": None, "upload_date": "20261001"},
_URL,
adapters.FacebookAdapter(),
7,
)
assert info["upload_date"] == "20261001"
def test_a_real_title_reads_nothing_more(monkeypatch): def test_a_real_title_reads_nothing_more(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Funny Dark Humor Dad Joke!", "description": None} info = {"title": "Funny Dark Humor Dad Joke!", "description": None}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -76,7 +104,7 @@ def test_a_placeholder_with_the_posts_text_behind_it_reads_nothing_more(monkeypa
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": "Office jokes, part two"} info = {"title": "Video", "description": "Office jokes, part two"}
assert backfill.public_title_info(info, _URL, adapters.FacebookAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.FacebookAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -84,7 +112,7 @@ def test_a_placeholder_the_signed_out_page_cannot_better_stays(monkeypatch):
_fake_ydl(monkeypatch, {"title": "Facebook", "description": None}) _fake_ydl(monkeypatch, {"title": "Facebook", "description": None})
fb = adapters.FacebookAdapter() fb = adapters.FacebookAdapter()
info = backfill.public_title_info({"title": "Facebook", "description": None}, _URL, fb, 7) info = backfill.public_page_info({"title": "Facebook", "description": None}, _URL, fb, 7)
assert fb.title(info) == "Facebook" assert fb.title(info) == "Facebook"
@@ -93,7 +121,7 @@ def test_another_source_is_never_read_again(monkeypatch):
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC)
info = {"title": "Video", "description": None} info = {"title": "Video", "description": None}
assert backfill.public_title_info(info, _URL, adapters.SourceAdapter(), 7) is info assert backfill.public_page_info(info, _URL, adapters.SourceAdapter(), 7) is info
assert seen == [] assert seen == []
@@ -180,7 +208,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
db.add(job) db.add(job)
db.commit() db.commit()
read_page = backfill.public_title_info read_page = backfill.public_page_info
def renamed_while_reading(*args): def renamed_while_reading(*args):
with SessionLocal() as other: with SessionLocal() as other:
@@ -192,7 +220,7 @@ def test_a_name_given_while_the_backfill_runs_stays(db, make_user, monkeypatch):
other.commit() other.commit()
return read_page(*args) return read_page(*args)
monkeypatch.setattr(backfill, "public_title_info", renamed_while_reading) monkeypatch.setattr(backfill, "public_page_info", renamed_while_reading)
monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None) monkeypatch.setattr(backfill.editmod, "ensure_poster", lambda a, root: None)
assert backfill.run()["retitled"] == 1 assert backfill.run()["retitled"] == 1
@@ -252,8 +280,8 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
db, make_user, monkeypatch, tmp_path db, make_user, monkeypatch, tmp_path
): ):
# The whole download path with yt-dlp and the media tools faked: the signed-in read gives only # The whole download path with yt-dlp and the media tools faked: the signed-in read gives only
# the placeholder, and the file and the download's name get the post's text. # the placeholder, and the file and the download's name get the post's text — and its date.
seen = _fake_ydl(monkeypatch, _PUBLIC) seen = _fake_ydl(monkeypatch, _PUBLIC_DATED)
monkeypatch.setattr(settings, "files_root", str(tmp_path)) monkeypatch.setattr(settings, "files_root", str(tmp_path))
monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None)
monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced) monkeypatch.setattr(pipeline, "_ensure_phone_safe", lambda _j, produced, *_a: produced)
@@ -301,3 +329,7 @@ def test_a_download_titled_facebook_is_named_from_the_signed_out_page(
assert db.get(DownloadJob, job.id).status == "done" assert db.get(DownloadJob, job.id).status == "done"
assert db.get(MediaAsset, asset.id).title == "sorry not helping" assert db.get(MediaAsset, asset.id).title == "sorry not helping"
assert db.get(DownloadJob, job.id).display_name == "sorry not helping" assert db.get(DownloadJob, job.id).display_name == "sorry not helping"
done = db.get(MediaAsset, asset.id)
assert done.upload_date is not None and done.upload_date.isoformat() == "2026-11-13"
# The date names the file and its season, not 0000-00-00 / Season_0.
assert done.rel_path.startswith("Work_dot_exe/Season_2026/Work_dot_exe_-_2026-11-13_-_")
+15 -3
View File
@@ -26,9 +26,21 @@ def test_sanitize_output_is_pure_ascii_without_spaces():
assert _is_ascii_slug(out), out assert _is_ascii_slug(out), out
def test_sanitize_non_latin_falls_back_not_crashes(): def test_sanitize_writes_a_non_latin_name_in_latin_letters():
# A title with no ASCII base collapses to the untitled fallback rather than an empty name. # Prod asset 503: a Hebrew uploader name was dropped whole, so the file went to "untitled/", the
assert storage.sanitize("日本語") == "untitled" # folder every Hebrew, Cyrillic, Greek or CJK name would share — one show in Plex for all of them.
names = ("מַלכָּה אָדוֹם", "Сергей Иванов", "Ελληνικά Νέα", "日本語", "محمد علي")
slugs = [storage.sanitize(n) for n in names]
for name, slug in zip(names, slugs, strict=True):
assert _is_ascii_slug(slug) and slug != "untitled", (name, slug)
assert len(set(slugs)) == len(slugs), slugs # two names, two folders
# The measured spellings (anyascii 0.3.3, the locked one).
assert slugs == ["mlkhh_dvm", "Sergei_Ivanov", "Ellinika_Nea", "RiBenYu", "mhmd_ly"]
def test_sanitize_falls_back_when_nothing_is_left():
# Emoji and symbols are dropped, and a name of nothing else still gets a folder.
assert storage.sanitize("🎉🎉") == "untitled"
def test_display_filename_uses_underscores_and_ascii(): def test_display_filename_uses_underscores_and_ascii():
-25
View File
@@ -22,31 +22,6 @@ from app.runstatus import RunStatus, RunTrigger
_JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs
class _StrictlyLater(datetime):
"""A clock that never repeats or goes back: each reading at least a microsecond after the last."""
last: datetime | None = None
clock: type[datetime] = datetime
@classmethod
def now(cls, tz=None):
t = cls.clock.now(tz)
if cls.last is not None and t <= cls.last:
t = cls.last + timedelta(microseconds=1)
cls.last = t
return t
@pytest.fixture(autouse=True)
def _runs_start_in_order(monkeypatch):
"""Runs started in a row start in that order. These tests open several runs within microseconds
and read them back newest first, by start time; the wall clock of a container on a VM can step
back between two readings (seen: the third of three runs sorted before the second)."""
_StrictlyLater.last = None
_StrictlyLater.clock = jobruns.datetime
monkeypatch.setattr(jobruns, "datetime", _StrictlyLater)
def _release(): def _release():
scheduler._record(_JOB, running=False) scheduler._record(_JOB, running=False)
@@ -1,4 +1,20 @@
[ [
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook download gets its upload date again, so its file and season are named by it, not 0000-00-00. A channel name in another script, such as Hebrew or Cyrillic, is written in Latin letters in file names instead of becoming \"untitled\"."
]
},
{
"version": "0.119.1",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with \"Site not supported\"."
]
},
{ {
"version": "0.119.0", "version": "0.119.0",
"date": "2026-11-13", "date": "2026-11-13",
@@ -1,4 +1,20 @@
[ [
{
"version": "0.119.2",
"date": "2026-11-14",
"new": [],
"fixed": [
"A Facebook-letöltések újra megkapják a feltöltés dátumát, így a fájl és az évad neve azt mutatja, nem 0000-00-00-t. A más írású, például héber vagy cirill csatornanév a fájlnevekben latin betűkkel jelenik meg, nem „untitled” lesz."
]
},
{
"version": "0.119.1",
"date": "2026-11-14",
"new": [],
"fixed": [
"Egy Facebook-csoport posztja a Megosztás linkjével vagy a csoport hírfolyamából megnyitva is a poszt saját videóját tölti le. Eddig a Megosztás linkje a hírfolyam egy másik videóját is hozhatta, a hírfolyam linkje pedig „Nem támogatott oldal” hibát adott."
]
},
{ {
"version": "0.119.0", "version": "0.119.0",
"date": "2026-11-13", "date": "2026-11-13",