From 4f7887720123600d0c76b2b8c6d96f8910000465 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Fri, 13 Nov 2026 03:02:00 +0100 Subject: [PATCH 1/3] test: the app's clock never goes back in the test suite The docker VM the suite runs in steps its wall clock back now and then (measured: 2 steps in 20 s, up to 0.5 s), so rows a test made one after another could read out of order (a files list sorted newest first, a scheduler's last runs). Every app module that took datetime from the datetime module reads a clock that never repeats or goes back; the scheduler test's own fixture for this goes. --- backend/tests/conftest.py | 36 +++++++++++++++++++++++++++++++++ backend/tests/test_scheduler.py | 25 ----------------------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 66cb9a8b..ba9f4f6d 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -13,6 +13,7 @@ that the migration chain applies cleanly. Each test then gets a clean database v tests that COMMIT (the whole point of this lane — transaction-ownership behaviour) don't leak. """ +import datetime as _dt import os import shutil from urllib.parse import quote @@ -317,6 +318,41 @@ def _reset_rate_limiters(): obj.reset() +class _NeverBack(_dt.datetime): + """The app's clock in tests: the wall clock, but never repeating and never going back — each + reading at least a microsecond after the last one in this process.""" + + last: _dt.datetime | None = None + clock: type[_dt.datetime] = ( + _dt.datetime + ) # what it reads; a probe can put a stepping one under it + + @classmethod + def now(cls, tz=None): + t = cls.clock.now(tz) + last = cls.last + if last is not None and (last.tzinfo is None) == (t.tzinfo is None) and t <= last: + t = last + _dt.timedelta(microseconds=1) + cls.last = t + return t + + +@pytest.fixture(autouse=True) +def _clock_never_goes_back(monkeypatch): + """Rows a test makes one after another are stamped in that order. The wall clock of the docker + VM the suite runs in steps BACK now and then (measured: 2 steps in 20 s, up to 0.5 s), and a + test that makes three files and reads them newest first then got them out of order. Every loaded + `app.*` module that took `datetime` from the datetime module reads this clock instead (a sweep, + as the limiter reset below does, so a new module is covered the day it lands). A database + default (`server_default=func.now()`) is Postgres's clock and is not covered: a test that orders + such rows made within a second of each other gives them distinct times itself.""" + import sys + + for name, mod in list(sys.modules.items()): + if (name == "app" or name.startswith("app.")) and vars(mod).get("datetime") is _dt.datetime: + monkeypatch.setattr(mod, "datetime", _NeverBack) + + @pytest.fixture(autouse=True) def _reset_log_erasures(): """`logs._erased` is module-level and remembers an erased account for an hour of monotonic time diff --git a/backend/tests/test_scheduler.py b/backend/tests/test_scheduler.py index 2465ed30..cab4b7bf 100644 --- a/backend/tests/test_scheduler.py +++ b/backend/tests/test_scheduler.py @@ -22,31 +22,6 @@ from app.runstatus import RunStatus, RunTrigger _JOB = "unit_test_claim" # not a real job id — exercises the primitive without touching live jobs -class _StrictlyLater(datetime): - """A clock that never repeats or goes back: each reading at least a microsecond after the last.""" - - last: datetime | None = None - clock: type[datetime] = datetime - - @classmethod - def now(cls, tz=None): - t = cls.clock.now(tz) - if cls.last is not None and t <= cls.last: - t = cls.last + timedelta(microseconds=1) - cls.last = t - return t - - -@pytest.fixture(autouse=True) -def _runs_start_in_order(monkeypatch): - """Runs started in a row start in that order. These tests open several runs within microseconds - and read them back newest first, by start time; the wall clock of a container on a VM can step - back between two readings (seen: the third of three runs sorted before the second).""" - _StrictlyLater.last = None - _StrictlyLater.clock = jobruns.datetime - monkeypatch.setattr(jobruns, "datetime", _StrictlyLater) - - def _release(): scheduler._record(_JOB, running=False) From fb3aca4c9c1e5892cbd7e6e1fecb6575d3642a19 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 14 Nov 2026 00:29:59 +0100 Subject: [PATCH 2/3] fix(downloads): a shared or feed-opened Facebook group post downloads its own video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Facebook share link (/share/v/…) redirects to a group post, but only the permalink form went through the post-page read, so yt-dlp followed the share link itself and downloaded an ad from the signed-in group feed (prod job 471). The link of a post opened from the group's feed (/groups//?multi_permalinks=) was not recognised at all, and yt-dlp answers it with "Unsupported URL" (prod job 472). With a cookie, a share link is now followed in the same request that reads the post page, and a feed link is read as the post it names. Without one, the feed link goes to yt-dlp in the permalink form it knows. A post page's "no single video" verdict now names the post for a share link too. --- backend/app/downloads/adapters.py | 49 +++++++++-- backend/app/downloads/errors.py | 6 +- backend/tests/test_download_errors.py | 6 ++ backend/tests/test_facebook_impersonation.py | 86 ++++++++++++++++++++ 4 files changed, 140 insertions(+), 7 deletions(-) diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 1a932fb3..18a6ec63 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -12,7 +12,7 @@ Resolution works both before extraction (from source_kind / URL host) and after """ import re -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse from yt_dlp.networking.impersonate import ImpersonateTarget @@ -183,10 +183,23 @@ class FacebookAdapter(SourceAdapter): # in the page scripts that carry the post id; its watch link downloads (plain first, then the # Chrome retry above). Needs the cookie: signed out, the page is the login wall. post_id = facebook_group_post_id(url) - if post_id is None or not ydl_opts.get("cookiefile"): + if not ydl_opts.get("cookiefile"): + # yt-dlp knows the permalink form and answers the feed's form with "Unsupported URL". + if post_id is not None and not _FB_GROUP_POST_PATH.match(urlparse(url).path): + return _group_permalink(url, post_id) + return url + # A share link is a post only once followed: handed to yt-dlp, it was followed to the group + # page and an ad from the signed-in feed came down (prod job 471). One request does both — the + # redirect lands on the post page, which is then read as any post link's. + if post_id is None and not _is_share_link(url): return url with egress.open_ydl(ydl_opts) as ydl: - raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES) + response = ydl.urlopen(url) + if post_id is None: + post_id = facebook_group_post_id(response.url) + if post_id is None: + return url # a shared reel or video: yt-dlp follows the link as it always has + raw = response.read(self._POST_PAGE_MAX_BYTES) page = raw.decode("utf-8", "replace") # Measured: a refused or expired cookie gets the login page (~450 KB, a login # form, no post id at all), which is not a post without a video — the user's cookie is the cause. @@ -246,6 +259,7 @@ class FacebookAdapter(SourceAdapter): _FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$") +_FB_GROUP_PATH = re.compile(r"^/groups/[^/]+/?$") _SCRIPT_BLOCK = re.compile(r"]*>(.*?)", re.S) _VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"') @@ -256,12 +270,37 @@ def _quoted(post_id: str) -> str: def facebook_group_post_id(url: str) -> str | None: - """The post id of a Facebook group post link (`/groups//permalink|posts//`), else None.""" + """The post id of a Facebook group post link, else None: `/groups//permalink|posts//`, + or a post opened from the group's feed, `/groups//?multi_permalinks=` (one id only — + with several, which one was meant is not ours to guess).""" parsed = urlparse(url) if not _host_is(parsed.netloc.lower(), "facebook.com"): return None match = _FB_GROUP_POST_PATH.match(parsed.path) - return match.group(1) if match else None + if match: + return match.group(1) + if not _FB_GROUP_PATH.match(parsed.path): + return None + ids = parse_qs(parsed.query).get("multi_permalinks", []) + return ids[0] if len(ids) == 1 and ids[0].isdigit() else None + + +def _group_permalink(url: str, post_id: str) -> str: + """The permalink form of a group post link — the one form yt-dlp itself accepts.""" + group = urlparse(url).path.split("/")[2] + return f"https://www.facebook.com/groups/{group}/permalink/{post_id}/" + + +def facebook_post_link(url: str) -> bool: + """A link that `resolve_url` may read as a group post: a post link, or a share link (which may + lead to one) — what the failure catalog accepts for the post page's own verdicts.""" + return facebook_group_post_id(url) is not None or _is_share_link(url) + + +def _is_share_link(url: str) -> bool: + # facebook.com/share// and /share/v|r|p// — a redirect to the shared object. + parsed = urlparse(url) + return _host_is(parsed.netloc.lower(), "facebook.com") and parsed.path.startswith("/share/") def facebook_post_video_id(page: str, post_id: str) -> str | None: diff --git a/backend/app/downloads/errors.py b/backend/app/downloads/errors.py index 8bf5a62c..6640ec4d 100644 --- a/backend/app/downloads/errors.py +++ b/backend/app/downloads/errors.py @@ -32,6 +32,7 @@ from app.downloads.adapters import ( POST_NOT_SHOWN_MESSAGE, POST_WITHOUT_VIDEO_MESSAGE, facebook_group_post_id, + facebook_post_link, ) from app.downloads.credentials import host_key_for_url @@ -227,13 +228,14 @@ _CATALOG: tuple[_Entry, ...] = ( # answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared # clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about # that video, not the post — the category says it. Only for group post links (the adapter's own - # test of one): a reel with no formats is a different failure. + # test of one): a reel with no formats is a different failure. The adapter's own text also comes + # from a share link that led to a post, so its test takes those too. _Entry( host="facebook", markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),), code="format_unavailable", detail="facebook.group_post_video", - link=facebook_group_post_id, + link=facebook_post_link, ), _Entry( host="facebook", diff --git a/backend/tests/test_download_errors.py b/backend/tests/test_download_errors.py index 270f171b..1df671f3 100644 --- a/backend/tests/test_download_errors.py +++ b/backend/tests/test_download_errors.py @@ -218,6 +218,12 @@ def test_our_post_without_a_video_verdict_names_the_group_link(): "format_unavailable", "facebook.group_post_video", ) + # The share link a post was reached by (prod job 471's kind) names the post too. + share = "https://www.facebook.com/share/v/19bCAHKyrd/" + assert dlerrors.diagnose(msg, url=share, cookie_attached=True) == ( + "format_unavailable", + "facebook.group_post_video", + ) assert dlerrors.diagnose(msg, url=_REEL, cookie_attached=True) == ("format_unavailable", None) diff --git a/backend/tests/test_facebook_impersonation.py b/backend/tests/test_facebook_impersonation.py index f1a0971b..4ca5e65a 100644 --- a/backend/tests/test_facebook_impersonation.py +++ b/backend/tests/test_facebook_impersonation.py @@ -27,6 +27,18 @@ _GROUP_LINK = f"https://www.facebook.com/groups/983554479435596/permalink/{_POST _POST_VIDEO = "1408015447857753" _FEED_VIDEO = "1056963063816966" # what yt-dlp downloaded from this post's link: a feed video _CHROME = ImpersonateTarget.from_str("chrome") +# The two other ways a group post reaches us (prod jobs 471 and 472, the same post): Facebook's "Share" +# link, which redirects to the post's permalink (measured), and the link of a post opened from the +# group's feed, which yt-dlp does not know at all ("Unsupported URL"). +_SHARE_LINK = "https://www.facebook.com/share/v/19bCAHKyrd/" +_LANDED = ( + f"https://www.facebook.com/groups/it.humor.and.memes/permalink/{_POST}/" + "?rdid=y9HAMntF7aCoAKZE&share_url=https%3A%2F%2Fwww.facebook.com%2Fshare%2Fv%2F19bCAHKyrd%2F" +) +_MULTI_LINK = ( + f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST}" + "&hoisted_section_header_type=recently_seen&__cft__[0]=AZgK&__tn__=%2CO%2CP-R" +) def _video_node(video_id: str) -> str: @@ -86,6 +98,9 @@ def ydl(monkeypatch): body = web["page"].encode() class Response: + # Where the request ended up after its redirects (yt-dlp's `Response.url`). + url = web.get("landed") or web["opened"][-1] + def read(self, amt=None): web["reads"].append(amt) return body @@ -228,12 +243,79 @@ def test_without_a_cookie_the_group_post_link_goes_to_ytdlp_unchanged(ydl, tmp_p assert web["extracted"] == [_GROUP_LINK] +def test_a_share_link_to_a_group_post_downloads_the_posts_own_video(ydl, tmp_path): + # Prod job 471: handed the share link, yt-dlp followed it to the group page and downloaded an + # ad from the signed-in feed. Followed here first, it is the group post it leads to. + _, outcomes, web = ydl + web["page"], web["landed"] = _page(_POST_VIDEO), _LANDED + outcomes += [{"id": _POST_VIDEO}] + + assert _run(tmp_path, "/cookies.txt", _SHARE_LINK) == {"id": _POST_VIDEO} + + assert web["opened"] == [_SHARE_LINK] # one request: the redirect and the post page at once + assert web["reads"] == [adapters.FacebookAdapter._POST_PAGE_MAX_BYTES] + assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"] + + +def test_a_share_link_to_a_reel_goes_to_ytdlp_unchanged(ydl, tmp_path): + _, outcomes, web = ydl + web["landed"] = _URL + outcomes += [{"id": "806379462330494"}] + + _run(tmp_path, "/cookies.txt", _SHARE_LINK) + + assert web["reads"] == [] # not a post: its page is not read + assert web["extracted"] == [_SHARE_LINK] # the path that already downloads shared reels + + +def test_a_share_link_that_fails_as_a_group_post_says_why(ydl, tmp_path): + _, _, web = ydl + web["page"], web["landed"] = _page(), _LANDED + + with pytest.raises(RuntimeError) as exc: + _run(tmp_path, "/cookies.txt", _SHARE_LINK) + + assert web["extracted"] == [] + # The job's own link is the share link: the verdict still names the post. + assert dlerrors.diagnose(str(exc.value), url=_SHARE_LINK, cookie_attached=True) == ( + "format_unavailable", + "facebook.group_post_video", + ) + + +def test_a_feed_opened_group_post_downloads_the_posts_own_video(ydl, tmp_path): + # Prod job 472: yt-dlp answers this link with "Unsupported URL". + _, outcomes, web = ydl + web["page"] = _page(_POST_VIDEO) + outcomes += [{"id": _POST_VIDEO}] + + assert _run(tmp_path, "/cookies.txt", _MULTI_LINK) == {"id": _POST_VIDEO} + + assert web["opened"] == [_MULTI_LINK] + assert web["extracted"] == [f"https://www.facebook.com/watch/?v={_POST_VIDEO}"] + + +def test_without_a_cookie_a_feed_opened_group_post_goes_as_its_permalink(ydl, tmp_path): + # Signed out there is no page to read, but yt-dlp knows the permalink form and not this one. + _, outcomes, web = ydl + outcomes += [{"id": _POST_VIDEO}] + + _run(tmp_path, None, _MULTI_LINK) + + assert web["opened"] == [] + assert web["extracted"] == [ + f"https://www.facebook.com/groups/983554479435596/permalink/{_POST}/" + ] + + def test_the_group_post_link_forms_are_recognised(): for url in ( _GROUP_LINK, f"https://www.facebook.com/groups/983554479435596/posts/{_POST}/", f"https://m.facebook.com/groups/983554479435596/permalink/{_POST}/?", f"https://www.facebook.com/groups/chatgptmagyarul/posts/{_POST}", + _MULTI_LINK, + f"https://www.facebook.com/groups/983554479435596?multi_permalinks={_POST}", ): assert adapters.facebook_group_post_id(url) == _POST, url for url in ( @@ -241,6 +323,10 @@ def test_the_group_post_link_forms_are_recognised(): f"https://www.facebook.com/watch/?v={_POST_VIDEO}", f"https://facebook.com.evil/groups/x/permalink/{_POST}/", "https://www.facebook.com/groups/983554479435596/", + # More than one post in the link: which one was meant is not ours to guess. + f"https://www.facebook.com/groups/983554479435596/?multi_permalinks={_POST},{_POST_VIDEO}", + f"https://www.facebook.com/groups/983554479435596/members/?multi_permalinks={_POST}", + _SHARE_LINK, # a share link is a post only once it is followed (`resolve_url`) ): assert adapters.facebook_group_post_id(url) is None, url assert adapters.SourceAdapter().resolve_url(_GROUP_LINK, {"cookiefile": "/c"}) == _GROUP_LINK From 9d979a520222a45ce4373b9dfac34af3f34b0ba8 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Sat, 14 Nov 2026 00:36:34 +0100 Subject: [PATCH 3/3] release: 0.119.1 --- CHANGELOG.md | 6 ++++++ VERSION | 2 +- frontend/src/content/release-notes/en.json | 8 ++++++++ frontend/src/content/release-notes/hu.json | 8 ++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41171261..98c7adb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.119.1 — 2026-11-14 + +### Fixed + +- A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with "Site not supported". + ## 0.119.0 — 2026-11-13 ### New diff --git a/VERSION b/VERSION index f34340fc..368ee87c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.119.0 +0.119.1 diff --git a/frontend/src/content/release-notes/en.json b/frontend/src/content/release-notes/en.json index e1cb31f8..dce00bab 100644 --- a/frontend/src/content/release-notes/en.json +++ b/frontend/src/content/release-notes/en.json @@ -1,4 +1,12 @@ [ + { + "version": "0.119.1", + "date": "2026-11-14", + "new": [], + "fixed": [ + "A Facebook group post pasted as its Share link, or opened from the group's feed, now downloads the post's own video. Before, a Share link could bring down another video from the feed, and a feed link failed with \"Site not supported\"." + ] + }, { "version": "0.119.0", "date": "2026-11-13", diff --git a/frontend/src/content/release-notes/hu.json b/frontend/src/content/release-notes/hu.json index 0ac04e11..a161a11d 100644 --- a/frontend/src/content/release-notes/hu.json +++ b/frontend/src/content/release-notes/hu.json @@ -1,4 +1,12 @@ [ + { + "version": "0.119.1", + "date": "2026-11-14", + "new": [], + "fixed": [ + "Egy Facebook-csoport posztja a Megosztás linkjével vagy a csoport hírfolyamából megnyitva is a poszt saját videóját tölti le. Eddig a Megosztás linkje a hírfolyam egy másik videóját is hozhatta, a hírfolyam linkje pedig „Nem támogatott oldal” hibát adott." + ] + }, { "version": "0.119.0", "date": "2026-11-13",