From 2cd02eb1af81e07a9a977ae5697c07036c7e8d15 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 13 Aug 2026 01:18:25 +0200 Subject: [PATCH] fix(downloads): netloc-scoped adapter host match (review round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_adapter matched sources by loose substring over the netloc, so a look-alike host ('myyoutube.com', 'youtube.com.evil') misresolved. Match the host exactly or as a subdomain via a _host_is() helper, and identify from the netloc of BOTH source_ref and yt-dlp's webpage_url — the latter by host only, never a substring over its path. This also restores a host signal to the early-fill path (resolve_adapter(None, None, info)), which has no source_ref, so a Facebook asset's uploader_url no longer depends on extractor_key being present in the progress-hook info_dict. Tests: look-alike hosts stay generic while real subdomains resolve; a webpage_url netloc identifies the source when extractor_key is absent. --- backend/app/downloads/adapters.py | 40 +++++++++++++++++++------------ backend/tests/test_adapters.py | 19 +++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index dff1ef6a..24282bf3 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -116,26 +116,36 @@ _REDDIT = RedditAdapter() _GENERIC = SourceAdapter() +def _host_is(host: str, *domains: str) -> bool: + """True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a + loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches.""" + return any(host == d or host.endswith("." + d) for d in domains) + + def resolve_adapter( source_kind: str | None, source_ref: str | None = None, info: dict | None = None ) -> SourceAdapter: - """Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key` - from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to - the generic adapter — so behaviour is unchanged for any source without a dedicated one.""" - # Identify from the yt-dlp extractor key alone — NOT the webpage_url, whose path could contain a - # sibling source's name (e.g. a reddit post slug '.../youtube_is_dead') and cause a misresolve. - tag = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else "" - host = urlparse(source_ref).netloc.lower() if source_ref else "" - if ( - source_kind == "youtube" - or "youtube" in tag - or "youtu.be" in host - or "youtube.com" in host - or "youtube-nocookie.com" in host + """Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info` + (post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref` + (pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and + the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling + source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source.""" + extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else "" + hosts = [] + if source_ref: + hosts.append(urlparse(source_ref).netloc.lower()) + if info and info.get("webpage_url"): + hosts.append(urlparse(info["webpage_url"]).netloc.lower()) + + def host_is(*domains: str) -> bool: + return any(_host_is(h, *domains) for h in hosts) + + if source_kind == "youtube" or "youtube" in extractor or host_is( + "youtube.com", "youtu.be", "youtube-nocookie.com" ): return _YOUTUBE - if "facebook" in tag or "facebook.com" in host or "fb.watch" in host: + if "facebook" in extractor or host_is("facebook.com", "fb.watch"): return _FACEBOOK - if "reddit" in tag or "reddit.com" in host: + if "reddit" in extractor or host_is("reddit.com"): return _REDDIT return _GENERIC diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index d52233c3..5b9ce684 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -40,6 +40,25 @@ def test_youtube_nocookie_host_still_gets_the_youtube_adapter(): assert resolve_adapter("url", "https://www.youtube-nocookie.com/embed/abc123").key == "youtube" +def test_lookalike_hosts_do_not_misresolve_but_subdomains_do(): + # Netloc-scoped match: a look-alike host that merely CONTAINS the domain string is NOT the source. + assert resolve_adapter("url", "https://myyoutube.com/watch?v=x").key == "generic" + assert resolve_adapter("url", "https://youtube.com.evil.example/x").key == "generic" + assert resolve_adapter("url", "https://notreddit.com/r/x").key == "generic" + # …but real subdomains still resolve. + assert resolve_adapter("url", "https://m.youtube.com/watch?v=x").key == "youtube" + assert resolve_adapter("url", "https://music.youtube.com/watch?v=x").key == "youtube" + + +def test_webpage_url_netloc_identifies_source_when_extractor_key_absent(): + # The early-fill path passes info with no source_ref; the webpage_url netloc still resolves the + # source even if extractor_key is missing from the progress-hook info_dict. + assert resolve_adapter(None, None, {"webpage_url": "https://www.facebook.com/reel/123"}).key == "facebook" + # A sibling source's name in the webpage_url PATH must not misresolve (host, not substring). + info = {"extractor_key": "Reddit", "webpage_url": "https://www.reddit.com/r/x/comments/y/youtube"} + assert resolve_adapter(None, None, info).key == "reddit" + + def test_uploader_url_youtube_uses_channel_url(): a = YouTubeAdapter() assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"