fix(downloads): netloc-scoped adapter host match (review round 2)
resolve_adapter matched sources by loose substring over the netloc, so a
look-alike host ('myyoutube.com', 'youtube.com.evil') misresolved. Match the
host exactly or as a subdomain via a _host_is() helper, and identify from the
netloc of BOTH source_ref and yt-dlp's webpage_url — the latter by host only,
never a substring over its path. This also restores a host signal to the
early-fill path (resolve_adapter(None, None, info)), which has no source_ref,
so a Facebook asset's uploader_url no longer depends on extractor_key being
present in the progress-hook info_dict.
Tests: look-alike hosts stay generic while real subdomains resolve; a
webpage_url netloc identifies the source when extractor_key is absent.
This commit is contained in:
@@ -116,26 +116,36 @@ _REDDIT = RedditAdapter()
|
||||
_GENERIC = SourceAdapter()
|
||||
|
||||
|
||||
def _host_is(host: str, *domains: str) -> bool:
|
||||
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
|
||||
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
|
||||
return any(host == d or host.endswith("." + d) for d in domains)
|
||||
|
||||
|
||||
def resolve_adapter(
|
||||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||||
) -> SourceAdapter:
|
||||
"""Pick the adapter for a job. Matches on `source_kind` first, then the yt-dlp `extractor_key`
|
||||
from `info` (post-extraction), then the URL host in `source_ref` (pre-extraction). Falls back to
|
||||
the generic adapter — so behaviour is unchanged for any source without a dedicated one."""
|
||||
# Identify from the yt-dlp extractor key alone — NOT the webpage_url, whose path could contain a
|
||||
# sibling source's name (e.g. a reddit post slug '.../youtube_is_dead') and cause a misresolve.
|
||||
tag = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||||
host = urlparse(source_ref).netloc.lower() if source_ref else ""
|
||||
if (
|
||||
source_kind == "youtube"
|
||||
or "youtube" in tag
|
||||
or "youtu.be" in host
|
||||
or "youtube.com" in host
|
||||
or "youtube-nocookie.com" in host
|
||||
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
|
||||
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
|
||||
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
|
||||
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
|
||||
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
|
||||
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||||
hosts = []
|
||||
if source_ref:
|
||||
hosts.append(urlparse(source_ref).netloc.lower())
|
||||
if info and info.get("webpage_url"):
|
||||
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
|
||||
|
||||
def host_is(*domains: str) -> bool:
|
||||
return any(_host_is(h, *domains) for h in hosts)
|
||||
|
||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
||||
):
|
||||
return _YOUTUBE
|
||||
if "facebook" in tag or "facebook.com" in host or "fb.watch" in host:
|
||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||
return _FACEBOOK
|
||||
if "reddit" in tag or "reddit.com" in host:
|
||||
if "reddit" in extractor or host_is("reddit.com"):
|
||||
return _REDDIT
|
||||
return _GENERIC
|
||||
|
||||
@@ -40,6 +40,25 @@ def test_youtube_nocookie_host_still_gets_the_youtube_adapter():
|
||||
assert resolve_adapter("url", "https://www.youtube-nocookie.com/embed/abc123").key == "youtube"
|
||||
|
||||
|
||||
def test_lookalike_hosts_do_not_misresolve_but_subdomains_do():
|
||||
# Netloc-scoped match: a look-alike host that merely CONTAINS the domain string is NOT the source.
|
||||
assert resolve_adapter("url", "https://myyoutube.com/watch?v=x").key == "generic"
|
||||
assert resolve_adapter("url", "https://youtube.com.evil.example/x").key == "generic"
|
||||
assert resolve_adapter("url", "https://notreddit.com/r/x").key == "generic"
|
||||
# …but real subdomains still resolve.
|
||||
assert resolve_adapter("url", "https://m.youtube.com/watch?v=x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://music.youtube.com/watch?v=x").key == "youtube"
|
||||
|
||||
|
||||
def test_webpage_url_netloc_identifies_source_when_extractor_key_absent():
|
||||
# The early-fill path passes info with no source_ref; the webpage_url netloc still resolves the
|
||||
# source even if extractor_key is missing from the progress-hook info_dict.
|
||||
assert resolve_adapter(None, None, {"webpage_url": "https://www.facebook.com/reel/123"}).key == "facebook"
|
||||
# A sibling source's name in the webpage_url PATH must not misresolve (host, not substring).
|
||||
info = {"extractor_key": "Reddit", "webpage_url": "https://www.reddit.com/r/x/comments/y/youtube"}
|
||||
assert resolve_adapter(None, None, info).key == "reddit"
|
||||
|
||||
|
||||
def test_uploader_url_youtube_uses_channel_url():
|
||||
a = YouTubeAdapter()
|
||||
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
|
||||
|
||||
Reference in New Issue
Block a user