diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index 7e4705db..16c2994d 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -61,6 +61,10 @@ class SourceAdapter: return False # --- metadata ----------------------------------------------------------------------------- + def title(self, info: dict) -> str | None: + """The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's.""" + return info.get("title") + def uploader_url(self, info: dict) -> str | None: """The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives `channel_url`), else None (reddit / a raw media URL have no derivable channel page).""" @@ -158,6 +162,23 @@ class FacebookAdapter(SourceAdapter): def retriable(self, error_text: str) -> bool: return FACEBOOK_PARSE_WALL in error_text.lower() + # Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an + # 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A + # signed-out read titles the page with that same text behind the engagement counts, cut short. + _PLACEHOLDER_TITLES = frozenset({"video", "facebook"}) + + def title(self, info: dict) -> str | None: + raw = info.get("title") + if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES: + return raw + line = next( + (ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), "" + ) + # The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into + # "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The + # stored title is unbounded; display_name and the file name clip it themselves. + return line or raw # no description either (the 18+ reel): nothing better on the page + def uploader_url(self, info: dict) -> str | None: # Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id` # (the page id) that resolves at facebook.com/. diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 040eb6f4..115b11f8 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -57,6 +57,10 @@ _DEFAULTS = { # opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped # the v3 net — must re-derive through the current browser-safe guarantee. _SIG_VERSION = 4 +# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every +# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that +# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive. +_AUDIO_SIG_VERSION = 5 _SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"] @@ -103,7 +107,7 @@ def format_sig(spec: dict, salt: str | None = None) -> str: always re-fetches into its own asset instead of sharing a cached file with other jobs).""" s = normalize(spec) data = {k: s[k] for k in _SIG_KEYS} - data["_v"] = _SIG_VERSION + data["_v"] = _AUDIO_SIG_VERSION if s["mode"] == "a" else _SIG_VERSION if salt is not None: data["_salt"] = salt payload = json.dumps(data, sort_keys=True, separators=(",", ":")) diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index d7308661..758d2f4f 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -113,9 +113,10 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None: # `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty # (e.g. a title that was only a social engagement prefix); the completion path later sets # the final title with its own source_ref fallback. - asset.title = normalize_title(info.get("title")) or info.get("id") + adapter = adapters.resolve_adapter(None, None, info) + asset.title = normalize_title(adapter.title(info)) or info.get("id") asset.uploader = info.get("uploader") or info.get("channel") - asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info) + asset.uploader_url = adapter.uploader_url(info) asset.thumbnail_url = info.get("thumbnail") asset.duration_s = int(info["duration"]) if info.get("duration") else None asset.upload_date = _parse_upload_date(info.get("upload_date")) @@ -403,8 +404,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the - file to store (converted, or the original) and corrects `info`'s codec fields to match disk.""" - if spec.get("container") != "mp4" or spec.get("vcodec"): + file to store (converted, or the original) and corrects `info`'s codec fields to match disk. + + An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which + every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4, + which iPad's WebKit refuses to play (prod job 268).""" + if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"): return produced vcodec, acodec = editmod.probe_codecs(produced) is_mp4 = produced.suffix.lower() == ".mp4" @@ -509,7 +514,7 @@ def _download( vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info) meta = storage.MediaMeta( video_id=info.get("id") or source_ref, - title=normalize_title(info.get("title")) or source_ref, + title=normalize_title(adapter.title(info)) or source_ref, uploader=info.get("uploader") or info.get("channel") or "Unknown Channel", upload_date=_parse_upload_date(info.get("upload_date")), uniqueid_type=adapter.nfo_uniqueid_type(), diff --git a/backend/tests/test_adapters.py b/backend/tests/test_adapters.py index d5909a44..ef5ac453 100644 --- a/backend/tests/test_adapters.py +++ b/backend/tests/test_adapters.py @@ -1,6 +1,7 @@ """Per-source adapter selection + the source-specific policy each one carries. These lock the behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch.""" +from app.titles import normalize_title from app.downloads.adapters import ( FacebookAdapter, RedditAdapter, @@ -139,3 +140,28 @@ def test_wants_metadata_backfill_matches_old_should_refetch(): def test_nfo_uniqueid_type_is_the_adapter_key(): assert YouTubeAdapter().nfo_uniqueid_type() == "youtube" assert SourceAdapter().nfo_uniqueid_type() == "generic" + + +def test_title_is_yt_dlps_for_a_source_without_placeholders(): + assert SourceAdapter().title({"title": "Video", "description": "text"}) == "Video" + assert SourceAdapter().title({}) is None + + +def test_facebook_placeholder_title_takes_the_posts_text(): + # Measured: signed in, yt-dlp titles every reel "Video" (an 18+ reel read as Chrome: "Facebook"), + # while the post's text is still in the description. + a = FacebookAdapter() + info = {"title": "Video", "description": "\n Tune of The Day\nsecond line"} + assert a.title(info) == "Tune of The Day" + assert a.title({**info, "title": "Facebook"}) == "Tune of The Day" + # A real title (signed out) is left for normalize_title, engagement prefix and all. + assert a.title({**info, "title": "456K views | Tune"}) == "456K views | Tune" + # Nothing better on the page: keep what yt-dlp gave. + assert a.title({"title": "Facebook", "description": None}) == "Facebook" + + +def test_facebook_title_from_a_long_description_is_the_whole_line_after_cleaning(): + # Dev UAT: a cut with "…" came out of normalize_title as "… live at." (NFKC "..." collapsed to "."), + # a sentence that seemed to end. The whole first line survives the cleaning intact instead. + line = 'Tune of The Day, Argy playing his version of the classic "Resurrection" with Paul Oakenfold live at Space' + assert normalize_title(FacebookAdapter().title({"title": "Video", "description": line})) == line diff --git a/backend/tests/test_download_failure_catalog.py b/backend/tests/test_download_failure_catalog.py index 44bc5542..ec241e26 100644 --- a/backend/tests/test_download_failure_catalog.py +++ b/backend/tests/test_download_failure_catalog.py @@ -314,3 +314,89 @@ def test_0084_backfills_only_the_facebook_parse_failures_it_measured(db): finally: if not upgraded: command.upgrade(cfg, "head") + + +def _ffmpeg_media(path, *codec_args): + subprocess.run( + ["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi", "-i", "sine=frequency=440"] + + list(codec_args) + + ["-t", "2", str(path)], + check=True, + ) + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") +def test_a_signed_in_facebook_download_is_titled_with_its_posts_text(db, monkeypatch, tmp_path): + # Prod jobs 265–268 (2026-09-14): with a cookie yt-dlp titles every reel "Video", so the row, the + # file name and the share preview all said "Video". The post's text is in the description. + u = _user(db, "fb-title@example.com") + a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") + monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) + + def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): + media = staging / "2131884261040652.m4a" + _ffmpeg_media(media, "-c:a", "aac") + return { + "id": "2131884261040652", + "ext": "m4a", + "title": "Video", + "description": "Tune of The Day, Argy playing his version", + "extractor_key": "Facebook", + "filepath": str(media), + } + + monkeypatch.setattr(pipeline, "_extract_with_fallback", extract) + + pipeline._download(j.id, a.id, "url", _REEL, {"mode": "a"}, u.id) + + db.expire_all() + asset, job = db.get(MediaAsset, a.id), db.get(DownloadJob, j.id) + assert asset.title == job.display_name == "Tune of The Day, Argy playing his version" + assert "Tune_of_The_Day" in asset.rel_path + + +def test_the_early_title_takes_the_posts_text_too(db): + u = _user(db, "fb-early-title@example.com") + a, _ = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") + a.title = None + db.commit() + + pipeline._fill_asset_meta_early( + a.id, + { + "id": "1", + "title": "Video", + "description": "Almás palacsinta", + "extractor_key": "Facebook", + }, + ) + + db.expire_all() + assert db.get(MediaAsset, a.id).title == "Almás palacsinta" + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="ffmpeg not available") +def test_an_mp3_download_is_stored_as_an_mp3(db, monkeypatch, tmp_path): + # Prod job 268: the "Audio (MP3)" profile's extracted .mp3 was remuxed into an .mp4 and served as + # video/mp4, which iPad's WebKit will not play. It stays an .mp3 (served as audio/mpeg). + u = _user(db, "fb-mp3@example.com") + a, j = _asset_and_job(db, u, sig="shared", asset_status="downloading", job_status="running") + monkeypatch.setattr(settings, "download_root", str(tmp_path)) + monkeypatch.setattr(pipeline.urlguard, "check_url", lambda url: None) + + def extract(job_id, asset_id, url, spec, staging, adapter, **kwargs): + media = staging / "2131884261040652.mp3" + _ffmpeg_media(media, "-c:a", "libmp3lame") + return {"id": "2131884261040652", "ext": "mp3", "title": "Reel", "filepath": str(media)} + + monkeypatch.setattr(pipeline, "_extract_with_fallback", extract) + + pipeline._download( + j.id, a.id, "url", _REEL, formats.normalize({"mode": "a", "audio_format": "mp3"}), u.id + ) + + db.expire_all() + asset = db.get(MediaAsset, a.id) + assert (asset.container, asset.acodec, asset.media_kind) == ("mp3", "mp3", "audio") + assert asset.rel_path.endswith(".mp3") diff --git a/backend/tests/test_formats.py b/backend/tests/test_formats.py index 6c1fbbdd..5eec3e51 100644 --- a/backend/tests/test_formats.py +++ b/backend/tests/test_formats.py @@ -158,3 +158,15 @@ def test_a_normal_cut_list_is_untouched(): segs = [{"start_s": 0, "end_s": 5}, {"start_s": 10, "end_s": 12}] out = editmod.normalize_edit_spec({"segments": segs}) assert len(out["segments"]) == 2 + + +def test_audio_versions_its_cache_identity_apart_from_video(monkeypatch): + # The .mp3/.m4a-stays-itself fix must re-derive cached remuxed audio WITHOUT re-downloading every + # cached video: bumping one version moves only its own mode's signature. + audio, video = {"mode": "a", "audio_format": "mp3"}, {"mode": "av"} + before = formats.format_sig(audio), formats.format_sig(video) + monkeypatch.setattr(formats, "_AUDIO_SIG_VERSION", formats._AUDIO_SIG_VERSION + 1) + assert formats.format_sig(audio) != before[0] + assert formats.format_sig(video) == before[1] + monkeypatch.setattr(formats, "_SIG_VERSION", formats._SIG_VERSION + 1) + assert formats.format_sig(video) != before[1]