fix(downloads): Facebook files get their upload date, non-Latin names their letters
Since 2026-10-28 the signed-in Facebook read gives no upload date, so every Facebook file since was named 0000-00-00 in Season_0 (31 on prod, none before). The signed-out read that already titles these downloads carries the date, so it now lends that too when the signed-in read has none (backfill.public_title_info is now public_page_info). On-disk names dropped every letter outside the Latin script, so a Hebrew, Cyrillic, Greek or CJK uploader went to one shared "untitled" folder (prod asset 503). anyascii now writes them in Latin letters (Сергей Иванов -> Sergei_Ivanov); Latin and accented names are unchanged.
This commit is contained in:
@@ -227,7 +227,7 @@ class FacebookAdapter(SourceAdapter):
|
||||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||||
# Measured (2026-11-01, yt-dlp 2026.08.19): since 2026-10-28 every signed-in reel read gives
|
||||
# "Facebook" and NO description, while a signed-out read of the same reel still carries the text —
|
||||
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_title_info`).
|
||||
# so a placeholder left with nothing behind it gets one signed-out read (`backfill.public_page_info`).
|
||||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||||
|
||||
def is_placeholder_title(self, title: str | None) -> bool:
|
||||
|
||||
@@ -8,7 +8,7 @@ Facebook sources (a direct media URL like a reddit HLS manifest carries no chann
|
||||
URL may have expired — those just get a locally-generated poster instead). A Facebook download
|
||||
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
|
||||
page, and so is each download name that only repeated the placeholder. That page read
|
||||
(`public_title_info`) also titles every new download, from the worker.
|
||||
(`public_page_info`) also titles and dates every new download, from the worker.
|
||||
|
||||
Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
@@ -71,16 +71,23 @@ def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: in
|
||||
return None
|
||||
|
||||
|
||||
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
|
||||
"""`info` to title a download from: itself, unless its title is a source's placeholder with
|
||||
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out,
|
||||
metadata-only read of the page lends its text — the description, else its title — as the
|
||||
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was."""
|
||||
def public_page_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
|
||||
"""`info` to title and date a download from: itself, unless its title is a source's placeholder
|
||||
with nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one
|
||||
signed-out, metadata-only read of the page lends its text — the description, else its title — as
|
||||
the description the adapter takes a placeholder's title from, and its upload date when `info` has
|
||||
none (measured on prod: the signed-in read lost both on 2026-10-28, so every Facebook file since
|
||||
was named 0000-00-00 in Season_0). Nothing better there: `info` as it was."""
|
||||
if not adapter.is_placeholder_title(adapter.title(info)):
|
||||
return info
|
||||
public = extract_meta(url, adapter, asset_id)
|
||||
text = public and (public.get("description") or public.get("title"))
|
||||
return {**info, "description": text} if text else info
|
||||
public = extract_meta(url, adapter, asset_id) or {}
|
||||
lent: dict = {}
|
||||
text = public.get("description") or public.get("title")
|
||||
if text:
|
||||
lent["description"] = text
|
||||
if not info.get("upload_date") and public.get("upload_date"):
|
||||
lent["upload_date"] = public["upload_date"]
|
||||
return {**info, **lent} if lent else info
|
||||
|
||||
|
||||
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
|
||||
@@ -88,7 +95,7 @@ def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: s
|
||||
names that only repeated it. A name its owner gave (anything else) stays."""
|
||||
if not adapter.is_placeholder_title(a.title):
|
||||
return False
|
||||
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id)
|
||||
info = public_page_info({"title": a.title, "description": None}, url, adapter, a.id)
|
||||
named = normalize_title(adapter.title(info))
|
||||
if not named or adapter.is_placeholder_title(named):
|
||||
return False
|
||||
|
||||
@@ -591,15 +591,16 @@ def _download(
|
||||
# too. Same probe the upload ingest uses.
|
||||
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
||||
# A placeholder title with nothing behind it ("Facebook" from a signed-in read) borrows the
|
||||
# page's text from one signed-out read; the download itself stays the signed-in one. The page
|
||||
# page's text (and a missing upload date) from one signed-out read; the download itself stays
|
||||
# the signed-in one. The page
|
||||
# yt-dlp read, not the link queued: a group post's link is rewritten to its video, and
|
||||
# signed out the post itself is the login wall.
|
||||
titled = backfill.public_title_info(info, info.get("webpage_url") or url, adapter, asset_id)
|
||||
titled = backfill.public_page_info(info, info.get("webpage_url") or url, adapter, asset_id)
|
||||
meta = storage.MediaMeta(
|
||||
video_id=info.get("id") or source_ref,
|
||||
title=normalize_title(adapter.title(titled)) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
upload_date=_parse_upload_date(titled.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=duration_s,
|
||||
description=info.get("description"),
|
||||
|
||||
@@ -20,6 +20,8 @@ from datetime import date
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from anyascii import anyascii
|
||||
|
||||
from app import logs
|
||||
from app.downloads import mediakind
|
||||
|
||||
@@ -121,12 +123,14 @@ def files_root_problem(
|
||||
|
||||
def _ascii_fold(text: str) -> str:
|
||||
"""Transliterate to pure ASCII: apply the explicit map, NFKD-split accented letters and drop
|
||||
the combining marks (á→a, ő→o), then strip any leftover non-ASCII (non-Latin scripts that have
|
||||
no ASCII base)."""
|
||||
the combining marks (á→a, ő→o), then write what is left of other scripts in Latin letters
|
||||
(`anyascii`: Сергей→Sergey). Before it, a non-Latin name was dropped whole, and every Hebrew,
|
||||
Cyrillic, Greek or CJK uploader shared one "untitled" folder (prod asset 503). Anything still
|
||||
not ASCII after that is stripped."""
|
||||
text = text.translate(_FOLD_MAP)
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = "".join(ch for ch in text if unicodedata.category(ch) != "Mn")
|
||||
return text.encode("ascii", "ignore").decode("ascii")
|
||||
return anyascii(text).encode("ascii", "ignore").decode("ascii")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user