peter
4e4fff87d4
The URL guard judged the address the user typed; a public host's 3xx into our network, a page's formats pointing inside, and a name rebound between the check and the fetch all got past it, and the stored error told a live internal port from a dead one. Every non-YouTube download now runs through an egress gate on the worker's loopback: it resolves each host once, refuses any forbidden answer before dialling, and connects to the address it vetted. Measured on the worker image: yt-dlp's proxy reaches the Requests and CurlCFFI handlers, native HLS and ffmpeg alike. Formats fetched over anything but http(s), which ffmpeg would dial directly, are dropped, and one factory opens every yt-dlp so none skips it.
158 lines
7.4 KiB
Python
158 lines
7.4 KiB
Python
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
|
|
|
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
|
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
|
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
|
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
|
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
|
URL may have expired — those just get a locally-generated poster instead). A Facebook download
|
|
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
|
|
page, and so is each download name that only repeated the placeholder. That page read
|
|
(`public_title_info`) also titles every new download, from the worker.
|
|
|
|
Run once per environment:
|
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
|
"""
|
|
|
|
from urllib.parse import urlsplit
|
|
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs
|
|
from app.config import settings
|
|
from app.db import SessionLocal
|
|
from app.downloads import adapters
|
|
from app.downloads import edit as editmod
|
|
from app.downloads import egress, formats, service
|
|
from app.downloads.models import DownloadJob, MediaAsset
|
|
from app.models import DISPLAY_TEXT_MAX
|
|
from app.titles import normalize_title
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
|
|
def extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
|
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
|
needs the POT/player-client strategy even just to extract)."""
|
|
with formats.hiding(url):
|
|
return _extract_meta_scoped(url, adapter, asset_id)
|
|
|
|
|
|
def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
|
for clients in adapter.client_sets():
|
|
opts = formats.build_ydl_opts(
|
|
{},
|
|
"%(id)s.%(ext)s",
|
|
lambda d: None,
|
|
extra=adapter.ydl_opts_extra(clients),
|
|
egress=adapter.egress_guarded,
|
|
)
|
|
opts["skip_download"] = True
|
|
try:
|
|
with egress.open_ydl(opts) as ydl:
|
|
info = ydl.extract_info(url, download=False)
|
|
if info:
|
|
return info
|
|
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
|
# The asset id, not the URL. A source URL is a row of this user's download
|
|
# history, and `redact_text` only takes credential params out of it — the rest
|
|
# of it goes in whole. The id names the same asset to anyone holding the
|
|
# database and nothing at all to anyone who is not; the host is kept because it
|
|
# is what decides which client set was tried. yt-dlp's message goes through
|
|
# `log_safe` for the same reason: it opens with the URL or the video id.
|
|
log.info(
|
|
"meta extract failed for asset %s on %s (%s): %s",
|
|
asset_id,
|
|
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
|
clients,
|
|
formats.log_safe(exc)[:100],
|
|
)
|
|
return None
|
|
|
|
|
|
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
|
|
"""`info` to title a download from: itself, unless its title is a source's placeholder with
|
|
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out,
|
|
metadata-only read of the page lends its text — the description, else its title — as the
|
|
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was."""
|
|
if not adapter.is_placeholder_title(adapter.title(info)):
|
|
return info
|
|
public = extract_meta(url, adapter, asset_id)
|
|
text = public and (public.get("description") or public.get("title"))
|
|
return {**info, "description": text} if text else info
|
|
|
|
|
|
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
|
|
"""Swap a placeholder title kept from an earlier download for the page's text, and the download
|
|
names that only repeated it. A name its owner gave (anything else) stays."""
|
|
if not adapter.is_placeholder_title(a.title):
|
|
return False
|
|
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id)
|
|
named = normalize_title(adapter.title(info))
|
|
if not named or adapter.is_placeholder_title(named):
|
|
return False
|
|
# One conditional write, not a read and a compare: a name its owner gives while the page is read
|
|
# is not overwritten by what was read before it. The asset first, as every other writer of the
|
|
# two locks them (`service.lock_own_job`): the session does not flush by itself.
|
|
placeholder = a.title
|
|
a.title = named
|
|
db.flush()
|
|
db.execute(
|
|
update(DownloadJob)
|
|
.where(DownloadJob.asset_id == a.id, DownloadJob.display_name == placeholder)
|
|
.values(display_name=named[:DISPLAY_TEXT_MAX])
|
|
)
|
|
return True
|
|
|
|
|
|
def run() -> dict:
|
|
filled_url = filled_poster = retitled = 0
|
|
with SessionLocal() as db:
|
|
assets = (
|
|
db.execute(
|
|
select(MediaAsset).where(
|
|
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
for a in assets:
|
|
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
|
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
|
info = extract_meta(url, adapter, a.id)
|
|
uu = adapter.uploader_url(info) if info else None
|
|
if uu:
|
|
a.uploader_url = uu
|
|
filled_url += 1
|
|
log.info(
|
|
"asset %s uploader_url filled", a.id
|
|
) # the URL names what they downloaded
|
|
if adapter.wants_metadata_backfill():
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
|
if _retitle(db, a, adapter, url):
|
|
# Before the poster step: it can cut a frame for 30 s, and these locks held
|
|
# across it block a rename and deadlock a delete.
|
|
db.commit()
|
|
retitled += 1
|
|
log.info("asset %s retitled", a.id) # the title names what they downloaded
|
|
if a.poster_path is None:
|
|
# Record a self-hosted poster for every download (the reliable og:image source):
|
|
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
|
rel = editmod.ensure_poster(a, settings.files_root)
|
|
if rel:
|
|
a.poster_path = rel
|
|
filled_poster += 1
|
|
log.info("asset %s poster filled", a.id) # the path carries the title
|
|
db.commit()
|
|
result = {
|
|
"assets": len(assets),
|
|
"filled_uploader_url": filled_url,
|
|
"filled_poster": filled_poster,
|
|
"retitled": retitled,
|
|
}
|
|
log.info("backfill done: %s", result)
|
|
return result
|