peter
c910cf2f20
- A press that begins while a field in the row is edited only ends the editing (rowOpen latches it at pointerdown); the card layout's row press is now tested. - A related video reset in the player forgets the session's stored position. - Removing the playing video passes over known-dead items and steps the way it went. - Prev/Next off only because of dead items stay focusable and say why (HU/EN). - The docked bar's scroller leaves room for focus rings. - A focus-opened About card is not closed by its own scroll into view. - EN "Clear watch progress" for the video reset, distinct from a page's "Reset view". - The signed-out title read goes to the page yt-dlp read; the retitle writes are conditional. - Tests: the cap of three errors, the bar's own Fullscreen, the group glyph, the backfill's reads.
149 lines
7.0 KiB
Python
149 lines
7.0 KiB
Python
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
|
|
|
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
|
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
|
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
|
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
|
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
|
URL may have expired — those just get a locally-generated poster instead). A Facebook download
|
|
titled with the source's placeholder ("Facebook", "Video") is retitled from a signed-out read of its
|
|
page, and so is each download name that only repeated the placeholder. That page read
|
|
(`public_title_info`) also titles every new download, from the worker.
|
|
|
|
Run once per environment:
|
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
|
"""
|
|
|
|
from urllib.parse import urlsplit
|
|
|
|
import yt_dlp
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs
|
|
from app.config import settings
|
|
from app.db import SessionLocal
|
|
from app.downloads import adapters
|
|
from app.downloads import edit as editmod
|
|
from app.downloads import formats, service
|
|
from app.downloads.models import DownloadJob, MediaAsset
|
|
from app.models import DISPLAY_TEXT_MAX
|
|
from app.titles import normalize_title
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
|
|
def extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
|
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
|
needs the POT/player-client strategy even just to extract)."""
|
|
with formats.hiding(url):
|
|
return _extract_meta_scoped(url, adapter, asset_id)
|
|
|
|
|
|
def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
|
for clients in adapter.client_sets():
|
|
opts = formats.build_ydl_opts(
|
|
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
|
)
|
|
opts["skip_download"] = True
|
|
try:
|
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
|
info = ydl.extract_info(url, download=False)
|
|
if info:
|
|
return info
|
|
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
|
# The asset id, not the URL. A source URL is a row of this user's download
|
|
# history, and `redact_text` only takes credential params out of it — the rest
|
|
# of it goes in whole. The id names the same asset to anyone holding the
|
|
# database and nothing at all to anyone who is not; the host is kept because it
|
|
# is what decides which client set was tried. yt-dlp's message goes through
|
|
# `log_safe` for the same reason: it opens with the URL or the video id.
|
|
log.info(
|
|
"meta extract failed for asset %s on %s (%s): %s",
|
|
asset_id,
|
|
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
|
clients,
|
|
formats.log_safe(exc)[:100],
|
|
)
|
|
return None
|
|
|
|
|
|
def public_title_info(info: dict, url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict:
|
|
"""`info` to title a download from: itself, unless its title is a source's placeholder with
|
|
nothing behind it (a signed-in Facebook read: "Facebook", no description). Then one signed-out,
|
|
metadata-only read of the page lends its text — the description, else its title — as the
|
|
description the adapter takes a placeholder's title from. Nothing better there: `info` as it was."""
|
|
if not adapter.is_placeholder_title(adapter.title(info)):
|
|
return info
|
|
public = extract_meta(url, adapter, asset_id)
|
|
text = public and (public.get("description") or public.get("title"))
|
|
return {**info, "description": text} if text else info
|
|
|
|
|
|
def _retitle(db: Session, a: MediaAsset, adapter: adapters.SourceAdapter, url: str) -> bool:
|
|
"""Swap a placeholder title kept from an earlier download for the page's text, and the download
|
|
names that only repeated it. A name its owner gave (anything else) stays."""
|
|
if not adapter.is_placeholder_title(a.title):
|
|
return False
|
|
info = public_title_info({"title": a.title, "description": None}, url, adapter, a.id)
|
|
named = normalize_title(adapter.title(info))
|
|
if not named or adapter.is_placeholder_title(named):
|
|
return False
|
|
# One conditional write, not a read and a compare: a name its owner gives while this runs (the
|
|
# poster step can take seconds before the commit) is not overwritten by what was read before it.
|
|
db.execute(
|
|
update(DownloadJob)
|
|
.where(DownloadJob.asset_id == a.id, DownloadJob.display_name == a.title)
|
|
.values(display_name=named[:DISPLAY_TEXT_MAX])
|
|
)
|
|
a.title = named
|
|
return True
|
|
|
|
|
|
def run() -> dict:
|
|
filled_url = filled_poster = retitled = 0
|
|
with SessionLocal() as db:
|
|
assets = (
|
|
db.execute(
|
|
select(MediaAsset).where(
|
|
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
for a in assets:
|
|
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
|
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
|
info = extract_meta(url, adapter, a.id)
|
|
uu = adapter.uploader_url(info) if info else None
|
|
if uu:
|
|
a.uploader_url = uu
|
|
filled_url += 1
|
|
log.info(
|
|
"asset %s uploader_url filled", a.id
|
|
) # the URL names what they downloaded
|
|
if adapter.wants_metadata_backfill():
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
|
if _retitle(db, a, adapter, url):
|
|
retitled += 1
|
|
log.info("asset %s retitled", a.id) # the title names what they downloaded
|
|
if a.poster_path is None:
|
|
# Record a self-hosted poster for every download (the reliable og:image source):
|
|
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
|
rel = editmod.ensure_poster(a, settings.files_root)
|
|
if rel:
|
|
a.poster_path = rel
|
|
filled_poster += 1
|
|
log.info("asset %s poster filled", a.id) # the path carries the title
|
|
db.commit()
|
|
result = {
|
|
"assets": len(assets),
|
|
"filled_uploader_url": filled_url,
|
|
"filled_poster": filled_poster,
|
|
"retitled": retitled,
|
|
}
|
|
log.info("backfill done: %s", result)
|
|
return result
|