peter
dc221338be
The E22 redactor is a safety net over addresses and credential params, and every one of these call sites was leaning on it to do the whole job. What it does not cover went in whole, and what it does cover came out as a constant. Not covered, so a real leak: - routes/search.py logged the raw search term at ERROR. The privacy policy shipped in 0.88.0 says a search string is kept only at Teljes/TRACE, with the user's own consent. The ERROR line now carries the shape of the failure (source, candidate and collected counts) and the term follows on a TRACE line, which nothing emits unless that consent is in force. - downloads/backfill.py logged the full source URL at INFO — a row of the user's download history. Now the asset id and the host. - youtube/client.py logged the upstream error body, which is prose that quotes the request back. Every failing YouTube call now goes through one _upstream_failure() that logs the API's `reason` code, keeps the body for the same consented TRACE line, and raises a YouTubeError carrying `reason` as a field — callers put that error in log lines of their own (routes/channels.py) and in a 422 answered to the user, so the body must not travel in it either. Covered, so merely useless: routes/auth.py named the account by an address that reached both sinks as `[email]`, and app/email.py printed recipient lists that read as `['[email]']`. Both now carry the uid (or a count) and a reason code, which is also the first half of E22.5 S4's failed-sign-in coverage. routes/channels.py branched on the text of that error message to recognise an already-existing subscription. It now branches on `exc.reason`, with the substring check kept beside it for a body that carried no reason at all. tests/test_log_pii.py pins the call sites and the family: a source-level contract test walks every log call in app/ and fails on an argument naming user-typed content unless the line says why it is safe (`# log-pii-ok: <why>`). Eight sites carry that marker today, each with its reason. Gate 8/8 green; all five cases mutation-checked.
97 lines
4.1 KiB
Python
97 lines
4.1 KiB
Python
"""One-off backfill for downloads made before per-asset channel URLs + poster fallbacks existed.
|
|
|
|
Fills `media_assets.uploader_url` (re-extracting metadata for YouTube/Facebook sources, so an
|
|
auto-detected channel becomes a clickable link) and `poster_path` (an ffmpeg frame for
|
|
thumbnail-less sources, so the card / watch page / link preview still shows an image). Best-effort
|
|
and safe to re-run: it only touches assets still missing the field, and only re-fetches YouTube /
|
|
Facebook sources (a direct media URL like a reddit HLS manifest carries no channel and its signed
|
|
URL may have expired — those just get a locally-generated poster instead).
|
|
|
|
Run once per environment:
|
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
|
"""
|
|
|
|
import logging
|
|
from urllib.parse import urlsplit
|
|
|
|
import yt_dlp
|
|
from sqlalchemy import select
|
|
|
|
from app.config import settings
|
|
from app.db import SessionLocal
|
|
from app.downloads import adapters
|
|
from app.downloads import edit as editmod
|
|
from app.downloads import formats, service
|
|
from app.models import MediaAsset
|
|
|
|
log = logging.getLogger("siftlode.backfill")
|
|
|
|
|
|
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
|
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
|
needs the POT/player-client strategy even just to extract)."""
|
|
for clients in adapter.client_sets():
|
|
opts = formats.build_ydl_opts(
|
|
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
|
)
|
|
opts["skip_download"] = True
|
|
try:
|
|
with yt_dlp.YoutubeDL(opts) as ydl:
|
|
info = ydl.extract_info(url, download=False)
|
|
if info:
|
|
return info
|
|
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
|
# The asset id, not the URL. A source URL is a row of this user's download
|
|
# history, and `redact_text` only takes credential params out of it — the rest
|
|
# of it goes in whole. The id names the same asset to anyone holding the
|
|
# database and nothing at all to anyone who is not; the host is kept because it
|
|
# is what decides which client set was tried.
|
|
log.info(
|
|
"meta extract failed for asset %s on %s (%s): %s",
|
|
asset_id,
|
|
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
|
clients,
|
|
str(exc)[:100],
|
|
)
|
|
return None
|
|
|
|
|
|
def run() -> dict:
|
|
filled_url = filled_poster = 0
|
|
with SessionLocal() as db:
|
|
assets = (
|
|
db.execute(
|
|
select(MediaAsset).where(
|
|
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
for a in assets:
|
|
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
|
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
|
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
|
info = _extract_meta(url, adapter, a.id)
|
|
uu = adapter.uploader_url(info) if info else None
|
|
if uu:
|
|
a.uploader_url = uu
|
|
filled_url += 1
|
|
log.info("asset %s uploader_url <- %s", a.id, uu)
|
|
if a.poster_path is None:
|
|
# Record a self-hosted poster for every download (the reliable og:image source):
|
|
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
|
rel = editmod.ensure_poster(a, settings.download_root)
|
|
if rel:
|
|
a.poster_path = rel
|
|
filled_poster += 1
|
|
log.info("asset %s poster <- %s", a.id, rel)
|
|
db.commit()
|
|
result = {
|
|
"assets": len(assets),
|
|
"filled_uploader_url": filled_url,
|
|
"filled_poster": filled_poster,
|
|
}
|
|
log.info("backfill done: %s", result)
|
|
return result
|