peter
5809f3e431
Transaction ownership (C-A5): - quota.record_usage / log_action write in their OWN SessionLocal(), committed immediately, and no longer take/commit the caller's db. The spend stays DURABLE regardless of the caller (read-only YouTube paths never commit; a failed job rolls back) — under-counting the shared daily budget would risk overspending the real YouTube quota — AND quota stops flushing a job's partial state mid-run behind its own `except: db.rollback()`. units_used_today switches to a scalar SELECT so measured()'s before/after diff sees the separate session's commits (READ COMMITTED). Call sites updated (youtube/client.py ×3, routes/search.py ×1). - plex sync commits PER SECTION (moved the end-of-loop commit inside), so a PlexError on a later library keeps the ones already mirrored (idempotent; next run reconciles). The sync jobs did NOT rely on record_usage's incidental commit for persistence — their per-item functions (apply_rss_feed, backfill_channel_recent, import_subscriptions, sync_user_playlists, apply_channel_autotags) all commit internally, so the in-loop rollback handlers already isolate a failed item. No loop restructuring needed. DB-lane tests (test_quota_ownership, 4): spend survives a caller rollback while the caller's uncommitted row does NOT leak; accumulation; measured-style mid-txn read. Gate: ruff clean; DB lane 180 green.
418 lines
17 KiB
Python
418 lines
17 KiB
Python
"""Mirror a locally-reachable Plex library into Siftlode's own catalog (plex_* tables).
|
|
|
|
Playback is from the local physical file, so this only mirrors METADATA + the physical-media facts
|
|
(codecs/container/path) needed to browse, search, and later play. It reuses Plex's own codec info
|
|
(no ffprobe needed) to classify each item's browser playability. Movies + episodes become playable
|
|
LEAF rows (plex_items); shows/seasons are mirrored for the drill-down + episode ordering.
|
|
|
|
The sync is a full reconcile of the enabled sections (upsert by rating_key). Pruning of items that
|
|
disappeared from Plex is intentionally left for later (a follow-up can diff rating_keys per library).
|
|
Cast + intro/credit markers are fetched lazily on the item-detail/player path (P2), not here.
|
|
"""
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import sysconfig
|
|
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
|
from app.plex import people
|
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
|
|
|
log = logging.getLogger("siftlode.plex")
|
|
|
|
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
|
_BROWSER_VIDEO = {"h264"}
|
|
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
|
_BROWSER_CONTAINER = {"mp4", "mov", "m4v"}
|
|
|
|
# Plex section item types.
|
|
_TYPE_MOVIE = 1
|
|
_TYPE_SHOW = 2
|
|
_TYPE_SEASON = 3
|
|
_TYPE_EPISODE = 4
|
|
|
|
_PAGE = 200
|
|
|
|
|
|
def classify_playable(vcodec: str | None, acodec: str | None, container: str | None) -> str:
|
|
"""How the browser can play this file: 'direct' (native <video>), 'remux' (copy video, fix
|
|
container / transcode audio — cheap), or 'transcode' (re-encode video — expensive)."""
|
|
v = (vcodec or "").lower()
|
|
a = (acodec or "").lower()
|
|
c = (container or "").lower()
|
|
if not v or v not in _BROWSER_VIDEO:
|
|
return "transcode"
|
|
if a and a not in _BROWSER_AUDIO:
|
|
return "remux"
|
|
if c and c not in _BROWSER_CONTAINER:
|
|
return "remux"
|
|
return "direct"
|
|
|
|
|
|
def _epoch(v) -> datetime | None:
|
|
try:
|
|
return datetime.fromtimestamp(int(v), tz=timezone.utc) if v else None
|
|
except (TypeError, ValueError, OSError):
|
|
return None
|
|
|
|
|
|
def _date(v):
|
|
"""Parse a Plex 'YYYY-MM-DD' string (originallyAvailableAt) → date, or None."""
|
|
try:
|
|
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _tags(meta: dict, key: str, limit: int | None = None) -> list | None:
|
|
"""The .tag values of a Plex child-tag array (Genre/Director/Role) present in the listing."""
|
|
vals = [t.get("tag") for t in (meta.get(key) or []) if t.get("tag")]
|
|
if limit:
|
|
vals = vals[:limit]
|
|
return vals or None
|
|
|
|
|
|
def _rating(meta: dict) -> float | None:
|
|
try:
|
|
return float(meta.get("audienceRating") or meta.get("rating"))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _media_facts(meta: dict) -> dict:
|
|
media = meta.get("Media") or []
|
|
if not media:
|
|
return {}
|
|
m = media[0]
|
|
part = (m.get("Part") or [{}])[0]
|
|
return {
|
|
"container": m.get("container") or part.get("container"),
|
|
"codec_video": m.get("videoCodec"),
|
|
"codec_audio": m.get("audioCodec"),
|
|
"size_bytes": part.get("size"),
|
|
"file_path": part.get("file"),
|
|
"part_key": part.get("key"),
|
|
}
|
|
|
|
|
|
def _dur_s(meta: dict) -> int | None:
|
|
d = meta.get("duration")
|
|
return int(d / 1000) if d else None
|
|
|
|
|
|
def _enabled_section_keys(db: Session) -> set[str] | None:
|
|
"""The configured allowlist of section keys, or None to mean 'all movie/show sections'."""
|
|
raw = sysconfig.get_str(db, "plex_libraries")
|
|
keys = {k.strip() for k in raw.split(",") if k.strip()}
|
|
return keys or None
|
|
|
|
|
|
def sync(db: Session) -> dict:
|
|
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
|
return {"skipped": "disabled"}
|
|
stats = {"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0, "collections": 0}
|
|
wanted = _enabled_section_keys(db)
|
|
try:
|
|
with PlexClient(db) as plex:
|
|
for s in plex.sections():
|
|
if s.get("type") not in ("movie", "show"):
|
|
continue
|
|
key = str(s.get("key"))
|
|
if wanted is not None and key not in wanted:
|
|
continue
|
|
lib = _upsert_library(db, key, s.get("title") or key, s.get("type"))
|
|
stats["libraries"] += 1
|
|
if s.get("type") == "movie":
|
|
_sync_movies(db, plex, lib, stats)
|
|
else:
|
|
_sync_shows(db, plex, lib, stats)
|
|
_sync_collections(db, plex, lib, stats)
|
|
# Commit each section as it completes: a PlexError on a LATER section then keeps the
|
|
# libraries already mirrored (idempotent — the next run reconciles the rest), instead
|
|
# of the old single end-of-loop commit that a late failure rolled back entirely.
|
|
db.commit()
|
|
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
|
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
|
try:
|
|
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
|
except Exception as e: # best-effort — never fail the sync over the backfill
|
|
log.warning("Plex people enrich batch failed: %s", e)
|
|
except PlexNotConfigured as e:
|
|
return {"skipped": str(e)}
|
|
except PlexError as e:
|
|
db.rollback()
|
|
log.warning("Plex sync failed: %s", e)
|
|
return {"error": str(e)}
|
|
# Prune the headshot cache: a reconcile may have removed titles, orphaning people who no longer
|
|
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
|
try:
|
|
stats["people_pruned"] = people.prune_orphan_people(db)
|
|
except Exception as e: # best-effort housekeeping — never fail the sync over it
|
|
log.warning("Plex people prune failed: %s", e)
|
|
log.info("Plex sync done: %s", stats)
|
|
return stats
|
|
|
|
|
|
# How many titles to rich-fetch per sync run for the cast backfill — bounds the extra Plex load; the
|
|
# whole library is covered over successive runs (each fetched item is stamped people_enriched_at once).
|
|
_ENRICH_BATCH = 60
|
|
|
|
|
|
def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BATCH) -> int:
|
|
"""Rich-fetch the FULL cast of up to `limit` not-yet-enriched movies/shows and persist it (via the
|
|
shared enrich_row_people). Oldest-id first, movies before shows. Stamps people_enriched_at only on
|
|
a SUCCESSFUL fetch, so a transient per-item failure retries next run rather than being lost.
|
|
Returns how many items were enriched this run."""
|
|
movies = (
|
|
db.query(PlexItem)
|
|
.filter(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None))
|
|
.order_by(PlexItem.id)
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
remaining = limit - len(movies)
|
|
shows = (
|
|
db.query(PlexShow)
|
|
.filter(PlexShow.people_enriched_at.is_(None))
|
|
.order_by(PlexShow.id)
|
|
.limit(remaining)
|
|
.all()
|
|
if remaining > 0
|
|
else []
|
|
)
|
|
now = datetime.now(timezone.utc)
|
|
n = 0
|
|
for row in [*movies, *shows]:
|
|
try:
|
|
meta = plex.metadata(row.rating_key) or {}
|
|
except Exception as e:
|
|
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
|
|
continue # leave people_enriched_at NULL so it's retried on a later run
|
|
people.enrich_row_people(db, row, people.cast_from_meta(meta))
|
|
row.people_enriched_at = now
|
|
n += 1
|
|
db.commit()
|
|
return n
|
|
|
|
|
|
def _upsert_library(db: Session, key: str, title: str, kind: str) -> PlexLibrary:
|
|
lib = db.query(PlexLibrary).filter_by(plex_key=key).first()
|
|
if lib is None:
|
|
lib = PlexLibrary(plex_key=key)
|
|
db.add(lib)
|
|
lib.title = title
|
|
lib.kind = kind
|
|
lib.synced_at = datetime.now(timezone.utc)
|
|
db.flush()
|
|
return lib
|
|
|
|
|
|
def _paginate(plex: PlexClient, key: str, item_type: int):
|
|
start = 0
|
|
while True:
|
|
items, total = plex.section_items(key, item_type, start, _PAGE)
|
|
if not items:
|
|
break
|
|
for it in items:
|
|
yield it
|
|
start += len(items)
|
|
if total and start >= total:
|
|
break
|
|
if len(items) < _PAGE:
|
|
break
|
|
|
|
|
|
def _apply_facet_fields(row, meta: dict) -> None:
|
|
"""Set the filterable/orderable metadata columns shared by plex_items + plex_shows (identical
|
|
column names) from the cheap section listing: rating, content_rating, studio, release date,
|
|
genres/directors/cast, and the searchable people blob folded into search_vector at sync time."""
|
|
row.rating = _rating(meta)
|
|
row.content_rating = (meta.get("contentRating") or None)
|
|
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
|
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
|
row.genres = _tags(meta, "Genre")
|
|
row.directors = _tags(meta, "Director")
|
|
row.cast_names = _tags(meta, "Role", limit=20)
|
|
people = list(dict.fromkeys((row.directors or []) + (row.cast_names or [])))
|
|
row.people_text = " ".join(people) or None
|
|
|
|
|
|
def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
|
row.library_id = lib_id
|
|
row.kind = kind
|
|
row.title = (meta.get("title") or "").strip() or "Untitled"
|
|
row.summary = meta.get("summary")
|
|
row.year = meta.get("year")
|
|
row.duration_s = _dur_s(meta)
|
|
row.thumb_key = meta.get("thumb")
|
|
row.art_key = meta.get("art") or meta.get("grandparentArt")
|
|
row.added_at = _epoch(meta.get("addedAt"))
|
|
_apply_facet_fields(row, meta) # filterable/orderable extras (shared with shows)
|
|
mf = _media_facts(meta)
|
|
row.file_path = mf.get("file_path")
|
|
row.part_key = mf.get("part_key")
|
|
row.container = mf.get("container")
|
|
row.codec_video = mf.get("codec_video")
|
|
row.codec_audio = mf.get("codec_audio")
|
|
row.size_bytes = mf.get("size_bytes")
|
|
row.playable = classify_playable(mf.get("codec_video"), mf.get("codec_audio"), mf.get("container"))
|
|
|
|
|
|
def _sync_movies(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
|
existing = {r.rating_key: r for r in db.query(PlexItem).filter_by(library_id=lib.id, kind="movie")}
|
|
for meta in _paginate(plex, lib.plex_key, _TYPE_MOVIE):
|
|
rk = str(meta.get("ratingKey") or "")
|
|
if not rk:
|
|
continue
|
|
row = existing.get(rk) or PlexItem(rating_key=rk)
|
|
if row.id is None:
|
|
db.add(row)
|
|
_apply_item(row, lib.id, "movie", meta)
|
|
stats["movies"] += 1
|
|
db.flush()
|
|
|
|
|
|
def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
|
# 1) shows
|
|
shows = {r.rating_key: r for r in db.query(PlexShow).filter_by(library_id=lib.id)}
|
|
for meta in _paginate(plex, lib.plex_key, _TYPE_SHOW):
|
|
rk = str(meta.get("ratingKey") or "")
|
|
if not rk:
|
|
continue
|
|
sh = shows.get(rk) or PlexShow(rating_key=rk)
|
|
if sh.id is None:
|
|
db.add(sh)
|
|
shows[rk] = sh
|
|
sh.library_id = lib.id
|
|
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
|
sh.summary = meta.get("summary")
|
|
sh.year = meta.get("year")
|
|
sh.thumb_key = meta.get("thumb")
|
|
sh.art_key = meta.get("art")
|
|
sh.child_count = meta.get("childCount")
|
|
sh.added_at = _epoch(meta.get("addedAt"))
|
|
_apply_facet_fields(sh, meta) # same filterable/orderable tags as movies
|
|
stats["shows"] += 1
|
|
db.flush()
|
|
show_id = {rk: sh.id for rk, sh in shows.items()}
|
|
|
|
# 2) seasons
|
|
seasons = {r.rating_key: r for r in db.query(PlexSeason).join(PlexShow).filter(PlexShow.library_id == lib.id)}
|
|
for meta in _paginate(plex, lib.plex_key, _TYPE_SEASON):
|
|
rk = str(meta.get("ratingKey") or "")
|
|
sid = show_id.get(str(meta.get("parentRatingKey") or ""))
|
|
if not rk or sid is None:
|
|
continue
|
|
se = seasons.get(rk) or PlexSeason(rating_key=rk)
|
|
if se.id is None:
|
|
db.add(se)
|
|
seasons[rk] = se
|
|
se.show_id = sid
|
|
se.title = meta.get("title")
|
|
se.season_number = meta.get("index")
|
|
se.thumb_key = meta.get("thumb")
|
|
stats["seasons"] += 1
|
|
db.flush()
|
|
season_id = {rk: se.id for rk, se in seasons.items()}
|
|
|
|
# 3) episodes (playable leaves)
|
|
episodes = {r.rating_key: r for r in db.query(PlexItem).filter_by(library_id=lib.id, kind="episode")}
|
|
for meta in _paginate(plex, lib.plex_key, _TYPE_EPISODE):
|
|
rk = str(meta.get("ratingKey") or "")
|
|
if not rk:
|
|
continue
|
|
row = episodes.get(rk) or PlexItem(rating_key=rk)
|
|
if row.id is None:
|
|
db.add(row)
|
|
_apply_item(row, lib.id, "episode", meta)
|
|
row.show_id = show_id.get(str(meta.get("grandparentRatingKey") or ""))
|
|
row.season_id = season_id.get(str(meta.get("parentRatingKey") or ""))
|
|
row.season_number = meta.get("parentIndex")
|
|
row.episode_number = meta.get("index")
|
|
stats["episodes"] += 1
|
|
db.flush()
|
|
|
|
|
|
def _upsert_collection(
|
|
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
|
) -> PlexCollection:
|
|
"""Get-or-create a PlexCollection row and apply its metadata from a Plex `meta` dict. `editable`
|
|
is user-managed and preserved (never set here). Shared by the full sync + the targeted re-sync."""
|
|
col = existing or PlexCollection(rating_key=rk)
|
|
if col.id is None:
|
|
db.add(col)
|
|
col.library_id = lib_id
|
|
col.title = (meta.get("title") or "").strip() or "Untitled"
|
|
col.summary = meta.get("summary")
|
|
col.thumb_key = meta.get("thumb")
|
|
col.child_count = meta.get("childCount")
|
|
col.smart = str(meta.get("smart")) == "1"
|
|
col.synced_at = datetime.now(timezone.utc)
|
|
return col
|
|
|
|
|
|
def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str) -> PlexCollection:
|
|
"""Targeted re-sync of ONE collection after a write-back (create/add/remove/rename) — avoids the
|
|
full ~4-min library sync. Upserts the row (preserving `editable`) and reconciles ONLY this
|
|
collection's membership on the affected member rows."""
|
|
meta = plex.metadata(rk) or {}
|
|
col = _upsert_collection(
|
|
db, db.query(PlexCollection).filter_by(rating_key=rk).first(), rk, lib.id, meta
|
|
)
|
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
|
new_members = {str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")}
|
|
current = db.query(Model).filter(Model.collection_keys.contains([rk])).all()
|
|
for row in current: # drop rk from rows that are no longer members
|
|
if row.rating_key not in new_members:
|
|
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
|
|
to_add = new_members - {r.rating_key for r in current}
|
|
if to_add:
|
|
for row in db.query(Model).filter(Model.rating_key.in_(to_add)):
|
|
row.collection_keys = list(dict.fromkeys([*(row.collection_keys or []), rk]))
|
|
db.flush()
|
|
return col
|
|
|
|
|
|
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
|
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
|
for row in db.query(Model).filter(Model.collection_keys.contains([rk])):
|
|
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
|
|
col = db.query(PlexCollection).filter_by(rating_key=rk).first()
|
|
if col is not None:
|
|
db.delete(col)
|
|
db.flush()
|
|
|
|
|
|
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
|
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
|
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
|
collections gone from Plex are pruned, and each member row's keys are rebuilt from scratch."""
|
|
cols = plex.collections(lib.plex_key)
|
|
existing = {c.rating_key: c for c in db.query(PlexCollection).filter_by(library_id=lib.id)}
|
|
seen: set[str] = set()
|
|
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
|
for meta in cols:
|
|
rk = str(meta.get("ratingKey") or "")
|
|
if not rk:
|
|
continue
|
|
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
|
seen.add(rk)
|
|
for child in plex.collection_children(rk):
|
|
crk = str(child.get("ratingKey") or "")
|
|
if crk:
|
|
membership.setdefault(crk, []).append(rk)
|
|
stats["collections"] += 1
|
|
db.flush()
|
|
for rk, col in existing.items():
|
|
if rk not in seen:
|
|
db.delete(col)
|
|
# Rebuild collection_keys on every member row of this library (clear + set).
|
|
for row in db.query(PlexItem).filter_by(library_id=lib.id):
|
|
row.collection_keys = membership.get(row.rating_key) or None
|
|
for row in db.query(PlexShow).filter_by(library_id=lib.id):
|
|
row.collection_keys = membership.get(row.rating_key) or None
|
|
db.flush()
|