style: apply ruff format across backend + scripts (line-length 100)
Mechanical, whole-tree `ruff format` pass — no behaviour change. Recorded in .git-blame-ignore-revs (next commit) so git blame skips it.
This commit is contained in:
@@ -5,6 +5,7 @@ detection over a sample of recent video titles. Topics are mapped from YouTube's
|
||||
topicDetails categories and the channel's dominant video category. System tags are
|
||||
regenerated freely; user tags are never touched here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
@@ -221,9 +222,7 @@ def apply_channel_autotags(db: Session, channel: Channel) -> None:
|
||||
desired[tag.id] = confidence
|
||||
|
||||
for topic in compute_channel_topics(db, channel):
|
||||
tag = ensure_system_tag(
|
||||
db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR)
|
||||
)
|
||||
tag = ensure_system_tag(db, topic, "topic", TOPIC_COLORS.get(topic, DEFAULT_TOPIC_COLOR))
|
||||
desired.setdefault(tag.id, None)
|
||||
|
||||
existing = {
|
||||
|
||||
@@ -7,6 +7,7 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
|
||||
— so one user's curiosity never floods everyone's catalog. Subscribing is the "keep" signal;
|
||||
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
||||
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
@@ -35,11 +36,7 @@ def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
if not video_ids:
|
||||
return 0
|
||||
videos = (
|
||||
db.execute(
|
||||
select(Video).where(
|
||||
Video.id.in_(video_ids), Video.enriched_at.is_(None)
|
||||
)
|
||||
)
|
||||
db.execute(select(Video).where(Video.id.in_(video_ids), Video.enriched_at.is_(None)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
@@ -101,17 +98,11 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
).rowcount
|
||||
db.commit()
|
||||
|
||||
sub_exists = (
|
||||
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
|
||||
)
|
||||
exp_exists = (
|
||||
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
|
||||
)
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
|
||||
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
|
||||
orphan_ids = (
|
||||
db.execute(
|
||||
select(Channel.id).where(
|
||||
Channel.from_explore.is_(True), ~sub_exists, ~exp_exists
|
||||
)
|
||||
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -158,9 +149,7 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
|
||||
cutoff = _now() - timedelta(days=grace)
|
||||
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
|
||||
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
|
||||
sub_exists = (
|
||||
select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
|
||||
)
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
|
||||
deleted = db.execute(
|
||||
delete(Video).where(
|
||||
Video.via_search.is_(True),
|
||||
|
||||
@@ -19,6 +19,7 @@ flag an `upcoming` premiere that blew past its scheduled start and never went li
|
||||
Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect the same
|
||||
backfill reserve as the other jobs so interactive syncs never starve.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
@@ -132,11 +133,7 @@ def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int:
|
||||
|
||||
def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
||||
"""Phase 1: recover or hard-delete videos already flagged unavailable."""
|
||||
flagged = (
|
||||
db.execute(select(Video).where(Video.unavailable_since.is_not(None)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
|
||||
if not flagged:
|
||||
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
|
||||
progress.report(0, len(flagged), "recheck")
|
||||
@@ -181,7 +178,9 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
||||
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
||||
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
||||
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
||||
batch = sysconfig.get_int(db, "maintenance_revalidate_batch") # admin override, else config default
|
||||
batch = sysconfig.get_int(
|
||||
db, "maintenance_revalidate_batch"
|
||||
) # admin override, else config default
|
||||
flagged_new = 0
|
||||
checked = 0
|
||||
now = _now()
|
||||
|
||||
@@ -5,6 +5,7 @@ edits out to YouTube (`plan_push` / `push_playlist`), behind the optional write
|
||||
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
||||
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
||||
ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
@@ -59,9 +60,7 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
|
||||
the API won't return are simply left out."""
|
||||
if not video_ids:
|
||||
return
|
||||
have = set(
|
||||
db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all()
|
||||
)
|
||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||
missing = [v for v in dict.fromkeys(video_ids) if v not in have]
|
||||
if not missing:
|
||||
return
|
||||
@@ -109,9 +108,7 @@ def _replace_items_from_youtube(
|
||||
ordered ids."""
|
||||
_ensure_videos(db, yt, ids)
|
||||
present = (
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all())
|
||||
if ids
|
||||
else set()
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
||||
)
|
||||
ordered: list[str] = []
|
||||
seen: set[str] = set()
|
||||
@@ -139,9 +136,7 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
existing = {
|
||||
pl.yt_playlist_id: pl
|
||||
for pl in db.execute(
|
||||
select(Playlist).where(
|
||||
Playlist.user_id == user.id, Playlist.source == "youtube"
|
||||
)
|
||||
select(Playlist).where(Playlist.user_id == user.id, Playlist.source == "youtube")
|
||||
).scalars()
|
||||
}
|
||||
# YouTube playlists that we already own as local (exported) playlists must not be
|
||||
|
||||
@@ -4,6 +4,7 @@ Channel/video data is shared across users, so background work acts through a "se
|
||||
user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configured for
|
||||
public reads.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
@@ -173,9 +174,7 @@ def run_recent_backfill(
|
||||
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
||||
if channels is None:
|
||||
channels = (
|
||||
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None)))
|
||||
.scalars()
|
||||
.all()
|
||||
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None))).scalars().all()
|
||||
)
|
||||
if max_channels:
|
||||
channels = channels[:max_channels]
|
||||
@@ -325,7 +324,9 @@ def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
|
||||
total_units = sum(units(c) for c in pending)
|
||||
daily_budget = max(
|
||||
1, sysconfig.get_int(db, "quota_daily_budget") - sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
1,
|
||||
sysconfig.get_int(db, "quota_daily_budget")
|
||||
- sysconfig.get_int(db, "backfill_quota_reserve"),
|
||||
)
|
||||
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -57,9 +58,7 @@ def apply_channel_details(db: Session, items: list[dict]) -> None:
|
||||
channel.title = snippet.get("title") or channel.title
|
||||
channel.handle = snippet.get("customUrl")
|
||||
channel.description = normalize_description(snippet.get("description"))
|
||||
channel.thumbnail_url = (
|
||||
best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
||||
)
|
||||
channel.thumbnail_url = best_thumbnail(snippet.get("thumbnails")) or channel.thumbnail_url
|
||||
channel.default_language = snippet.get("defaultLanguage")
|
||||
channel.country = snippet.get("country")
|
||||
channel.uploads_playlist_id = content.get("relatedPlaylists", {}).get("uploads")
|
||||
@@ -125,16 +124,15 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
# (see `should_prune`).
|
||||
removed = 0
|
||||
existing = (
|
||||
db.execute(select(Subscription).where(Subscription.user_id == user.id))
|
||||
.scalars()
|
||||
.all()
|
||||
db.execute(select(Subscription).where(Subscription.user_id == user.id)).scalars().all()
|
||||
)
|
||||
prune_skipped = not should_prune(len(fetched), len(existing))
|
||||
if prune_skipped:
|
||||
log.warning(
|
||||
"Subscription import (user %s): YouTube returned 0 subscriptions while %d exist "
|
||||
"locally — skipping the prune (treating it as a transient empty response)",
|
||||
user.id, len(existing),
|
||||
user.id,
|
||||
len(existing),
|
||||
)
|
||||
else:
|
||||
for sub_row in existing:
|
||||
|
||||
+11
-19
@@ -1,6 +1,7 @@
|
||||
"""Video ingestion: free RSS detection, recent-first (and deep) backfill from the
|
||||
uploads playlist, and enrichment via videos.list (duration, stats, category, Shorts
|
||||
and livestream classification)."""
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -16,9 +17,7 @@ from app.titles import normalize_title
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
_DURATION_RE = re.compile(
|
||||
r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE
|
||||
)
|
||||
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -177,9 +176,7 @@ def backfill_channel_deep(
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
rows = [
|
||||
s
|
||||
for s in (
|
||||
stub_from_playlist_item(it, channel.id) for it in data.get("items", [])
|
||||
)
|
||||
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
||||
if s is not None
|
||||
]
|
||||
total += insert_stubs(db, rows)
|
||||
@@ -201,11 +198,7 @@ def reconcile_full_history(db: Session) -> int:
|
||||
deep-requested, so the demand-driven deep job never ran). Without this such a channel
|
||||
sits in 'needs full history' forever despite having every video. Idempotent, no API
|
||||
quota; returns the number of channels flipped."""
|
||||
stored = (
|
||||
select(func.count(Video.id))
|
||||
.where(Video.channel_id == Channel.id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
stored = select(func.count(Video.id)).where(Video.channel_id == Channel.id).scalar_subquery()
|
||||
result = db.execute(
|
||||
update(Channel)
|
||||
.where(
|
||||
@@ -263,9 +256,7 @@ def apply_video_details(video: Video, item: dict) -> None:
|
||||
tags = snippet.get("tags")
|
||||
video.keywords = " ".join(tags) if tags else None
|
||||
video.topic_categories = topics.get("topicCategories")
|
||||
video.default_language = snippet.get("defaultLanguage") or snippet.get(
|
||||
"defaultAudioLanguage"
|
||||
)
|
||||
video.default_language = snippet.get("defaultLanguage") or snippet.get("defaultAudioLanguage")
|
||||
video.live_status = _live_status(snippet, live)
|
||||
# status part: used by the maintenance/validation job to judge playability.
|
||||
if status:
|
||||
@@ -309,9 +300,7 @@ def _mark_enriched(video: Video) -> None:
|
||||
def enrich_pending(db: Session, yt: YouTubeClient, limit: int | None = None) -> int:
|
||||
limit = limit or sysconfig.get_int(db, "enrich_batch_size")
|
||||
videos = (
|
||||
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit))
|
||||
.scalars()
|
||||
.all()
|
||||
db.execute(select(Video).where(Video.enriched_at.is_(None)).limit(limit)).scalars().all()
|
||||
)
|
||||
return _apply_video_batch(db, yt, list(videos), _mark_enriched)
|
||||
|
||||
@@ -377,12 +366,14 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
# 2) Probe the remaining candidates.
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(Video).where(
|
||||
select(Video)
|
||||
.where(
|
||||
Video.shorts_probed.is_(False),
|
||||
Video.enriched_at.is_not(None),
|
||||
Video.duration_seconds <= probe_max,
|
||||
Video.live_status == "none",
|
||||
).limit(limit)
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -393,6 +384,7 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
probed = 0
|
||||
shorts = 0
|
||||
with make_client() as client:
|
||||
|
||||
def work(video: Video):
|
||||
return video, probe_is_short(client, video.id)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user