From 4bfba4aa86cb807e2223c5fad388fcfacabb49a2 Mon Sep 17 00:00:00 2001 From: npeter83 Date: Thu, 17 Sep 2026 03:53:29 +0200 Subject: [PATCH] refactor: one definition of a followed channel and of a choice spelling The explore purge rebuilt followed_channel's two EXISTS clauses inline, so what is polled and what may be deleted could drift; it now uses the predicate, which moved to sync/explore.py. The env validator and sysconfig each folded a choice value on their own; both call config.canonical_choice. The deep-backfill comment no longer calls recent backfill unconditional. --- backend/app/config.py | 11 +++++++++-- backend/app/routes/scheduler.py | 3 ++- backend/app/sync/explore.py | 25 +++++++++++++++++++------ backend/app/sync/runner.py | 22 +++++----------------- backend/app/sysconfig.py | 11 +++++------ 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 24b58dc5..d8782e2f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -40,6 +40,14 @@ def choices_of(annotation: object) -> tuple[str, ...]: return tuple(get_args(annotation)) if get_origin(annotation) is Literal else () +def canonical_choice(options: tuple[str, ...], raw: object) -> str | None: + """The option `raw` names, in its canonical spelling (any letter case, surrounding space + ignored), or None. The ONE matching rule for a choice value from the env and from the admin + API, so a spelling one accepts the other cannot refuse.""" + folded = str(raw).strip().casefold() + return next((option for option in options if option.casefold() == folded), None) + + class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") @@ -370,8 +378,7 @@ class Settings(BaseSettings): field = cls.model_fields.get(info.field_name or "") options = choices_of(field.annotation) if field else () if options and isinstance(value, str): - folded = value.strip().casefold() - return next((o for o in options if o.casefold() == folded), value) + return canonical_choice(options, value) or value return value @model_validator(mode="after") diff --git a/backend/app/routes/scheduler.py b/backend/app/routes/scheduler.py index 60489bbe..0683afaa 100644 --- a/backend/app/routes/scheduler.py +++ b/backend/app/routes/scheduler.py @@ -22,7 +22,8 @@ from app.scheduler import ( trigger_all, trigger_job, ) -from app.sync.runner import estimate_deep_backfill, followed_channel +from app.sync.explore import followed_channel +from app.sync.runner import estimate_deep_backfill from app.routes._common import not_found router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"]) diff --git a/backend/app/sync/explore.py b/backend/app/sync/explore.py index 894eabfb..e89feb42 100644 --- a/backend/app/sync/explore.py +++ b/backend/app/sync/explore.py @@ -11,7 +11,7 @@ grace period (mirrors the maintenance job's grace-then-delete safety).""" import logging from datetime import timedelta -from sqlalchemy import delete, select +from sqlalchemy import delete, or_, select from sqlalchemy.orm import Session from app import sysconfig @@ -31,6 +31,23 @@ from app.youtube.client import YouTubeClient log = logging.getLogger("siftlode.explore") +def followed_channel(): + """WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its + grace window (`purge_unkept_explored` deletes the expired ExploredChannel rows first). ONE + definition for both halves of the question: what the scheduled RSS poll and recent backfill cover + (sync/runner.py) and what the explore purge may delete. + + The poll and backfill cover only these (owner decision, 2026-09-17). They used to cover EVERY + channel row, but rows also come from search results, playlist sync and dropped subscriptions, and + nothing deletes those: on prod 582 of 979 polled channels had no follower, three of them dead + feeds answering 404 on every poll. The cost of narrowing is deliberate: the shared Library stops + receiving new uploads from channels nobody follows.""" + return or_( + select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(), + select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(), + ) + + def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int: """Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50).""" if not video_ids: @@ -98,12 +115,8 @@ def purge_unkept_explored(db: Session) -> dict: ).rowcount db.commit() - sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists() - exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists() orphan_ids = ( - db.execute( - select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists) - ) + db.execute(select(Channel.id).where(Channel.from_explore.is_(True), ~followed_channel())) .scalars() .all() ) diff --git a/backend/app/sync/runner.py b/backend/app/sync/runner.py index 24111d77..948c92f5 100644 --- a/backend/app/sync/runner.py +++ b/backend/app/sync/runner.py @@ -10,14 +10,15 @@ import threading from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed -from sqlalchemy import or_, select +from sqlalchemy import select from sqlalchemy.orm import Session from app import logs, progress, quota, sysconfig log = logging.getLogger("siftlode.sync") from app.db import SessionLocal -from app.models import Channel, ExploredChannel, OAuthToken, Subscription, User +from app.models import Channel, OAuthToken, Subscription, User +from app.sync.explore import followed_channel from app.sync.subscriptions import import_subscriptions from app.sync.videos import ( apply_rss_feed, @@ -54,20 +55,6 @@ def get_service_user(db: Session) -> User | None: return users[0] if users else None -def followed_channel(): - """WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its - grace window. The scheduled RSS poll and recent backfill cover only these (owner decision, - 2026-09-17). They used to cover EVERY channel row, but rows also come from search results, - playlist sync and dropped subscriptions, and nothing deletes those: on prod 582 of 979 polled - channels had no follower, three of them dead feeds answering 404 on every poll. The cost of - narrowing is deliberate: the shared Library stops receiving new uploads from channels nobody - follows.""" - return or_( - select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(), - select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(), - ) - - def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]: """Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did _not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler @@ -258,7 +245,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) - if user is None: return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"} # Demand-driven: only deep-backfill channels at least one user has opted into - # (Subscription.deep_requested). Recent backfill stays unconditional (cheap). + # (Subscription.deep_requested). Recent backfill needs no opt-in: it covers every followed channel + # (`followed_channel`). deep_requested = ( select(Subscription.id) .where( diff --git a/backend/app/sysconfig.py b/backend/app/sysconfig.py index 20549187..016080be 100644 --- a/backend/app/sysconfig.py +++ b/backend/app/sysconfig.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from sqlalchemy.orm import Session from app import security -from app.config import Settings, choices_of, settings +from app.config import Settings, canonical_choice, choices_of, settings from app.models import SystemConfig log = logging.getLogger("siftlode.config") @@ -210,11 +210,10 @@ def _default(s: ConfigSpec): def _choice(s: ConfigSpec, raw: object) -> str: """The canonical spelling of `raw` among the key's choices (any letter case), or ValueError.""" - folded = str(raw).strip().casefold() - for option in s.choices: - if option.casefold() == folded: - return option - raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}") + option = canonical_choice(s.choices, raw) + if option is None: + raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}") + return option def _coerce(s: ConfigSpec, raw: str):