refactor: one definition of a followed channel and of a choice spelling

The explore purge rebuilt followed_channel's two EXISTS clauses inline, so what is polled and what may be deleted could drift; it now uses the predicate, which moved to sync/explore.py. The env validator and sysconfig each folded a choice value on their own; both call config.canonical_choice. The deep-backfill comment no longer calls recent backfill unconditional.
This commit is contained in:
2026-09-17 03:53:29 +02:00
parent 86bc008a81
commit 4bfba4aa86
5 changed files with 40 additions and 32 deletions
+9 -2
View File
@@ -40,6 +40,14 @@ def choices_of(annotation: object) -> tuple[str, ...]:
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else () return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
API, so a spelling one accepts the other cannot refuse."""
folded = str(raw).strip().casefold()
return next((option for option in options if option.casefold() == folded), None)
class Settings(BaseSettings): class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore") model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -370,8 +378,7 @@ class Settings(BaseSettings):
field = cls.model_fields.get(info.field_name or "") field = cls.model_fields.get(info.field_name or "")
options = choices_of(field.annotation) if field else () options = choices_of(field.annotation) if field else ()
if options and isinstance(value, str): if options and isinstance(value, str):
folded = value.strip().casefold() return canonical_choice(options, value) or value
return next((o for o in options if o.casefold() == folded), value)
return value return value
@model_validator(mode="after") @model_validator(mode="after")
+2 -1
View File
@@ -22,7 +22,8 @@ from app.scheduler import (
trigger_all, trigger_all,
trigger_job, trigger_job,
) )
from app.sync.runner import estimate_deep_backfill, followed_channel from app.sync.explore import followed_channel
from app.sync.runner import estimate_deep_backfill
from app.routes._common import not_found from app.routes._common import not_found
router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"]) router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"])
+19 -6
View File
@@ -11,7 +11,7 @@ grace period (mirrors the maintenance job's grace-then-delete safety)."""
import logging import logging
from datetime import timedelta from datetime import timedelta
from sqlalchemy import delete, select from sqlalchemy import delete, or_, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import sysconfig from app import sysconfig
@@ -31,6 +31,23 @@ from app.youtube.client import YouTubeClient
log = logging.getLogger("siftlode.explore") log = logging.getLogger("siftlode.explore")
def followed_channel():
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
grace window (`purge_unkept_explored` deletes the expired ExploredChannel rows first). ONE
definition for both halves of the question: what the scheduled RSS poll and recent backfill cover
(sync/runner.py) and what the explore purge may delete.
The poll and backfill cover only these (owner decision, 2026-09-17). They used to cover EVERY
channel row, but rows also come from search results, playlist sync and dropped subscriptions, and
nothing deletes those: on prod 582 of 979 polled channels had no follower, three of them dead
feeds answering 404 on every poll. The cost of narrowing is deliberate: the shared Library stops
receiving new uploads from channels nobody follows."""
return or_(
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
)
def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int: def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
"""Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50).""" """Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50)."""
if not video_ids: if not video_ids:
@@ -98,12 +115,8 @@ def purge_unkept_explored(db: Session) -> dict:
).rowcount ).rowcount
db.commit() db.commit()
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
orphan_ids = ( orphan_ids = (
db.execute( db.execute(select(Channel.id).where(Channel.from_explore.is_(True), ~followed_channel()))
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
)
.scalars() .scalars()
.all() .all()
) )
+5 -17
View File
@@ -10,14 +10,15 @@ import threading
from collections import Counter from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import or_, select from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import logs, progress, quota, sysconfig from app import logs, progress, quota, sysconfig
log = logging.getLogger("siftlode.sync") log = logging.getLogger("siftlode.sync")
from app.db import SessionLocal from app.db import SessionLocal
from app.models import Channel, ExploredChannel, OAuthToken, Subscription, User from app.models import Channel, OAuthToken, Subscription, User
from app.sync.explore import followed_channel
from app.sync.subscriptions import import_subscriptions from app.sync.subscriptions import import_subscriptions
from app.sync.videos import ( from app.sync.videos import (
apply_rss_feed, apply_rss_feed,
@@ -54,20 +55,6 @@ def get_service_user(db: Session) -> User | None:
return users[0] if users else None return users[0] if users else None
def followed_channel():
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
grace window. The scheduled RSS poll and recent backfill cover only these (owner decision,
2026-09-17). They used to cover EVERY channel row, but rows also come from search results,
playlist sync and dropped subscriptions, and nothing deletes those: on prod 582 of 979 polled
channels had no follower, three of them dead feeds answering 404 on every poll. The cost of
narrowing is deliberate: the shared Library stops receiving new uploads from channels nobody
follows."""
return or_(
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
)
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]: def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
"""Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did """Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler _not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
@@ -258,7 +245,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
if user is None: if user is None:
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"} return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
# Demand-driven: only deep-backfill channels at least one user has opted into # Demand-driven: only deep-backfill channels at least one user has opted into
# (Subscription.deep_requested). Recent backfill stays unconditional (cheap). # (Subscription.deep_requested). Recent backfill needs no opt-in: it covers every followed channel
# (`followed_channel`).
deep_requested = ( deep_requested = (
select(Subscription.id) select(Subscription.id)
.where( .where(
+5 -6
View File
@@ -14,7 +14,7 @@ from dataclasses import dataclass
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import security from app import security
from app.config import Settings, choices_of, settings from app.config import Settings, canonical_choice, choices_of, settings
from app.models import SystemConfig from app.models import SystemConfig
log = logging.getLogger("siftlode.config") log = logging.getLogger("siftlode.config")
@@ -210,11 +210,10 @@ def _default(s: ConfigSpec):
def _choice(s: ConfigSpec, raw: object) -> str: def _choice(s: ConfigSpec, raw: object) -> str:
"""The canonical spelling of `raw` among the key's choices (any letter case), or ValueError.""" """The canonical spelling of `raw` among the key's choices (any letter case), or ValueError."""
folded = str(raw).strip().casefold() option = canonical_choice(s.choices, raw)
for option in s.choices: if option is None:
if option.casefold() == folded: raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}")
return option return option
raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}")
def _coerce(s: ConfigSpec, raw: str): def _coerce(s: ConfigSpec, raw: str):