refactor: one definition of a followed channel and of a choice spelling
The explore purge rebuilt followed_channel's two EXISTS clauses inline, so what is polled and what may be deleted could drift; it now uses the predicate, which moved to sync/explore.py. The env validator and sysconfig each folded a choice value on their own; both call config.canonical_choice. The deep-backfill comment no longer calls recent backfill unconditional.
This commit is contained in:
@@ -40,6 +40,14 @@ def choices_of(annotation: object) -> tuple[str, ...]:
|
||||
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
|
||||
|
||||
|
||||
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
|
||||
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
|
||||
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
|
||||
API, so a spelling one accepts the other cannot refuse."""
|
||||
folded = str(raw).strip().casefold()
|
||||
return next((option for option in options if option.casefold() == folded), None)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -370,8 +378,7 @@ class Settings(BaseSettings):
|
||||
field = cls.model_fields.get(info.field_name or "")
|
||||
options = choices_of(field.annotation) if field else ()
|
||||
if options and isinstance(value, str):
|
||||
folded = value.strip().casefold()
|
||||
return next((o for o in options if o.casefold() == folded), value)
|
||||
return canonical_choice(options, value) or value
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
|
||||
@@ -22,7 +22,8 @@ from app.scheduler import (
|
||||
trigger_all,
|
||||
trigger_job,
|
||||
)
|
||||
from app.sync.runner import estimate_deep_backfill, followed_channel
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"])
|
||||
|
||||
@@ -11,7 +11,7 @@ grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
@@ -31,6 +31,23 @@ from app.youtube.client import YouTubeClient
|
||||
log = logging.getLogger("siftlode.explore")
|
||||
|
||||
|
||||
def followed_channel():
|
||||
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
|
||||
grace window (`purge_unkept_explored` deletes the expired ExploredChannel rows first). ONE
|
||||
definition for both halves of the question: what the scheduled RSS poll and recent backfill cover
|
||||
(sync/runner.py) and what the explore purge may delete.
|
||||
|
||||
The poll and backfill cover only these (owner decision, 2026-09-17). They used to cover EVERY
|
||||
channel row, but rows also come from search results, playlist sync and dropped subscriptions, and
|
||||
nothing deletes those: on prod 582 of 979 polled channels had no follower, three of them dead
|
||||
feeds answering 404 on every poll. The cost of narrowing is deliberate: the shared Library stops
|
||||
receiving new uploads from channels nobody follows."""
|
||||
return or_(
|
||||
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
|
||||
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
|
||||
)
|
||||
|
||||
|
||||
def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
"""Enrich just the not-yet-enriched videos among `video_ids` (videos.list, 1 unit/50)."""
|
||||
if not video_ids:
|
||||
@@ -98,12 +115,8 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
).rowcount
|
||||
db.commit()
|
||||
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Channel.id).exists()
|
||||
exp_exists = select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists()
|
||||
orphan_ids = (
|
||||
db.execute(
|
||||
select(Channel.id).where(Channel.from_explore.is_(True), ~sub_exists, ~exp_exists)
|
||||
)
|
||||
db.execute(select(Channel.id).where(Channel.from_explore.is_(True), ~followed_channel()))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
@@ -10,14 +10,15 @@ import threading
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, ExploredChannel, OAuthToken, Subscription, User
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
apply_rss_feed,
|
||||
@@ -54,20 +55,6 @@ def get_service_user(db: Session) -> User | None:
|
||||
return users[0] if users else None
|
||||
|
||||
|
||||
def followed_channel():
|
||||
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
|
||||
grace window. The scheduled RSS poll and recent backfill cover only these (owner decision,
|
||||
2026-09-17). They used to cover EVERY channel row, but rows also come from search results,
|
||||
playlist sync and dropped subscriptions, and nothing deletes those: on prod 582 of 979 polled
|
||||
channels had no follower, three of them dead feeds answering 404 on every poll. The cost of
|
||||
narrowing is deliberate: the shared Library stops receiving new uploads from channels nobody
|
||||
follows."""
|
||||
return or_(
|
||||
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
|
||||
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
|
||||
)
|
||||
|
||||
|
||||
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
|
||||
"""Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did
|
||||
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
|
||||
@@ -258,7 +245,8 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
if user is None:
|
||||
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
||||
# Demand-driven: only deep-backfill channels at least one user has opted into
|
||||
# (Subscription.deep_requested). Recent backfill stays unconditional (cheap).
|
||||
# (Subscription.deep_requested). Recent backfill needs no opt-in: it covers every followed channel
|
||||
# (`followed_channel`).
|
||||
deep_requested = (
|
||||
select(Subscription.id)
|
||||
.where(
|
||||
|
||||
@@ -14,7 +14,7 @@ from dataclasses import dataclass
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app.config import Settings, choices_of, settings
|
||||
from app.config import Settings, canonical_choice, choices_of, settings
|
||||
from app.models import SystemConfig
|
||||
|
||||
log = logging.getLogger("siftlode.config")
|
||||
@@ -210,11 +210,10 @@ def _default(s: ConfigSpec):
|
||||
|
||||
def _choice(s: ConfigSpec, raw: object) -> str:
|
||||
"""The canonical spelling of `raw` among the key's choices (any letter case), or ValueError."""
|
||||
folded = str(raw).strip().casefold()
|
||||
for option in s.choices:
|
||||
if option.casefold() == folded:
|
||||
return option
|
||||
option = canonical_choice(s.choices, raw)
|
||||
if option is None:
|
||||
raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}")
|
||||
return option
|
||||
|
||||
|
||||
def _coerce(s: ConfigSpec, raw: str):
|
||||
|
||||
Reference in New Issue
Block a user