fix: poll and backfill only channels somebody follows

The scheduled RSS poll and recent backfill covered every channel row,
but rows also come from search results, playlist sync and dropped
subscriptions, and nothing deletes them. On prod 582 of 979 polled
channels had no follower, three of them dead feeds answering 404 on
every poll. Both jobs now take subscribed or explored channels only,
and the scheduler card counts the same set. As a deliberate trade-off,
the shared Library no longer receives new uploads from channels nobody
follows.
This commit is contained in:
2026-09-17 00:38:46 +02:00
parent b59fd38a9c
commit 0517de1f59
3 changed files with 92 additions and 7 deletions
+23 -5
View File
@@ -10,14 +10,14 @@ import threading
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from app import logs, progress, quota, sysconfig
log = logging.getLogger("siftlode.sync")
from app.db import SessionLocal
from app.models import Channel, OAuthToken, Subscription, User
from app.models import Channel, ExploredChannel, OAuthToken, Subscription, User
from app.sync.subscriptions import import_subscriptions
from app.sync.videos import (
apply_rss_feed,
@@ -54,8 +54,22 @@ def get_service_user(db: Session) -> User | None:
return users[0] if users else None
def followed_channel():
"""WHERE clause: a channel somebody follows — a subscription, or an exploration still inside its
grace window. The scheduled RSS poll and recent backfill cover only these (owner decision,
2026-09-17). They used to cover EVERY channel row, but rows also come from search results,
playlist sync and dropped subscriptions, and nothing deletes those: on prod 582 of 979 polled
channels had no follower, three of them dead feeds answering 404 on every poll. The cost of
narrowing is deliberate: the shared Library stops receiving new uploads from channels nobody
follows."""
return or_(
select(Subscription.id).where(Subscription.channel_id == Channel.id).exists(),
select(ExploredChannel.id).where(ExploredChannel.channel_id == Channel.id).exists(),
)
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
"""Poll every channel's free RSS feed. Returns `{"new": inserted, "failed": channels_that_did
"""Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
card and the audit summary can tell "no new uploads" apart from "nothing worked" (an egress
outage, or a database that can't accept writes, otherwise renders identically to a quiet hour).
@@ -70,7 +84,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
carries the FIRST underlying error, because on a small instance "all channels failed" is also
what a single dead feed id looks like."""
if channels is None:
channels = db.execute(select(Channel)).scalars().all()
channels = db.execute(select(Channel).where(followed_channel())).scalars().all()
total = len(channels)
new = 0
done = 0
@@ -210,7 +224,11 @@ def run_recent_backfill(
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
if channels is None:
channels = (
db.execute(select(Channel).where(Channel.recent_synced_at.is_(None))).scalars().all()
db.execute(
select(Channel).where(Channel.recent_synced_at.is_(None), followed_channel())
)
.scalars()
.all()
)
if max_channels:
channels = channels[:max_channels]