peter/siftlode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(maintenance): one clock per run, so a video phase 1 recovers is not fetched again in phase 2

Browse Source 
E32 review round 1 (B7). Each phase took its own now; phase 2's "not checked this run" test then
let a just-recovered video through and spent another videos.list call on it.
This commit is contained in:
peter
2026-10-04 02:14:04 +02:00
parent 887b4e546c
commit 38943fa93e
2 changed files with 49 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/sync/maintenance.py
+10 -8
View File
@@ -20,7 +20,7 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
backfill reserve as the other jobs so interactive syncs never starve.
"""
from datetime import timedelta
from datetime import datetime, timedelta
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
@@ -129,14 +129,14 @@ def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int:
return len(impact)
def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
"""Phase 1: recover or hard-delete videos already flagged unavailable."""
def _recheck_flagged(db: Session, yt: YouTubeClient, now: datetime) -> dict:
"""Phase 1: recover or hard-delete videos already flagged unavailable. `now` is the run's:
phase 2 skips a video stamped with it, so one recovered here is not fetched again."""
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
if not flagged:
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
progress.report(0, len(flagged), "recheck")
items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])}
now = _now()
recovered = 0
to_delete: list[Video] = []
for video in flagged:
@@ -173,7 +173,7 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
}
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
def _revalidate_rolling(db: Session, yt: YouTubeClient, now: datetime) -> dict:
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
batch = sysconfig.get_int(
@@ -181,7 +181,6 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
) # admin override, else config default
flagged_new = 0
checked = 0
now = _now()
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
while checked < batch:
if quota.backfill_reserve_reached():
@@ -224,10 +223,13 @@ def run_maintenance(db: Session) -> dict:
user = get_service_user(db)
if user is None:
return {"reason": "no service user"}
# One `now` for the whole run: each phase taking its own let a video phase 1 had just recovered
# (stamped with phase 1's) pass phase 2's "not checked this run" test and be fetched twice.
now = _now()
with YouTubeClient(db, user) as yt:
phase1 = _recheck_flagged(db, yt)
phase1 = _recheck_flagged(db, yt, now)
phase2 = (
_revalidate_rolling(db, yt)
_revalidate_rolling(db, yt, now)
if not quota.backfill_reserve_reached()
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
)