peter
4aaaaa7860
The code cited its plan instead of saying why: `E28 review J4`, `(R6 S3b)`, `S1 justified…`,
`deep-review round 5`. That plan lives in the private ops repo, so for every reader of this one the
citation was noise, and a sentence whose subject was the plan step ("round 2 showed…") hid its
lesson behind an id. Each one is gone and the "why" kept: a bare citation is deleted, a sentence
carrying a lesson is reworded around it ("a later attempt showed…").
Measured (`scripts/audit_metrics.py`, product code): 652 lines in 270 files carried an epic code by
the definition the next commit holds (507 / 241 by the old, two-digit-only one) -> 0. Across all
code, tests, scripts and configs: ~1100 epic-code lines plus ~320 sprint/round ids.
Behaviour unchanged, checked mechanically: every changed .py file has the same AST once docstrings
are blanked; css/sh/toml/yml/json files are identical once comments are stripped; the only
non-comment changes in TS/JS are test titles, one description string in a contract table, and five
ESLint messages, whose text is all the gates snapshot diff holds. Not touched: Markdown (docs and
the changelog keep their history), Alembic migrations, TV episode notation (`S2 · E5`), DESIGN.md
ids (`C1`, `B4`, `S3h`), tracker ids (`SEC-16`, `DX-17`).
One stale claim fixed on the way: test_hls_reaper's docstring said the player checkpoints "every
10 s whether or not it is playing", which plex/stream.py calls wrong; it now says what the code does.
521 lines
24 KiB
Python
521 lines
24 KiB
Python
"""Reusable sync routines shared by the scheduler and the manual /api/sync endpoints.
|
|
|
|
Channel/video data is shared across users, so background work acts through a "service
|
|
user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configured for
|
|
public reads.
|
|
"""
|
|
|
|
import logging
|
|
import threading
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import func, or_, select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import logs, progress, quota, sysconfig
|
|
from app.db import SessionLocal
|
|
from app.models import Channel, OAuthToken, Subscription, User
|
|
from app.sync.explore import followed_channel
|
|
from app.sync.subscriptions import import_subscriptions
|
|
from app.sync.videos import (
|
|
apply_rss_feed,
|
|
backfill_channel_deep,
|
|
backfill_channel_recent,
|
|
enrich_pending,
|
|
insert_stubs,
|
|
reconcile_full_history,
|
|
refresh_live,
|
|
run_shorts_classification,
|
|
stub_from_playlist_item,
|
|
)
|
|
from app.youtube.client import YouTubeClient, YouTubeError, is_quota_refusal
|
|
from app.youtube.rss import RssError, fetch_channel_feed
|
|
|
|
log = logs.get_logger(__name__)
|
|
|
|
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
|
RSS_POLL_WORKERS = 16
|
|
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
|
RSS_FAILURE_EXAMPLES = 5
|
|
|
|
|
|
def token_users(db: Session) -> list[User]:
|
|
"""Every user with a stored YouTube refresh token (the read-scope service accounts), by id."""
|
|
return list(
|
|
db.execute(
|
|
select(User)
|
|
.join(OAuthToken)
|
|
.where(OAuthToken.refresh_token_enc.is_not(None))
|
|
.order_by(User.id)
|
|
).scalars()
|
|
)
|
|
|
|
|
|
def get_service_user(db: Session) -> User | None:
|
|
users = token_users(db)
|
|
return users[0] if users else None
|
|
|
|
|
|
def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str, int]:
|
|
"""Poll the free RSS feed of every followed channel (`followed_channel`). Returns `{"new": inserted, "failed": channels_that_did
|
|
_not_complete}` — the failure count is part of the RESULT, not just a log line, so the scheduler
|
|
card and the audit summary can tell "no new uploads" apart from "nothing worked" (an egress
|
|
outage, or a database that can't accept writes, otherwise renders identically to a quiet hour).
|
|
|
|
`failed` counts a channel whose feed could not be READ *or* whose rows could not be APPLIED —
|
|
both leave the channel un-polled, and counting only the first would move the blind spot rather
|
|
than remove it.
|
|
|
|
Raises RuntimeError when NOTHING could be polled (`failed == total`, total > 0): a run that
|
|
achieved literally nothing is a failed run, and only an exception turns the scheduler card red
|
|
— a returned count leaves it green with the outage buried in the summary line. The message
|
|
carries the FIRST underlying error, because on a small instance "all channels failed" is also
|
|
what a single dead feed id looks like."""
|
|
if channels is None:
|
|
channels = db.execute(select(Channel).where(followed_channel())).scalars().all()
|
|
total = len(channels)
|
|
new = 0
|
|
done = 0
|
|
failed = 0
|
|
# Kept for the total-outage message: "failed for all N channels" alone can't tell an egress
|
|
# outage from one permanently dead feed id — which is the whole difference on an instance with
|
|
# a single subscription, where BOTH render as 100% failed.
|
|
first_error: str | None = None
|
|
# The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key).
|
|
first_failure: tuple[str, str] | None = None
|
|
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
|
|
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
|
|
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
|
|
reasons: Counter[str] = Counter()
|
|
examples: list[str] = []
|
|
# Channels whose FEED would not answer (not those whose rows failed to apply: the API would
|
|
# hand the same rows to the same broken write) — the feed fallback's candidates.
|
|
unread: list[str] = []
|
|
|
|
def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None:
|
|
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
|
|
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
|
|
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
|
|
tell nobody anything the first one didn't. Call it from inside the `except` block."""
|
|
nonlocal failed, first_error, first_failure
|
|
if unexpected is not None:
|
|
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
|
|
log.log(level, unexpected, exc_info=True)
|
|
failed += 1
|
|
reasons[reason] += 1
|
|
first_error = first_error or detail
|
|
first_failure = first_failure or (channel_id, reason)
|
|
if len(examples) < RSS_FAILURE_EXAMPLES:
|
|
examples.append(detail)
|
|
|
|
# Fetch feeds concurrently (network-bound, no DB), then apply inserts on this thread as
|
|
# each fetch returns — a SQLAlchemy session isn't thread-safe. Pass the channel id (a
|
|
# plain str) to the workers; the ORM instance is touched only here on the main thread.
|
|
futures = {}
|
|
with ThreadPoolExecutor(max_workers=RSS_POLL_WORKERS) as pool:
|
|
for channel in channels:
|
|
futures[pool.submit(fetch_channel_feed, channel.id)] = channel
|
|
for fut in as_completed(futures):
|
|
channel = futures[fut]
|
|
try:
|
|
entries = fut.result()
|
|
# A failed poll is NOT an empty feed: skip the apply so `last_rss_at` stays where it
|
|
# was (stamping it made a permanently-broken feed look freshly polled). The shared
|
|
# counter/progress tail lives after the statement, so the two outcomes can stay two
|
|
# clauses: an RssError is EXPECTED (404, timeout) and is only counted; anything else is a
|
|
# bug on our side and earns a traceback (the first of its kind per poll).
|
|
except RssError as exc:
|
|
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
|
|
fail(exc.reason, channel.id, f"{channel.id}: {exc}")
|
|
unread.append(channel.id)
|
|
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
|
|
# traceback, once per reason per poll, which is the flood control this loop needs.
|
|
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
|
|
fail(
|
|
"error",
|
|
channel.id,
|
|
f"{channel.id}: {exc}",
|
|
unexpected=f"RSS fetch failed for {channel.id}",
|
|
)
|
|
else:
|
|
try:
|
|
new += apply_rss_feed(db, channel, entries)
|
|
# silent-ok: `fail(unexpected=…)` below is the report — ERROR with the traceback,
|
|
# once per reason per poll.
|
|
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
|
|
# A write failure (read-only DB, failover) leaves the channel just as un-polled
|
|
# as an unreachable feed — count it, or a wholly broken run reports "failed=0".
|
|
db.rollback()
|
|
fail(
|
|
"apply",
|
|
channel.id,
|
|
f"{channel.id} (apply): {exc}",
|
|
unexpected=f"RSS apply failed for {channel.id}",
|
|
)
|
|
# One place that advances the counter, whatever happened above.
|
|
done += 1
|
|
progress.report(done, total, "rss_poll")
|
|
fallback = run_feed_fallback(db, unread)
|
|
if failed:
|
|
# The fallback's count rides on this line because this is the one the stored log keeps (an
|
|
# INFO line outside a request never reaches `log_events`): the console should show whether
|
|
# the uploads came in another way.
|
|
log.warning(
|
|
"RSS poll: %d/%d channels could not be polled; %d read through the API instead",
|
|
failed,
|
|
total,
|
|
fallback["read"],
|
|
extra=logs.event(
|
|
"sync.rss.failed",
|
|
failed=failed,
|
|
total=total,
|
|
reasons=dict(reasons),
|
|
examples=examples,
|
|
fallback_read=fallback["read"],
|
|
fallback_new=fallback["new"],
|
|
),
|
|
)
|
|
if total and failed == total:
|
|
# Nothing was polled at all — an egress outage (the tinyproxy-after-reboot incident), a
|
|
# dead DNS, a read-only database. Returning normally would record the run as `ok` with the
|
|
# failure hidden inside the result string, i.e. a green pill for a total outage.
|
|
# The first error TRAVELS WITH the message on purpose: on an instance with one or two
|
|
# subscriptions "all channels failed" is also what ONE dead feed id looks like, and a red
|
|
# card the user can't act on is its own kind of dishonest. `channel: HTTP 404` says
|
|
# "fix this subscription"; `Connection refused` says "the egress is down".
|
|
# The fallback's count rides along: a red card during an RSS outage should still say
|
|
# whether the uploads are arriving another way.
|
|
# As data too: the audit log and the inbox say it in the reader's language, the first
|
|
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
|
|
first_channel, first_reason = first_failure or ("", "error")
|
|
raise progress.JobFailed(
|
|
"rss_all_failed",
|
|
f"RSS poll failed for all {total} channels — first: {first_error}"
|
|
f" (read through the API instead: {fallback['read']})",
|
|
total=total,
|
|
channel=first_channel,
|
|
cause=first_reason,
|
|
read=fallback["read"],
|
|
)
|
|
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
|
|
|
|
|
|
def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]:
|
|
"""Read the newest uploads through the Data API for the channels among `channel_ids` whose feed
|
|
has not answered for `rss_fallback_hours` — and that were not read this way within as long.
|
|
|
|
YouTube's RSS goes dark per channel for hours at a time (2026-09-18/19: 393 of 396 feeds answered
|
|
404 for four hours, and a retry of the same channel gets the same 404), and nothing else reads a
|
|
followed channel's new uploads: recent backfill runs once per channel, ever. One page of the
|
|
uploads playlist is 1 unit, so a whole day of total outage costs 12 units per channel at the
|
|
default 2 hours; it yields to the backfill reserve like every other background read. The clock is
|
|
`last_rss_at` or `feed_fallback_at`, whichever is later, so a channel is read at most once per
|
|
window whether its feed came back in between or not. A read the API REFUSED stamps the clock
|
|
too: a terminated channel's uploads playlist answers 404 for good, and without the stamp it
|
|
would be charged again on every 20-minute poll instead of once per window. A failure that is not
|
|
about the channel (no connection, no token, the quota spent) stamps nothing and ends the round."""
|
|
hours = sysconfig.get_int(db, "rss_fallback_hours")
|
|
if hours <= 0 or not channel_ids:
|
|
return {"read": 0, "new": 0, "failed": 0}
|
|
now = datetime.now(timezone.utc)
|
|
last_read = func.greatest(Channel.last_rss_at, Channel.feed_fallback_at)
|
|
due = (
|
|
db.execute(
|
|
select(Channel)
|
|
.where(
|
|
Channel.id.in_(channel_ids),
|
|
Channel.uploads_playlist_id.is_not(None),
|
|
or_(last_read.is_(None), last_read < now - timedelta(hours=hours)),
|
|
)
|
|
.order_by(Channel.id)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
user = get_service_user(db) if due else None
|
|
if user is None:
|
|
return {"read": 0, "new": 0, "failed": 0}
|
|
read = new = failed = 0
|
|
with (
|
|
quota.attribute(None, quota.QuotaAction.VIDEOS_FEED_FALLBACK),
|
|
YouTubeClient(db, user) as yt,
|
|
):
|
|
for channel in due:
|
|
if quota.backfill_reserve_reached():
|
|
break
|
|
try:
|
|
data = yt.get_playlist_items(channel.uploads_playlist_id, None)
|
|
# silent-ok: counted, and the one summary line below names the count — the client
|
|
# already logged the call's own failure (`youtube.api.error`).
|
|
except YouTubeError as exc:
|
|
failed += 1
|
|
# Not an answer about THIS channel — no connection, no token, no quota: every
|
|
# further channel would fail the same way, and none of them has used its window.
|
|
if exc.path != "playlistItems" or is_quota_refusal(exc):
|
|
break
|
|
_stamp_fallback_attempt(db, channel.id)
|
|
continue
|
|
rows = [
|
|
s
|
|
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
|
if s is not None
|
|
]
|
|
try:
|
|
new += insert_stubs(db, rows)
|
|
channel.feed_fallback_at = datetime.now(timezone.utc)
|
|
db.add(channel)
|
|
db.commit()
|
|
except Exception: # one channel must not stop the rest
|
|
db.rollback()
|
|
log.exception(
|
|
"Feed fallback could not store channel %s",
|
|
channel.id,
|
|
extra=logs.event("sync.rss.fallback_failed"),
|
|
)
|
|
failed += 1
|
|
_stamp_fallback_attempt(db, channel.id)
|
|
continue
|
|
read += 1
|
|
log.info(
|
|
"Feed fallback: read %d of %d channels through the API, %d new videos",
|
|
read,
|
|
len(due),
|
|
new,
|
|
extra=logs.event("sync.rss.fallback", read=read, due=len(due), new=new, failed=failed),
|
|
)
|
|
return {"read": read, "new": new, "failed": failed}
|
|
|
|
|
|
def _stamp_fallback_attempt(db: Session, channel_id: str) -> None:
|
|
"""Start a channel's fallback window after a read that did not land, so it waits the window out
|
|
like a read that did. Its own commit: the failure already rolled the channel's work back."""
|
|
try:
|
|
db.execute(
|
|
update(Channel)
|
|
.where(Channel.id == channel_id)
|
|
.values(feed_fallback_at=datetime.now(timezone.utc))
|
|
)
|
|
db.commit()
|
|
# silent-ok: the failure that brought us here was already logged and counted; an unstamped
|
|
# channel is only read again at the next poll.
|
|
except Exception: # noqa: BLE001
|
|
db.rollback()
|
|
|
|
|
|
def run_enrich(db: Session, max_batches: int = 200, floor: int = 200) -> int:
|
|
user = get_service_user(db)
|
|
if user is None:
|
|
return 0
|
|
total = 0
|
|
with YouTubeClient(db, user) as yt:
|
|
for _ in range(max_batches):
|
|
if quota.remaining_today(db) <= floor:
|
|
break
|
|
n = enrich_pending(db, yt)
|
|
total += n
|
|
progress.report(total, None, "enrich") # total unknown (drains until empty)
|
|
if n == 0:
|
|
break
|
|
# Revisit transient live/upcoming videos (enrich_pending is one-shot) so an ended
|
|
# stream gains its final duration + was_live status instead of staying "live" forever.
|
|
if quota.remaining_today(db) > floor:
|
|
total += refresh_live(db, yt)
|
|
return total
|
|
|
|
|
|
def run_shorts(db: Session) -> dict:
|
|
return run_shorts_classification(db)
|
|
|
|
|
|
def run_subscription_resync(db: Session) -> dict:
|
|
"""Re-import every user's subscriptions so unsubscribes and new subscriptions on
|
|
YouTube are reflected automatically."""
|
|
users = token_users(db)
|
|
for i, user in enumerate(users, 1):
|
|
try:
|
|
import_subscriptions(db, user)
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("Subscription resync failed for user %s", user.id)
|
|
progress.report(i, len(users), "subscriptions")
|
|
return {"users": len(users)}
|
|
|
|
|
|
def run_recent_backfill(
|
|
db: Session, channels: list[Channel] | None = None, max_channels: int | None = None
|
|
) -> dict:
|
|
user = get_service_user(db)
|
|
if user is None:
|
|
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
|
if channels is None:
|
|
channels = (
|
|
db.execute(
|
|
select(Channel).where(Channel.recent_synced_at.is_(None), followed_channel())
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
if max_channels:
|
|
channels = channels[:max_channels]
|
|
processed = 0
|
|
videos_added = 0
|
|
with YouTubeClient(db, user) as yt:
|
|
for channel in channels:
|
|
if quota.backfill_reserve_reached():
|
|
break
|
|
try:
|
|
videos_added += backfill_channel_recent(db, yt, channel)
|
|
processed += 1
|
|
progress.report(processed, len(channels), "backfill_recent")
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("Recent backfill failed for channel %s", channel.id)
|
|
reconciled = reconcile_full_history(db)
|
|
return {
|
|
"channels_processed": processed,
|
|
"videos_added": videos_added,
|
|
"reconciled_full": reconciled,
|
|
}
|
|
|
|
|
|
def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -> dict:
|
|
user = get_service_user(db)
|
|
if user is None:
|
|
return {"channels_processed": 0, "videos_added": 0, "reason": "no service user"}
|
|
# Demand-driven: only deep-backfill channels at least one user has opted into
|
|
# (Subscription.deep_requested). Recent backfill needs no opt-in: it covers every followed channel
|
|
# (`followed_channel`).
|
|
deep_requested = (
|
|
select(Subscription.id)
|
|
.where(
|
|
Subscription.channel_id == Channel.id,
|
|
Subscription.deep_requested.is_(True),
|
|
)
|
|
.exists()
|
|
)
|
|
channels = (
|
|
db.execute(
|
|
select(Channel).where(
|
|
Channel.backfill_done.is_(False),
|
|
Channel.recent_synced_at.is_not(None),
|
|
deep_requested,
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)[:max_channels]
|
|
processed = 0
|
|
videos_added = 0
|
|
with YouTubeClient(db, user) as yt:
|
|
for channel in channels:
|
|
if quota.backfill_reserve_reached():
|
|
break
|
|
# Claim the channel (or skip it if an interactive backfill already owns it) so the two
|
|
# never advance the same cursor concurrently — symmetric with backfill_channel_now.
|
|
with _backfill_lock:
|
|
if channel.id in _backfill_in_progress:
|
|
continue
|
|
_backfill_in_progress.add(channel.id)
|
|
try:
|
|
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
|
|
processed += 1
|
|
progress.report(processed, len(channels), "backfill_deep")
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("Deep backfill failed for channel %s", channel.id)
|
|
finally:
|
|
with _backfill_lock:
|
|
_backfill_in_progress.discard(channel.id)
|
|
reconciled = reconcile_full_history(db)
|
|
return {
|
|
"channels_processed": processed,
|
|
"videos_added": videos_added,
|
|
"reconciled_full": reconciled,
|
|
}
|
|
|
|
|
|
# Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a
|
|
# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread).
|
|
# BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page
|
|
# the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running`
|
|
# counts only the interactive path, capped so one user's burst can't drain the shared budget (the
|
|
# scheduler is already quota-paced). Single uvicorn process → one shared module state, lock suffices.
|
|
_backfill_in_progress: set[str] = set()
|
|
_immediate_running = 0
|
|
_backfill_lock = threading.Lock()
|
|
_MAX_CONCURRENT_IMMEDIATE = 2
|
|
|
|
|
|
def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None:
|
|
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
|
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
|
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish
|
|
any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69
|
|
pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in
|
|
over the scheduler's 10-minute ticks and looked stuck). backfill_channel_deep re-checks the shared
|
|
budget before each page and yields at the reserve, so a high cap still can't drain the day's quota.
|
|
Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the
|
|
shared backfill budget by opting many channels in at once (the overflow, and any channel too large
|
|
to finish under the reserve, is finished by the scheduler since deep_requested stays set), and
|
|
mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and
|
|
swallows failures."""
|
|
global _immediate_running
|
|
with _backfill_lock:
|
|
if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE:
|
|
return # already backfilling this channel, or too many interactive runs — leave it to the scheduler
|
|
_backfill_in_progress.add(channel_id)
|
|
_immediate_running += 1
|
|
try:
|
|
with SessionLocal() as db:
|
|
user = get_service_user(db)
|
|
if user is None:
|
|
return
|
|
channel = db.get(Channel, channel_id)
|
|
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
|
return
|
|
if quota.backfill_reserve_reached():
|
|
return
|
|
try:
|
|
# Attribute the spend to the requesting user under the full-backfill action, so it
|
|
# shows up correctly in the usage breakdown (not the default OTHER/system bucket).
|
|
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL):
|
|
with YouTubeClient(db, user) as yt:
|
|
backfill_channel_deep(db, yt, channel, max_pages)
|
|
reconcile_full_history(db)
|
|
except Exception:
|
|
db.rollback()
|
|
log.exception("Immediate backfill failed for channel %s", channel_id)
|
|
finally:
|
|
with _backfill_lock:
|
|
_backfill_in_progress.discard(channel_id)
|
|
_immediate_running -= 1
|
|
|
|
|
|
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
|
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
|
|
estimated API units / the daily budget left for backfill. Deliberately approximate —
|
|
a channel's cost ~= playlistItems pages (1 unit / 50 videos) + enrichment (likewise)."""
|
|
pending = [c for c in channels if not c.backfill_done]
|
|
|
|
def units(c: Channel) -> int:
|
|
videos = c.video_count or 0
|
|
pages = max(1, (videos + 49) // 50)
|
|
return 2 * pages
|
|
|
|
total_units = sum(units(c) for c in pending)
|
|
daily_budget = max(
|
|
1,
|
|
sysconfig.get_int(db, "quota_daily_budget")
|
|
- sysconfig.get_int(db, "backfill_quota_reserve"),
|
|
)
|
|
eta_seconds = int(total_units / daily_budget * 86_400) if pending else 0
|
|
return {
|
|
"channels_pending": len(pending),
|
|
"videos_pending_est": sum(c.video_count or 0 for c in pending),
|
|
"units_est": total_units,
|
|
"daily_budget": daily_budget,
|
|
"eta_seconds": eta_seconds,
|
|
}
|