refactor(config): fold maintenance_revalidate_batch from app_state into sysconfig (E3 S3, B-core-11)
The admin-tunable maintenance re-validation batch lived in its own app_state column with a hand-rolled override-or-default reader in app.state — a fourth config store duplicating the sysconfig SPECS pattern. Move it to a sysconfig key (`maintenance` group, min=100 max=100_000): the reader (sync/maintenance.py) and the Scheduler dashboard (routes/scheduler.py PATCH + GET) now go through sysconfig.get_int / set_value / spec, and app.state loses get/set_maintenance_batch + the MIN/MAX constants + the settings import. Migration 0070 carries any existing override into the system_config store before dropping the app_state column (reversible; restores + copies back on downgrade). Reuses the existing sysconfig.spec() for the dashboard's min/max, so the bounds have one source. Full lane 365 pass (migration drift + reversibility exercised), ruff clean.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
"""Fold the maintenance re-validation batch from app_state into sysconfig (B-core-11)
|
||||
|
||||
The admin-tunable `maintenance_revalidate_batch` lived in its own `app_state` column with a
|
||||
hand-rolled override-or-default reader in `app.state`, duplicating the sysconfig SPECS pattern. It's
|
||||
now a sysconfig key (`system_config` store), so every admin knob lives in one place. This migration
|
||||
carries any existing override into the sysconfig store, then drops the dead column. Reversible.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0070_fold_maintenance_batch"
|
||||
down_revision: Union[str, None] = "0069_job_index_hygiene"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Preserve a customized override: copy it into the sysconfig store before the column goes.
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO system_config (key, value, encrypted)
|
||||
SELECT 'maintenance_revalidate_batch', maintenance_revalidate_batch::text, false
|
||||
FROM app_state
|
||||
WHERE id = 1 AND maintenance_revalidate_batch IS NOT NULL
|
||||
ON CONFLICT (key) DO NOTHING
|
||||
"""
|
||||
)
|
||||
op.drop_column("app_state", "maintenance_revalidate_batch")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column(
|
||||
"app_state",
|
||||
sa.Column("maintenance_revalidate_batch", sa.Integer(), nullable=True),
|
||||
)
|
||||
# Carry the value back onto the row, then drop the sysconfig override (symmetric restore).
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE app_state SET maintenance_revalidate_batch =
|
||||
(SELECT value::int FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||
WHERE id = 1
|
||||
AND EXISTS (SELECT 1 FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||
"""
|
||||
)
|
||||
op.execute("DELETE FROM system_config WHERE key = 'maintenance_revalidate_batch'")
|
||||
@@ -547,9 +547,6 @@ class AppState(Base):
|
||||
sync_paused: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false"
|
||||
)
|
||||
# Admin override for the maintenance job's rolling re-validation batch (videos re-checked
|
||||
# per run). NULL = use the config/env default (settings.maintenance_revalidate_batch).
|
||||
maintenance_revalidate_batch: Mapped[int | None] = mapped_column(Integer)
|
||||
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
|
||||
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
|
||||
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
|
||||
|
||||
@@ -91,16 +91,14 @@ def set_maintenance_settings(
|
||||
) -> dict:
|
||||
"""Admin-tune the maintenance job's rolling re-validation batch (videos re-checked per
|
||||
run). Persisted to app_state; the job reads it each run."""
|
||||
value = body.revalidate_batch
|
||||
if not (state.MAINTENANCE_BATCH_MIN <= value <= state.MAINTENANCE_BATCH_MAX):
|
||||
s = sysconfig.spec("maintenance_revalidate_batch")
|
||||
if not (s.min <= body.revalidate_batch <= s.max):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"revalidate_batch must be between {state.MAINTENANCE_BATCH_MIN} and "
|
||||
f"{state.MAINTENANCE_BATCH_MAX}"
|
||||
),
|
||||
detail=f"revalidate_batch must be between {s.min} and {s.max}",
|
||||
)
|
||||
new = state.set_maintenance_batch(db, value)
|
||||
sysconfig.set_value(db, "maintenance_revalidate_batch", body.revalidate_batch)
|
||||
new = sysconfig.get_int(db, "maintenance_revalidate_batch")
|
||||
audit.record(db, admin.id, AuditAction.MAINTENANCE_TUNE, summary=f"revalidate batch = {new}", after={"revalidate_batch": new})
|
||||
db.commit()
|
||||
return {"revalidate_batch": new}
|
||||
@@ -174,9 +172,9 @@ def get_scheduler(
|
||||
"queue": queue,
|
||||
"quota": quota_info,
|
||||
"maintenance": {
|
||||
"revalidate_batch": state.get_maintenance_batch(db),
|
||||
"revalidate_batch": sysconfig.get_int(db, "maintenance_revalidate_batch"),
|
||||
"revalidate_batch_default": settings.maintenance_revalidate_batch,
|
||||
"min": state.MAINTENANCE_BATCH_MIN,
|
||||
"max": state.MAINTENANCE_BATCH_MAX,
|
||||
"min": sysconfig.spec("maintenance_revalidate_batch").min,
|
||||
"max": sysconfig.spec("maintenance_revalidate_batch").max,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -4,16 +4,11 @@ import secrets
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.models import AppState
|
||||
from app.security import hash_token
|
||||
|
||||
log = logging.getLogger("siftlode.state")
|
||||
|
||||
# Bounds for the admin-set maintenance re-validation batch (videos re-checked per run).
|
||||
MAINTENANCE_BATCH_MIN = 100
|
||||
MAINTENANCE_BATCH_MAX = 100_000
|
||||
|
||||
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
||||
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
||||
_configured_cache = False
|
||||
@@ -90,20 +85,3 @@ def check_setup_token(db: Session, raw: str | None) -> bool:
|
||||
return False
|
||||
stored = _row(db).setup_token_hash
|
||||
return stored is not None and secrets.compare_digest(stored, hash_token(raw))
|
||||
|
||||
|
||||
def get_maintenance_batch(db: Session) -> int:
|
||||
"""Effective maintenance re-validation batch: the admin override if set, else the
|
||||
env/config default."""
|
||||
return _row(db).maintenance_revalidate_batch or settings.maintenance_revalidate_batch
|
||||
|
||||
|
||||
def set_maintenance_batch(db: Session, value: int) -> int:
|
||||
"""Clamp and persist the maintenance batch override. Returns the applied value."""
|
||||
value = max(MAINTENANCE_BATCH_MIN, min(MAINTENANCE_BATCH_MAX, int(value)))
|
||||
row = _row(db)
|
||||
row.maintenance_revalidate_batch = value
|
||||
db.add(row)
|
||||
db.commit()
|
||||
log.info("Maintenance re-validation batch set to %s", value)
|
||||
return value
|
||||
|
||||
@@ -25,11 +25,10 @@ from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota
|
||||
from app import progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video
|
||||
from app.notifications import create_notification
|
||||
from app.state import get_maintenance_batch
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.utils import now_utc as _now
|
||||
@@ -182,7 +181,7 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
||||
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
||||
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
||||
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
||||
batch = get_maintenance_batch(db) # admin override, else config default
|
||||
batch = sysconfig.get_int(db, "maintenance_revalidate_batch") # admin override, else config default
|
||||
flagged_new = 0
|
||||
checked = 0
|
||||
now = _now()
|
||||
|
||||
@@ -59,6 +59,10 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
# --- Batch sizes ---
|
||||
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
|
||||
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
|
||||
# --- Maintenance (rolling re-validation of already-known videos) ---
|
||||
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
|
||||
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
|
||||
ConfigSpec("maintenance_revalidate_batch", "int", "maintenance", "maintenance_revalidate_batch", min=100, max=100_000),
|
||||
# --- Discovery cleanup (search + explore ephemeral content) ---
|
||||
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
||||
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
||||
|
||||
Reference in New Issue
Block a user