refactor(config): fold maintenance_revalidate_batch from app_state into sysconfig (E3 S3, B-core-11)
The admin-tunable maintenance re-validation batch lived in its own app_state column with a hand-rolled override-or-default reader in app.state — a fourth config store duplicating the sysconfig SPECS pattern. Move it to a sysconfig key (`maintenance` group, min=100 max=100_000): the reader (sync/maintenance.py) and the Scheduler dashboard (routes/scheduler.py PATCH + GET) now go through sysconfig.get_int / set_value / spec, and app.state loses get/set_maintenance_batch + the MIN/MAX constants + the settings import. Migration 0070 carries any existing override into the system_config store before dropping the app_state column (reversible; restores + copies back on downgrade). Reuses the existing sysconfig.spec() for the dashboard's min/max, so the bounds have one source. Full lane 365 pass (migration drift + reversibility exercised), ruff clean.
This commit is contained in:
@@ -0,0 +1,47 @@
|
|||||||
|
"""Fold the maintenance re-validation batch from app_state into sysconfig (B-core-11)
|
||||||
|
|
||||||
|
The admin-tunable `maintenance_revalidate_batch` lived in its own `app_state` column with a
|
||||||
|
hand-rolled override-or-default reader in `app.state`, duplicating the sysconfig SPECS pattern. It's
|
||||||
|
now a sysconfig key (`system_config` store), so every admin knob lives in one place. This migration
|
||||||
|
carries any existing override into the sysconfig store, then drops the dead column. Reversible.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0070_fold_maintenance_batch"
|
||||||
|
down_revision: Union[str, None] = "0069_job_index_hygiene"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# Preserve a customized override: copy it into the sysconfig store before the column goes.
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO system_config (key, value, encrypted)
|
||||||
|
SELECT 'maintenance_revalidate_batch', maintenance_revalidate_batch::text, false
|
||||||
|
FROM app_state
|
||||||
|
WHERE id = 1 AND maintenance_revalidate_batch IS NOT NULL
|
||||||
|
ON CONFLICT (key) DO NOTHING
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.drop_column("app_state", "maintenance_revalidate_batch")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"app_state",
|
||||||
|
sa.Column("maintenance_revalidate_batch", sa.Integer(), nullable=True),
|
||||||
|
)
|
||||||
|
# Carry the value back onto the row, then drop the sysconfig override (symmetric restore).
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
UPDATE app_state SET maintenance_revalidate_batch =
|
||||||
|
(SELECT value::int FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||||
|
WHERE id = 1
|
||||||
|
AND EXISTS (SELECT 1 FROM system_config WHERE key = 'maintenance_revalidate_batch')
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
op.execute("DELETE FROM system_config WHERE key = 'maintenance_revalidate_batch'")
|
||||||
@@ -547,9 +547,6 @@ class AppState(Base):
|
|||||||
sync_paused: Mapped[bool] = mapped_column(
|
sync_paused: Mapped[bool] = mapped_column(
|
||||||
Boolean, default=False, server_default="false"
|
Boolean, default=False, server_default="false"
|
||||||
)
|
)
|
||||||
# Admin override for the maintenance job's rolling re-validation batch (videos re-checked
|
|
||||||
# per run). NULL = use the config/env default (settings.maintenance_revalidate_batch).
|
|
||||||
maintenance_revalidate_batch: Mapped[int | None] = mapped_column(Integer)
|
|
||||||
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
|
# First-run install wizard. `configured` is False on a fresh instance (the app runs in
|
||||||
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
|
# setup mode: only the wizard works) and flipped True when the wizard finishes. While
|
||||||
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
|
# unconfigured, a one-time setup token (only its SHA-256 hash stored) gates the wizard;
|
||||||
|
|||||||
@@ -91,16 +91,14 @@ def set_maintenance_settings(
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""Admin-tune the maintenance job's rolling re-validation batch (videos re-checked per
|
"""Admin-tune the maintenance job's rolling re-validation batch (videos re-checked per
|
||||||
run). Persisted to app_state; the job reads it each run."""
|
run). Persisted to app_state; the job reads it each run."""
|
||||||
value = body.revalidate_batch
|
s = sysconfig.spec("maintenance_revalidate_batch")
|
||||||
if not (state.MAINTENANCE_BATCH_MIN <= value <= state.MAINTENANCE_BATCH_MAX):
|
if not (s.min <= body.revalidate_batch <= s.max):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
detail=(
|
detail=f"revalidate_batch must be between {s.min} and {s.max}",
|
||||||
f"revalidate_batch must be between {state.MAINTENANCE_BATCH_MIN} and "
|
|
||||||
f"{state.MAINTENANCE_BATCH_MAX}"
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
new = state.set_maintenance_batch(db, value)
|
sysconfig.set_value(db, "maintenance_revalidate_batch", body.revalidate_batch)
|
||||||
|
new = sysconfig.get_int(db, "maintenance_revalidate_batch")
|
||||||
audit.record(db, admin.id, AuditAction.MAINTENANCE_TUNE, summary=f"revalidate batch = {new}", after={"revalidate_batch": new})
|
audit.record(db, admin.id, AuditAction.MAINTENANCE_TUNE, summary=f"revalidate batch = {new}", after={"revalidate_batch": new})
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"revalidate_batch": new}
|
return {"revalidate_batch": new}
|
||||||
@@ -174,9 +172,9 @@ def get_scheduler(
|
|||||||
"queue": queue,
|
"queue": queue,
|
||||||
"quota": quota_info,
|
"quota": quota_info,
|
||||||
"maintenance": {
|
"maintenance": {
|
||||||
"revalidate_batch": state.get_maintenance_batch(db),
|
"revalidate_batch": sysconfig.get_int(db, "maintenance_revalidate_batch"),
|
||||||
"revalidate_batch_default": settings.maintenance_revalidate_batch,
|
"revalidate_batch_default": settings.maintenance_revalidate_batch,
|
||||||
"min": state.MAINTENANCE_BATCH_MIN,
|
"min": sysconfig.spec("maintenance_revalidate_batch").min,
|
||||||
"max": state.MAINTENANCE_BATCH_MAX,
|
"max": sysconfig.spec("maintenance_revalidate_batch").max,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,16 +4,11 @@ import secrets
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.config import settings
|
|
||||||
from app.models import AppState
|
from app.models import AppState
|
||||||
from app.security import hash_token
|
from app.security import hash_token
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.state")
|
log = logging.getLogger("siftlode.state")
|
||||||
|
|
||||||
# Bounds for the admin-set maintenance re-validation batch (videos re-checked per run).
|
|
||||||
MAINTENANCE_BATCH_MIN = 100
|
|
||||||
MAINTENANCE_BATCH_MAX = 100_000
|
|
||||||
|
|
||||||
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
||||||
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
||||||
_configured_cache = False
|
_configured_cache = False
|
||||||
@@ -90,20 +85,3 @@ def check_setup_token(db: Session, raw: str | None) -> bool:
|
|||||||
return False
|
return False
|
||||||
stored = _row(db).setup_token_hash
|
stored = _row(db).setup_token_hash
|
||||||
return stored is not None and secrets.compare_digest(stored, hash_token(raw))
|
return stored is not None and secrets.compare_digest(stored, hash_token(raw))
|
||||||
|
|
||||||
|
|
||||||
def get_maintenance_batch(db: Session) -> int:
|
|
||||||
"""Effective maintenance re-validation batch: the admin override if set, else the
|
|
||||||
env/config default."""
|
|
||||||
return _row(db).maintenance_revalidate_batch or settings.maintenance_revalidate_batch
|
|
||||||
|
|
||||||
|
|
||||||
def set_maintenance_batch(db: Session, value: int) -> int:
|
|
||||||
"""Clamp and persist the maintenance batch override. Returns the applied value."""
|
|
||||||
value = max(MAINTENANCE_BATCH_MIN, min(MAINTENANCE_BATCH_MAX, int(value)))
|
|
||||||
row = _row(db)
|
|
||||||
row.maintenance_revalidate_batch = value
|
|
||||||
db.add(row)
|
|
||||||
db.commit()
|
|
||||||
log.info("Maintenance re-validation batch set to %s", value)
|
|
||||||
return value
|
|
||||||
|
|||||||
@@ -25,11 +25,10 @@ from datetime import timedelta
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import progress, quota
|
from app import progress, quota, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import Playlist, PlaylistItem, Video
|
from app.models import Playlist, PlaylistItem, Video
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.state import get_maintenance_batch
|
|
||||||
from app.sync.runner import get_service_user
|
from app.sync.runner import get_service_user
|
||||||
from app.sync.videos import apply_video_details, parse_dt
|
from app.sync.videos import apply_video_details, parse_dt
|
||||||
from app.utils import now_utc as _now
|
from app.utils import now_utc as _now
|
||||||
@@ -182,7 +181,7 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
||||||
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
||||||
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
||||||
batch = get_maintenance_batch(db) # admin override, else config default
|
batch = sysconfig.get_int(db, "maintenance_revalidate_batch") # admin override, else config default
|
||||||
flagged_new = 0
|
flagged_new = 0
|
||||||
checked = 0
|
checked = 0
|
||||||
now = _now()
|
now = _now()
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ SPECS: tuple[ConfigSpec, ...] = (
|
|||||||
# --- Batch sizes ---
|
# --- Batch sizes ---
|
||||||
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
|
ConfigSpec("enrich_batch_size", "int", "batch", "enrich_batch_size", min=1, max=50),
|
||||||
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
|
ConfigSpec("autotag_title_sample", "int", "batch", "autotag_title_sample", min=1, max=1_000),
|
||||||
|
# --- Maintenance (rolling re-validation of already-known videos) ---
|
||||||
|
# Videos re-checked per maintenance run. Folded here from app_state (B-core-11) so every admin
|
||||||
|
# knob lives in one store; read by sync/maintenance.py, tuned from the Scheduler dashboard.
|
||||||
|
ConfigSpec("maintenance_revalidate_batch", "int", "maintenance", "maintenance_revalidate_batch", min=100, max=100_000),
|
||||||
# --- Discovery cleanup (search + explore ephemeral content) ---
|
# --- Discovery cleanup (search + explore ephemeral content) ---
|
||||||
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
# Days an explored-but-unsubscribed channel is kept before the cleanup job purges it.
|
||||||
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
ConfigSpec("explore_grace_days", "int", "explore", "explore_grace_days", min=0, max=3_650),
|
||||||
|
|||||||
Reference in New Issue
Block a user