refactor(logs): name every logger after its module, and hold it there
25 literal names covered 34 modules. plex/client.py, plex/stream.py, plex/subtitles.py, plex/sync.py and two Plex route modules all called themselves `siftlode.plex`; downloads/pipeline.py called itself `siftlode.worker`; routes/channels.py called itself `siftlode.api`. The `logger` column is in every stored row so E22.6's console can filter on it, and so an operator can pin a level to one part of the system — neither works when six modules share a name. `logs.get_logger(__name__)` derives it: app/plex/sync.py -> siftlode.plex.sync. One name per module, and it cannot drift from the file it sits in. tests/test_logger_names.py pins the derivation, that everything stays under the one `siftlode` tree the handler and the tests attach to, and that no module hand-rolls a name again unless the line says why (`# logger-name-ok: <why>`). Two do: get_logger's own call, and `siftlode.http`, which is named for the traffic it carries rather than the module it lives in — it replaced uvicorn's access log and is what an operator pins when they want request lines and nothing else. app/scheduler.py called its logger `logger`; every other module calls it `log`. ruff gains LOG, G, T20, TRY400, TRY401 and S112. Four findings cleared with them: worker.py logged `str(exc)[:120]` where the traceback was the whole question, and three prints/f-strings that are deliberate now say so on the line. S110 and BLE001 stay off until S3 — 23 handlers are outstanding and each is a judgement about one of them; enabling the rules first would just mean 23 blanket noqas.
This commit is contained in:
+1
-3
@@ -1,5 +1,3 @@
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
||||
from sqlalchemy import delete, func, select
|
||||
@@ -39,7 +37,7 @@ READ_SCOPES = f"{BASE_SCOPES} {READ_SCOPE}"
|
||||
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
||||
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
||||
|
||||
log = logging.getLogger("siftlode.auth")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def google_enabled(db: Session) -> bool:
|
||||
|
||||
@@ -11,12 +11,12 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
@@ -24,7 +24,7 @@ from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
|
||||
@@ -23,19 +23,19 @@ Security controls enforced here:
|
||||
logged, or written to a job's error/snapshot.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class HostSpec:
|
||||
|
||||
@@ -23,7 +23,6 @@ for a player hover-preview later.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
@@ -33,9 +32,10 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.edit")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def encode_thread_cap() -> int:
|
||||
@@ -368,7 +368,9 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
|
||||
@@ -16,7 +16,6 @@ Five passes each run:
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -24,14 +23,13 @@ from pathlib import Path
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
@@ -9,7 +8,7 @@ from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
@@ -29,7 +28,7 @@ from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class _Aborted(Exception):
|
||||
|
||||
@@ -11,7 +11,6 @@ writes the terminal job/asset state itself (done / error, or a cooperative pause
|
||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -24,7 +23,7 @@ from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logging.getLogger("siftlode.worker.runner")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _deprioritize() -> None:
|
||||
@@ -57,7 +56,11 @@ def main() -> None:
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
# noqa T201: this runs BEFORE configure_logging, and a usage error has to reach the
|
||||
# operator who typed it whether or not logging was ever set up.
|
||||
print( # noqa: T201
|
||||
"usage: python -m app.downloads.runner <job_id>", file=sys.stderr
|
||||
)
|
||||
raise SystemExit(2)
|
||||
job_id = int(sys.argv[1])
|
||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||
|
||||
@@ -11,7 +11,6 @@ to be a person telling them nothing arrived.
|
||||
"""
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import smtplib
|
||||
import ssl
|
||||
@@ -21,15 +20,14 @@ from email.utils import formatdate
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import mailtext, sysconfig
|
||||
from app import logs, mailtext, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.notifications import create_notification
|
||||
from app.ratelimit import RateLimiter
|
||||
|
||||
log = logging.getLogger("siftlode.email")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
||||
# stay readable; the log keeps the full record either way.
|
||||
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
+19
-2
@@ -15,7 +15,7 @@ issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
log = logging.getLogger("siftlode.downloads")
|
||||
log = logs.get_logger(__name__) # app/downloads/pipeline.py -> siftlode.downloads.pipeline
|
||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||
|
||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||
@@ -401,6 +401,20 @@ _THIRD_PARTY_LEVELS = {
|
||||
}
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""The logger for one module: `logs.get_logger(__name__)` in `app/plex/sync.py` is
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again."""
|
||||
stem = name.removeprefix("app.") if name.startswith("app.") else ""
|
||||
# logger-name-ok: this call IS the derivation
|
||||
return logging.getLogger(f"siftlode.{stem}" if stem else "siftlode")
|
||||
|
||||
|
||||
def resolve_level(name: str) -> int:
|
||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||
@@ -566,7 +580,10 @@ _REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
# A FAILURE on either still logs at WARNING/ERROR — this lowers the floor, it does not hide.
|
||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/")
|
||||
|
||||
_request_log = logging.getLogger("siftlode.http")
|
||||
# Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is
|
||||
# what an operator pins a level on when they want request lines and nothing else. The module it
|
||||
# happens to live in is not what the name is about.
|
||||
_request_log = logging.getLogger("siftlode.http") # logger-name-ok: named for the traffic
|
||||
|
||||
|
||||
async def log_requests(request, call_next):
|
||||
|
||||
@@ -66,7 +66,7 @@ _LIMITS = {
|
||||
"exc": 32_000,
|
||||
}
|
||||
|
||||
_log = logging.getLogger("siftlode.logsink")
|
||||
_log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def should_store(record: logging.LogRecord) -> bool:
|
||||
@@ -254,8 +254,12 @@ class LogSink(logging.Handler):
|
||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||
conn.execute(insert(LogEvent), without_erased(rows))
|
||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||
print(
|
||||
f"log sink: could not store {len(rows)} log events: {logs.redact_text(str(exc))[:300]}",
|
||||
# noqa T201: the sink's OWN failure. A log call here would be handed to this sink,
|
||||
# which is the thing that just failed — straight into a loop. stderr is the one way
|
||||
# out that does not re-enter.
|
||||
print( # noqa: T201
|
||||
f"log sink: could not store {len(rows)} log events: "
|
||||
f"{logs.redact_text(str(exc))[:300]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
+2
-2
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
@@ -12,11 +11,12 @@ from fastapi.exceptions import RequestValidationError
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
configure_logging("api")
|
||||
|
||||
log = logging.getLogger("siftlode.app")
|
||||
log = logs.get_logger(__name__)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
@@ -6,16 +6,15 @@ metadata, image proxying, and (P5) optional watch-state write-back. Requests JSO
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class PlexError(Exception):
|
||||
|
||||
@@ -18,7 +18,6 @@ segments. The remux is CPU-light (video copy + a tiny audio transcode), so it ru
|
||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
@@ -31,15 +30,14 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads.edit import encode_thread_cap
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths
|
||||
from app.plex.sync import HW_DECODABLE_CODECS
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_HLS_ROOT = (
|
||||
Path(settings.plex_hls_dir)
|
||||
if settings.plex_hls_dir
|
||||
|
||||
@@ -33,7 +33,6 @@ zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -44,6 +43,7 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths as plex_paths
|
||||
@@ -51,8 +51,7 @@ from app.plex.cache import CACHE_DIR
|
||||
from app.plex.client import PlexClient, PlexError
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
||||
BITMAP_SUBS = {
|
||||
"pgs",
|
||||
|
||||
@@ -15,20 +15,18 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
||||
item-detail/player path (P2), not here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||
_BROWSER_VIDEO = {"h264"}
|
||||
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
||||
|
||||
@@ -10,7 +10,6 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -19,12 +18,11 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
|
||||
log = logging.getLogger("siftlode.tmdb")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
_MIN_INTERVAL = 0.03
|
||||
|
||||
|
||||
@@ -17,14 +17,13 @@ and the incremental pull resolves a genuine divergence to Plex's state (see `_pu
|
||||
running a last-write-wins comparison.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem, PlexLink, PlexState
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -36,8 +35,7 @@ from app.plex.sync import (
|
||||
_paginate,
|
||||
)
|
||||
|
||||
log = logging.getLogger("siftlode.plex.watch")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
||||
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
||||
_PROGRESS_MIN_S = 5
|
||||
|
||||
@@ -13,12 +13,12 @@ mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logging.getLogger("siftlode.realtime")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
||||
(subscribe / unsubscribe) that require the optional write scope."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
@@ -10,7 +9,7 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -35,7 +34,7 @@ from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||
|
||||
log = logging.getLogger("siftlode.api")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("")
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
"""Shared helpers, serializers and constants for the plex route package."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexItem
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Plex people: cast/crew filter-chip cards + a single person's filmography page."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -8,6 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.auth import current_user
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import (
|
||||
@@ -22,7 +22,7 @@ from app.plex.cards import movie_card, show_card
|
||||
from app.plex.query import content_libs, split_csv
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_known_for(
|
||||
|
||||
@@ -13,7 +13,6 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi
|
||||
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
||||
dropped from the search response up front)."""
|
||||
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -45,8 +44,7 @@ from app.youtube import search_scrape
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
log = logging.getLogger("siftlode.search")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
# search.list costs this many quota units per page.
|
||||
|
||||
+10
-12
@@ -2,7 +2,6 @@
|
||||
recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||
|
||||
import contextvars
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Callable
|
||||
@@ -10,7 +9,7 @@ from typing import Callable
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import audit, logsink, progress, quota
|
||||
from app import audit, logs, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
@@ -35,8 +34,7 @@ from app.sync.runner import (
|
||||
run_subscription_resync,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("siftlode.scheduler")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_scheduler: BackgroundScheduler | None = None
|
||||
|
||||
# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the
|
||||
@@ -104,7 +102,7 @@ def apply_interval(db, job_id: str, minutes: int) -> int:
|
||||
db.commit()
|
||||
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
||||
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
||||
logger.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
log.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||
return minutes
|
||||
|
||||
|
||||
@@ -131,7 +129,7 @@ def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | Non
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("failed to write completion notification for job %s", job_id)
|
||||
log.exception("failed to write completion notification for job %s", job_id)
|
||||
|
||||
|
||||
def _run_changed(result) -> bool:
|
||||
@@ -167,7 +165,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.exception("failed to write audit row for job %s", job_id)
|
||||
log.exception("failed to write audit row for job %s", job_id)
|
||||
|
||||
|
||||
def _job(name: str, fn) -> None:
|
||||
@@ -175,7 +173,7 @@ def _job(name: str, fn) -> None:
|
||||
# HERE, atomically: whichever reaches this first wins, the other skips. This is the real
|
||||
# single-run guard — trigger_job's pre-check is only a fast best-effort answer for the UI.
|
||||
if not _claim_running(name):
|
||||
logger.info("job %s already running — skipping this trigger", name)
|
||||
log.info("job %s already running — skipping this trigger", name)
|
||||
return
|
||||
db = SessionLocal()
|
||||
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
||||
@@ -188,7 +186,7 @@ def _job(name: str, fn) -> None:
|
||||
|
||||
try:
|
||||
if is_sync_paused(db):
|
||||
logger.info("job %s skipped (sync paused)", name)
|
||||
log.info("job %s skipped (sync paused)", name)
|
||||
_record(
|
||||
name,
|
||||
status="skipped",
|
||||
@@ -203,14 +201,14 @@ def _job(name: str, fn) -> None:
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
summary = _summarize(result)
|
||||
logger.info("job %s -> %s", name, result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "ok", summary)
|
||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.exception("job %s failed", name)
|
||||
log.exception("job %s failed", name)
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
||||
if actor_id is not None:
|
||||
@@ -493,7 +491,7 @@ def start_scheduler() -> None:
|
||||
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
||||
scheduler.start()
|
||||
_scheduler = scheduler
|
||||
logger.info("scheduler started")
|
||||
log.info("scheduler started")
|
||||
|
||||
|
||||
def shutdown_scheduler() -> None:
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
"""Global admin-controlled app state (e.g. pausing background sync, first-run setup)."""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import AppState
|
||||
from app.security import hash_token
|
||||
|
||||
log = logging.getLogger("siftlode.state")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
||||
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
||||
_configured_cache = False
|
||||
|
||||
@@ -6,17 +6,16 @@ topicDetails categories and the channel's dominant video category. System tags a
|
||||
regenerated freely; user tags are never touched here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy import and_, exists, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
log = logging.getLogger("siftlode.autotag")
|
||||
|
||||
from app import progress, sysconfig
|
||||
from app import logs, progress, sysconfig
|
||||
from app.models import Channel, ChannelTag, Tag, Video
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# --- language id (offline, deterministic, small) ---
|
||||
try:
|
||||
from py3langid.langid import MODEL_FILE, LanguageIdentifier
|
||||
|
||||
@@ -8,13 +8,12 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
|
||||
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
||||
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import (
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
@@ -28,7 +27,7 @@ from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_d
|
||||
from app.utils import now_utc as _now
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
log = logging.getLogger("siftlode.explore")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def followed_channel():
|
||||
|
||||
@@ -20,13 +20,12 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
|
||||
backfill reserve as the other jobs so interactive syncs never starve.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota, sysconfig
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video
|
||||
from app.notifications import create_notification
|
||||
@@ -35,8 +34,7 @@ from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.utils import now_utc as _now
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
log = logging.getLogger("siftlode.maintenance")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Reasons recorded on Video.unavailable_reason. "removed" = absent from videos.list
|
||||
# (deleted or made private); "abandoned" = an upcoming premiere that never went live.
|
||||
GRACE_DAYS_BY_REASON = {
|
||||
|
||||
@@ -8,7 +8,6 @@ ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
@@ -17,15 +16,14 @@ from typing import TypeVar
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
||||
from app.sync.runner import token_users
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||
WRITE_UNIT_COST = 50
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
log = logs.get_logger(__name__)
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.explore import followed_channel
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import Channel, Subscription, User
|
||||
from app.utils import normalize_description
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _to_int(value) -> int | None:
|
||||
|
||||
@@ -8,17 +8,15 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr
|
||||
stored in the DB and stay env-only).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import security
|
||||
from app import logs, security
|
||||
from app.config import Settings, canonical_choice, choices_of, settings
|
||||
from app.models import SystemConfig
|
||||
|
||||
log = logging.getLogger("siftlode.config")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
||||
# key, not once per request (bounded by the number of secret config keys).
|
||||
_decrypt_warned: set[str] = set()
|
||||
|
||||
@@ -16,7 +16,6 @@ Design:
|
||||
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
@@ -45,8 +44,7 @@ from app.downloads.jobdb import (
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
|
||||
|
||||
@@ -73,13 +71,14 @@ def _wait_for_schema(timeout: float = 300.0) -> bool:
|
||||
with SessionLocal() as db:
|
||||
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 — table not migrated yet / DB not up yet
|
||||
except Exception: # noqa: BLE001 — table not migrated yet / DB not up yet
|
||||
if time.monotonic() > deadline:
|
||||
log.error(
|
||||
"download schema still absent after %ss — the API's migrations have not landed; "
|
||||
"exiting so the container restarts: %s",
|
||||
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
||||
# before it exits, and which driver call failed is the whole question.
|
||||
log.exception(
|
||||
"download schema still absent after %ss — the API's migrations have not "
|
||||
"landed; exiting so the container restarts",
|
||||
timeout,
|
||||
str(exc)[:120],
|
||||
)
|
||||
return False
|
||||
log.info("waiting for the database schema (API migrations)…")
|
||||
|
||||
@@ -6,7 +6,6 @@ so they don't depend on a specific user's token; subscriptions.list?mine=true al
|
||||
uses OAuth.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -16,8 +15,7 @@ from app import logs, quota, sysconfig
|
||||
from app.models import User
|
||||
from app.security import decrypt, decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.youtube")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
|
||||
API_BASE = "https://www.googleapis.com/youtube/v3"
|
||||
|
||||
|
||||
@@ -9,4 +9,19 @@
|
||||
line-length = 100
|
||||
|
||||
[lint]
|
||||
# Beyond the default set, the rules that hold a LOG LINE to what E22.5 makes of it:
|
||||
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
||||
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
||||
# never `str(exc)` where `exc_info` would carry the traceback
|
||||
# T20 — `print` instead of a log line. Two deliberate ones remain, each with its `# noqa`
|
||||
# and a why: `downloads/runner.py` prints before logging is configured, and
|
||||
# `logsink.py` reports the SINK's own failure, which cannot be logged into the sink.
|
||||
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
||||
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
||||
# S112 — `except: continue`, the silent swallow inside a loop
|
||||
# S110 (`except: pass`) and BLE001 (blind `except Exception`) are NOT here yet: 5 and 18 sites are
|
||||
# outstanding, and each is a judgement about one handler — E22.5 S3 rules on them and turns the two
|
||||
# rules on in the same commit. A rule enabled ahead of that work is a rule enabled with 23 blanket
|
||||
# `noqa`s, which is worse than no rule.
|
||||
extend-select = ["LOG", "G", "T20", "TRY400", "TRY401", "S112"]
|
||||
ignore = ["E402"]
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""E22.5 S2 — every module's logger is named after the module.
|
||||
|
||||
The names used to be literals, and 25 of them covered 34 modules: `plex/client.py`, `plex/stream.py`,
|
||||
`plex/subtitles.py`, `plex/sync.py` and two route modules all called themselves `siftlode.plex`, and
|
||||
`downloads/pipeline.py` called itself `siftlode.worker`. A level pinned per logger could not separate
|
||||
them, and neither can E22.6's console filter — the `logger` field is in every stored row precisely so
|
||||
it can be filtered on.
|
||||
|
||||
So the name is derived, not written: `logs.get_logger(__name__)`. These cases pin the derivation and
|
||||
the rule, because the rule is only worth having if a hand-rolled name fails the gate.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import logging
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from app import logs
|
||||
|
||||
APP = pathlib.Path(logs.__file__).resolve().parent
|
||||
_ESCAPE = "logger-name-ok"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module", "expected"),
|
||||
[
|
||||
("app.plex.sync", "siftlode.plex.sync"),
|
||||
("app.plex.stream", "siftlode.plex.stream"),
|
||||
("app.routes.channels", "siftlode.routes.channels"),
|
||||
("app.worker", "siftlode.worker"),
|
||||
# Not a module of ours (a script, a doctest): it lands on the root of our tree rather than
|
||||
# inventing `siftlode.__main__`, which nothing would think to filter on.
|
||||
("__main__", "siftlode"),
|
||||
],
|
||||
)
|
||||
def test_the_name_follows_the_module(module, expected):
|
||||
assert logs.get_logger(module).name == expected
|
||||
|
||||
|
||||
def test_every_derived_logger_stays_under_the_one_tree():
|
||||
"""`configure_logging` hangs the level and the handler off the root, and the tests attach their
|
||||
sink to `siftlode` — a name outside that tree would be invisible to both."""
|
||||
for module in ("app.main", "app.plex.sync", "app.routes.plex.people", "app"):
|
||||
assert logs.get_logger(module).name.startswith("siftlode")
|
||||
|
||||
|
||||
def test_the_module_logger_is_reachable_by_its_derived_name():
|
||||
"""Not a new logger object per call: the stdlib registry is keyed by name, and an operator
|
||||
pinning `siftlode.plex.sync` has to reach the same instance the module logs through."""
|
||||
from app.plex import sync
|
||||
|
||||
assert sync.log is logging.getLogger("siftlode.plex.sync")
|
||||
|
||||
|
||||
def test_no_module_hand_rolls_a_logger_name():
|
||||
offenders = []
|
||||
for path in sorted(APP.rglob("*.py")):
|
||||
src = path.read_text(encoding="utf-8")
|
||||
lines = src.splitlines()
|
||||
for node in ast.walk(ast.parse(src)):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
func = node.func
|
||||
if not (isinstance(func, ast.Attribute) and func.attr == "getLogger"):
|
||||
continue
|
||||
# A no-argument `logging.getLogger()` is the ROOT logger — level plumbing, not a
|
||||
# module's own logger, and nothing to name.
|
||||
if not node.args:
|
||||
continue
|
||||
start = max(0, node.lineno - 2) # the call's lines, plus the one above it
|
||||
span = " ".join(lines[start : (node.end_lineno or node.lineno)])
|
||||
if _ESCAPE in span:
|
||||
continue
|
||||
offenders.append(f"{path.relative_to(APP)}:{node.lineno}")
|
||||
assert offenders == [], (
|
||||
"a module names its own logger; use `logs.get_logger(__name__)`, or say why the name is "
|
||||
f"not the module's with `# {_ESCAPE}: <why>` — {offenders}"
|
||||
)
|
||||
@@ -225,7 +225,9 @@ def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
||||
log.addHandler(sink)
|
||||
try:
|
||||
for i in range(logsink.BATCH_MAX + 5):
|
||||
log.warning(f"line {i}") # distinct templates: no flood limiter involved
|
||||
# noqa G004: the f-string is the POINT — a distinct rendered message per line is
|
||||
# what keeps the flood limiter (which groups by template) out of this case.
|
||||
log.warning(f"line {i}") # noqa: G004
|
||||
finally:
|
||||
log.removeHandler(sink)
|
||||
sink._stopping.set()
|
||||
|
||||
+2
-1
@@ -166,7 +166,8 @@ def main() -> None:
|
||||
saved = svid.id
|
||||
db.commit()
|
||||
|
||||
print(
|
||||
# noqa T201: this script's stdout IS its interface — the e2e runner parses this JSON.
|
||||
print( # noqa: T201
|
||||
json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
|
||||
Reference in New Issue
Block a user