refactor(logs): name every logger after its module, and hold it there
25 literal names covered 34 modules. plex/client.py, plex/stream.py, plex/subtitles.py, plex/sync.py and two Plex route modules all called themselves `siftlode.plex`; downloads/pipeline.py called itself `siftlode.worker`; routes/channels.py called itself `siftlode.api`. The `logger` column is in every stored row so E22.6's console can filter on it, and so an operator can pin a level to one part of the system — neither works when six modules share a name. `logs.get_logger(__name__)` derives it: app/plex/sync.py -> siftlode.plex.sync. One name per module, and it cannot drift from the file it sits in. tests/test_logger_names.py pins the derivation, that everything stays under the one `siftlode` tree the handler and the tests attach to, and that no module hand-rolls a name again unless the line says why (`# logger-name-ok: <why>`). Two do: get_logger's own call, and `siftlode.http`, which is named for the traffic it carries rather than the module it lives in — it replaced uvicorn's access log and is what an operator pins when they want request lines and nothing else. app/scheduler.py called its logger `logger`; every other module calls it `log`. ruff gains LOG, G, T20, TRY400, TRY401 and S112. Four findings cleared with them: worker.py logged `str(exc)[:120]` where the traceback was the whole question, and three prints/f-strings that are deliberate now say so on the line. S110 and BLE001 stay off until S3 — 23 handlers are outstanding and each is a judgement about one of them; enabling the rules first would just mean 23 blanket noqas.
This commit is contained in:
+1
-3
@@ -1,5 +1,3 @@
|
|||||||
import logging
|
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
||||||
from sqlalchemy import delete, func, select
|
from sqlalchemy import delete, func, select
|
||||||
@@ -39,7 +37,7 @@ READ_SCOPES = f"{BASE_SCOPES} {READ_SCOPE}"
|
|||||||
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
||||||
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.auth")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def google_enabled(db: Session) -> bool:
|
def google_enabled(db: Session) -> bool:
|
||||||
|
|||||||
@@ -11,12 +11,12 @@ Run once per environment:
|
|||||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import adapters
|
from app.downloads import adapters
|
||||||
@@ -24,7 +24,7 @@ from app.downloads import edit as editmod
|
|||||||
from app.downloads import formats, service
|
from app.downloads import formats, service
|
||||||
from app.models import MediaAsset
|
from app.models import MediaAsset
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.backfill")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||||
|
|||||||
@@ -23,19 +23,19 @@ Security controls enforced here:
|
|||||||
logged, or written to a job's error/snapshot.
|
logged, or written to a job's error/snapshot.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.downloads import formats
|
from app.downloads import formats
|
||||||
from app.downloads.adapters import _host_is
|
from app.downloads.adapters import _host_is
|
||||||
from app.models import DownloadCredential, MediaAsset
|
from app.models import DownloadCredential, MediaAsset
|
||||||
from app.security import decrypt_optional, encrypt
|
from app.security import decrypt_optional, encrypt
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.downloads.credentials")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class HostSpec:
|
class HostSpec:
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ for a player hover-preview later.
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import queue
|
import queue
|
||||||
@@ -33,9 +32,10 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.edit")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def encode_thread_cap() -> int:
|
def encode_thread_cap() -> int:
|
||||||
@@ -368,7 +368,9 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
|||||||
try:
|
try:
|
||||||
proc.wait(timeout=grace_s)
|
proc.wait(timeout=grace_s)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||||
|
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||||
|
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||||
|
|
||||||
|
|
||||||
def _stop(proc: subprocess.Popen) -> None:
|
def _stop(proc: subprocess.Popen) -> None:
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ Five passes each run:
|
|||||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -24,14 +23,13 @@ from pathlib import Path
|
|||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads import storage
|
from app.downloads import storage
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.models import DownloadJob, MediaAsset
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.downloads.gc")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
import time
|
||||||
@@ -9,7 +8,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
@@ -29,7 +28,7 @@ from app.models import DownloadJob, MediaAsset
|
|||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.titles import normalize_title
|
from app.titles import normalize_title
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.worker")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class _Aborted(Exception):
|
class _Aborted(Exception):
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ writes the terminal job/asset state itself (done / error, or a cooperative pause
|
|||||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -24,7 +23,7 @@ from app.db import SessionLocal
|
|||||||
from app.downloads import edit as editmod
|
from app.downloads import edit as editmod
|
||||||
from app.models import DownloadJob
|
from app.models import DownloadJob
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.worker.runner")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _deprioritize() -> None:
|
def _deprioritize() -> None:
|
||||||
@@ -57,7 +56,11 @@ def main() -> None:
|
|||||||
logs.configure_logging("runner")
|
logs.configure_logging("runner")
|
||||||
_deprioritize()
|
_deprioritize()
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
# noqa T201: this runs BEFORE configure_logging, and a usage error has to reach the
|
||||||
|
# operator who typed it whether or not logging was ever set up.
|
||||||
|
print( # noqa: T201
|
||||||
|
"usage: python -m app.downloads.runner <job_id>", file=sys.stderr
|
||||||
|
)
|
||||||
raise SystemExit(2)
|
raise SystemExit(2)
|
||||||
job_id = int(sys.argv[1])
|
job_id = int(sys.argv[1])
|
||||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ to be a person telling them nothing arrived.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import html
|
import html
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import smtplib
|
import smtplib
|
||||||
import ssl
|
import ssl
|
||||||
@@ -21,15 +20,14 @@ from email.utils import formatdate
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import mailtext, sysconfig
|
from app import logs, mailtext, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.ratelimit import RateLimiter
|
from app.ratelimit import RateLimiter
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.email")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
||||||
# stay readable; the log keeps the full record either way.
|
# stay readable; the log keeps the full record either way.
|
||||||
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
||||||
|
|||||||
+19
-2
@@ -15,7 +15,7 @@ issue attachment will carry, so it is deliberately flat and stable.
|
|||||||
|
|
||||||
Call sites stay ordinary stdlib logging:
|
Call sites stay ordinary stdlib logging:
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.downloads")
|
log = logs.get_logger(__name__) # app/downloads/pipeline.py -> siftlode.downloads.pipeline
|
||||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||||
|
|
||||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||||
@@ -401,6 +401,20 @@ _THIRD_PARTY_LEVELS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_logger(name: str) -> logging.Logger:
|
||||||
|
"""The logger for one module: `logs.get_logger(__name__)` in `app/plex/sync.py` is
|
||||||
|
`siftlode.plex.sync`.
|
||||||
|
|
||||||
|
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||||
|
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||||
|
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||||
|
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||||
|
module hand-rolls one again."""
|
||||||
|
stem = name.removeprefix("app.") if name.startswith("app.") else ""
|
||||||
|
# logger-name-ok: this call IS the derivation
|
||||||
|
return logging.getLogger(f"siftlode.{stem}" if stem else "siftlode")
|
||||||
|
|
||||||
|
|
||||||
def resolve_level(name: str) -> int:
|
def resolve_level(name: str) -> int:
|
||||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||||
@@ -566,7 +580,10 @@ _REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
|||||||
# A FAILURE on either still logs at WARNING/ERROR — this lowers the floor, it does not hide.
|
# A FAILURE on either still logs at WARNING/ERROR — this lowers the floor, it does not hide.
|
||||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/")
|
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/")
|
||||||
|
|
||||||
_request_log = logging.getLogger("siftlode.http")
|
# Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is
|
||||||
|
# what an operator pins a level on when they want request lines and nothing else. The module it
|
||||||
|
# happens to live in is not what the name is about.
|
||||||
|
_request_log = logging.getLogger("siftlode.http") # logger-name-ok: named for the traffic
|
||||||
|
|
||||||
|
|
||||||
async def log_requests(request, call_next):
|
async def log_requests(request, call_next):
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ _LIMITS = {
|
|||||||
"exc": 32_000,
|
"exc": 32_000,
|
||||||
}
|
}
|
||||||
|
|
||||||
_log = logging.getLogger("siftlode.logsink")
|
_log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def should_store(record: logging.LogRecord) -> bool:
|
def should_store(record: logging.LogRecord) -> bool:
|
||||||
@@ -254,8 +254,12 @@ class LogSink(logging.Handler):
|
|||||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||||
conn.execute(insert(LogEvent), without_erased(rows))
|
conn.execute(insert(LogEvent), without_erased(rows))
|
||||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||||
print(
|
# noqa T201: the sink's OWN failure. A log call here would be handed to this sink,
|
||||||
f"log sink: could not store {len(rows)} log events: {logs.redact_text(str(exc))[:300]}",
|
# which is the thing that just failed — straight into a loop. stderr is the one way
|
||||||
|
# out that does not re-enter.
|
||||||
|
print( # noqa: T201
|
||||||
|
f"log sink: could not store {len(rows)} log events: "
|
||||||
|
f"{logs.redact_text(str(exc))[:300]}",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,4 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
|
||||||
import math
|
import math
|
||||||
from contextlib import asynccontextmanager, suppress
|
from contextlib import asynccontextmanager, suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -12,11 +11,12 @@ from fastapi.exceptions import RequestValidationError
|
|||||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||||
|
from app import logs
|
||||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||||
|
|
||||||
configure_logging("api")
|
configure_logging("api")
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.app")
|
log = logs.get_logger(__name__)
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|||||||
@@ -6,16 +6,15 @@ metadata, image proxying, and (P5) optional watch-state write-back. Requests JSO
|
|||||||
``Accept: application/json`` header (Plex serves XML by default).
|
``Accept: application/json`` header (Plex serves XML by default).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class PlexError(Exception):
|
class PlexError(Exception):
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ segments. The remux is CPU-light (video copy + a tiny audio transcode), so it ru
|
|||||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import secrets
|
import secrets
|
||||||
@@ -31,15 +30,14 @@ from pathlib import Path
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session as DbSession
|
from sqlalchemy.orm import Session as DbSession
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.downloads.edit import encode_thread_cap
|
from app.downloads.edit import encode_thread_cap
|
||||||
from app.models import PlexItem
|
from app.models import PlexItem
|
||||||
from app.plex import paths
|
from app.plex import paths
|
||||||
from app.plex.sync import HW_DECODABLE_CODECS
|
from app.plex.sync import HW_DECODABLE_CODECS
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
_HLS_ROOT = (
|
_HLS_ROOT = (
|
||||||
Path(settings.plex_hls_dir)
|
Path(settings.plex_hls_dir)
|
||||||
if settings.plex_hls_dir
|
if settings.plex_hls_dir
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -44,6 +43,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import PlexItem
|
from app.models import PlexItem
|
||||||
from app.plex import paths as plex_paths
|
from app.plex import paths as plex_paths
|
||||||
@@ -51,8 +51,7 @@ from app.plex.cache import CACHE_DIR
|
|||||||
from app.plex.client import PlexClient, PlexError
|
from app.plex.client import PlexClient, PlexError
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
||||||
BITMAP_SUBS = {
|
BITMAP_SUBS = {
|
||||||
"pgs",
|
"pgs",
|
||||||
|
|||||||
@@ -15,20 +15,18 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
|||||||
item-detail/player path (P2), not here.
|
item-detail/player path (P2), not here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||||
from app.plex import detail, people
|
from app.plex import detail, people
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||||
_BROWSER_VIDEO = {"h264"}
|
_BROWSER_VIDEO = {"h264"}
|
||||||
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
|||||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
@@ -19,12 +18,11 @@ from sqlalchemy import func, select
|
|||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import PlexItem, PlexPerson, PlexShow
|
from app.models import PlexItem, PlexPerson, PlexShow
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.tmdb")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||||
_MIN_INTERVAL = 0.03
|
_MIN_INTERVAL = 0.03
|
||||||
|
|
||||||
|
|||||||
@@ -17,14 +17,13 @@ and the incremental pull resolves a genuine divergence to Plex's state (see `_pu
|
|||||||
running a last-write-wins comparison.
|
running a last-write-wins comparison.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import PlexItem, PlexLink, PlexState
|
from app.models import PlexItem, PlexLink, PlexState
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
@@ -36,8 +35,7 @@ from app.plex.sync import (
|
|||||||
_paginate,
|
_paginate,
|
||||||
)
|
)
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex.watch")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
||||||
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
||||||
_PROGRESS_MIN_S = 5
|
_PROGRESS_MIN_S = 5
|
||||||
|
|||||||
@@ -13,12 +13,12 @@ mutated from a single thread.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
from starlette.websockets import WebSocket
|
from starlette.websockets import WebSocket
|
||||||
|
from app import logs
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.realtime")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class ConnectionManager:
|
class ConnectionManager:
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
||||||
(subscribe / unsubscribe) that require the optional write scope."""
|
(subscribe / unsubscribe) that require the optional write scope."""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||||
@@ -10,7 +9,7 @@ from pydantic import BaseModel
|
|||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import quota
|
from app import logs, quota
|
||||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.models import (
|
from app.models import (
|
||||||
@@ -35,7 +34,7 @@ from app.youtube.client import YouTubeClient, YouTubeError
|
|||||||
|
|
||||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.api")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
"""Shared helpers, serializers and constants for the plex route package."""
|
"""Shared helpers, serializers and constants for the plex route package."""
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.models import PlexItem
|
from app.models import PlexItem
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
"""Plex people: cast/crew filter-chip cards + a single person's filmography page."""
|
"""Plex people: cast/crew filter-chip cards + a single person's filmography page."""
|
||||||
|
|
||||||
import logging
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
|
||||||
@@ -8,6 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends
|
|||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import SessionLocal, get_db
|
from app.db import SessionLocal, get_db
|
||||||
from app.models import (
|
from app.models import (
|
||||||
@@ -22,7 +22,7 @@ from app.plex.cards import movie_card, show_card
|
|||||||
from app.plex.query import content_libs, split_csv
|
from app.plex.query import content_libs, split_csv
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||||
log = logging.getLogger("siftlode.plex")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_known_for(
|
def _resolve_known_for(
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ this search-discovered "noise" by default. Shorts and currently-live/upcoming vi
|
|||||||
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
ingested this way (Shorts are confirmed by the youtube.com/shorts probe; live/upcoming are
|
||||||
dropped from the search response up front)."""
|
dropped from the search response up front)."""
|
||||||
|
|
||||||
import logging
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
@@ -45,8 +44,7 @@ from app.youtube import search_scrape
|
|||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
from app.youtube.shorts import make_client, probe_is_short
|
from app.youtube.shorts import make_client, probe_is_short
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.search")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||||
|
|
||||||
# search.list costs this many quota units per page.
|
# search.list costs this many quota units per page.
|
||||||
|
|||||||
+10
-12
@@ -2,7 +2,6 @@
|
|||||||
recent-first / deep backfill. Runs inside the API process (single worker)."""
|
recent-first / deep backfill. Runs inside the API process (single worker)."""
|
||||||
|
|
||||||
import contextvars
|
import contextvars
|
||||||
import logging
|
|
||||||
import threading
|
import threading
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Callable
|
from typing import Callable
|
||||||
@@ -10,7 +9,7 @@ from typing import Callable
|
|||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app import audit, logsink, progress, quota
|
from app import audit, logs, logsink, progress, quota
|
||||||
from app.audit import AuditAction
|
from app.audit import AuditAction
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
@@ -35,8 +34,7 @@ from app.sync.runner import (
|
|||||||
run_subscription_resync,
|
run_subscription_resync,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger("siftlode.scheduler")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
_scheduler: BackgroundScheduler | None = None
|
_scheduler: BackgroundScheduler | None = None
|
||||||
|
|
||||||
# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the
|
# Per-job run activity, kept in-memory for the admin Scheduler dashboard to read (the
|
||||||
@@ -104,7 +102,7 @@ def apply_interval(db, job_id: str, minutes: int) -> int:
|
|||||||
db.commit()
|
db.commit()
|
||||||
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
if _scheduler is not None and _scheduler.get_job(job_id) is not None:
|
||||||
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
_scheduler.reschedule_job(job_id, trigger="interval", minutes=minutes)
|
||||||
logger.info("rescheduled job %s to every %s min", job_id, minutes)
|
log.info("rescheduled job %s to every %s min", job_id, minutes)
|
||||||
return minutes
|
return minutes
|
||||||
|
|
||||||
|
|
||||||
@@ -131,7 +129,7 @@ def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | Non
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
logger.exception("failed to write completion notification for job %s", job_id)
|
log.exception("failed to write completion notification for job %s", job_id)
|
||||||
|
|
||||||
|
|
||||||
def _run_changed(result) -> bool:
|
def _run_changed(result) -> bool:
|
||||||
@@ -167,7 +165,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
|||||||
db.commit()
|
db.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
logger.exception("failed to write audit row for job %s", job_id)
|
log.exception("failed to write audit row for job %s", job_id)
|
||||||
|
|
||||||
|
|
||||||
def _job(name: str, fn) -> None:
|
def _job(name: str, fn) -> None:
|
||||||
@@ -175,7 +173,7 @@ def _job(name: str, fn) -> None:
|
|||||||
# HERE, atomically: whichever reaches this first wins, the other skips. This is the real
|
# HERE, atomically: whichever reaches this first wins, the other skips. This is the real
|
||||||
# single-run guard — trigger_job's pre-check is only a fast best-effort answer for the UI.
|
# single-run guard — trigger_job's pre-check is only a fast best-effort answer for the UI.
|
||||||
if not _claim_running(name):
|
if not _claim_running(name):
|
||||||
logger.info("job %s already running — skipping this trigger", name)
|
log.info("job %s already running — skipping this trigger", name)
|
||||||
return
|
return
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
||||||
@@ -188,7 +186,7 @@ def _job(name: str, fn) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
if is_sync_paused(db):
|
if is_sync_paused(db):
|
||||||
logger.info("job %s skipped (sync paused)", name)
|
log.info("job %s skipped (sync paused)", name)
|
||||||
_record(
|
_record(
|
||||||
name,
|
name,
|
||||||
status="skipped",
|
status="skipped",
|
||||||
@@ -203,14 +201,14 @@ def _job(name: str, fn) -> None:
|
|||||||
with progress.bind(sink):
|
with progress.bind(sink):
|
||||||
result = fn(db)
|
result = fn(db)
|
||||||
summary = _summarize(result)
|
summary = _summarize(result)
|
||||||
logger.info("job %s -> %s", name, result)
|
log.info("job %s -> %s", name, result)
|
||||||
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
_notify_done(db, actor_id, name, "ok", summary)
|
_notify_done(db, actor_id, name, "ok", summary)
|
||||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
logger.exception("job %s failed", name)
|
log.exception("job %s failed", name)
|
||||||
err = str(exc) or exc.__class__.__name__
|
err = str(exc) or exc.__class__.__name__
|
||||||
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
@@ -493,7 +491,7 @@ def start_scheduler() -> None:
|
|||||||
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
scheduler.add_job(fn, "interval", minutes=iv[job_id], id=job_id)
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
_scheduler = scheduler
|
_scheduler = scheduler
|
||||||
logger.info("scheduler started")
|
log.info("scheduler started")
|
||||||
|
|
||||||
|
|
||||||
def shutdown_scheduler() -> None:
|
def shutdown_scheduler() -> None:
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
"""Global admin-controlled app state (e.g. pausing background sync, first-run setup)."""
|
"""Global admin-controlled app state (e.g. pausing background sync, first-run setup)."""
|
||||||
|
|
||||||
import logging
|
|
||||||
import secrets
|
import secrets
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.models import AppState
|
from app.models import AppState
|
||||||
from app.security import hash_token
|
from app.security import hash_token
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.state")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
# Once configuration completes in this process it never reverts, so cache it to skip the per-request
|
||||||
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
# DB hit in the setup-mode gate (see main.setup_gate). Starts False; refreshed from the DB.
|
||||||
_configured_cache = False
|
_configured_cache = False
|
||||||
|
|||||||
@@ -6,17 +6,16 @@ topicDetails categories and the channel's dominant video category. System tags a
|
|||||||
regenerated freely; user tags are never touched here.
|
regenerated freely; user tags are never touched here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from sqlalchemy import and_, exists, func, select
|
from sqlalchemy import and_, exists, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.autotag")
|
from app import logs, progress, sysconfig
|
||||||
|
|
||||||
from app import progress, sysconfig
|
|
||||||
from app.models import Channel, ChannelTag, Tag, Video
|
from app.models import Channel, ChannelTag, Tag, Video
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# --- language id (offline, deterministic, small) ---
|
# --- language id (offline, deterministic, small) ---
|
||||||
try:
|
try:
|
||||||
from py3langid.langid import MODEL_FILE, LanguageIdentifier
|
from py3langid.langid import MODEL_FILE, LanguageIdentifier
|
||||||
|
|||||||
@@ -8,13 +8,12 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
|
|||||||
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
||||||
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
from sqlalchemy import delete, or_, select
|
from sqlalchemy import delete, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import sysconfig
|
from app import logs, sysconfig
|
||||||
from app.models import (
|
from app.models import (
|
||||||
Channel,
|
Channel,
|
||||||
ExploredChannel,
|
ExploredChannel,
|
||||||
@@ -28,7 +27,7 @@ from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_d
|
|||||||
from app.utils import now_utc as _now
|
from app.utils import now_utc as _now
|
||||||
from app.youtube.client import YouTubeClient
|
from app.youtube.client import YouTubeClient
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.explore")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def followed_channel():
|
def followed_channel():
|
||||||
|
|||||||
@@ -20,13 +20,12 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
|
|||||||
backfill reserve as the other jobs so interactive syncs never starve.
|
backfill reserve as the other jobs so interactive syncs never starve.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import Playlist, PlaylistItem, Video
|
from app.models import Playlist, PlaylistItem, Video
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
@@ -35,8 +34,7 @@ from app.sync.videos import apply_video_details, parse_dt
|
|||||||
from app.utils import now_utc as _now
|
from app.utils import now_utc as _now
|
||||||
from app.youtube.client import YouTubeClient
|
from app.youtube.client import YouTubeClient
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.maintenance")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Reasons recorded on Video.unavailable_reason. "removed" = absent from videos.list
|
# Reasons recorded on Video.unavailable_reason. "removed" = absent from videos.list
|
||||||
# (deleted or made private); "abandoned" = an upcoming premiere that never went live.
|
# (deleted or made private); "abandoned" = an upcoming premiere that never went live.
|
||||||
GRACE_DAYS_BY_REASON = {
|
GRACE_DAYS_BY_REASON = {
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ ingested (with a stub channel) so the mirror is faithful."""
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
@@ -17,15 +16,14 @@ from typing import TypeVar
|
|||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import progress, quota
|
from app import logs, progress, quota
|
||||||
from app.auth import has_read_scope
|
from app.auth import has_read_scope
|
||||||
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
||||||
from app.sync.runner import token_users
|
from app.sync.runner import token_users
|
||||||
from app.sync.videos import apply_video_details, parse_dt
|
from app.sync.videos import apply_video_details, parse_dt
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.sync")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||||
WRITE_UNIT_COST = 50
|
WRITE_UNIT_COST = 50
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs, progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.sync")
|
log = logs.get_logger(__name__)
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import Channel, OAuthToken, Subscription, User
|
from app.models import Channel, OAuthToken, Subscription, User
|
||||||
from app.sync.explore import followed_channel
|
from app.sync.explore import followed_channel
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs
|
||||||
from app.models import Channel, Subscription, User
|
from app.models import Channel, Subscription, User
|
||||||
from app.utils import normalize_description
|
from app.utils import normalize_description
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.sync")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _to_int(value) -> int | None:
|
def _to_int(value) -> int | None:
|
||||||
|
|||||||
@@ -8,17 +8,15 @@ stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secr
|
|||||||
stored in the DB and stay env-only).
|
stored in the DB and stay env-only).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import security
|
from app import logs, security
|
||||||
from app.config import Settings, canonical_choice, choices_of, settings
|
from app.config import Settings, canonical_choice, choices_of, settings
|
||||||
from app.models import SystemConfig
|
from app.models import SystemConfig
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.config")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
||||||
# key, not once per request (bounded by the number of secret config keys).
|
# key, not once per request (bounded by the number of secret config keys).
|
||||||
_decrypt_warned: set[str] = set()
|
_decrypt_warned: set[str] = set()
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ Design:
|
|||||||
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
|
* Crash-safe: on startup, orphaned `running` jobs are requeued and `downloading` assets reset.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import signal
|
import signal
|
||||||
@@ -45,8 +44,7 @@ from app.downloads.jobdb import (
|
|||||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||||
from app.models import DownloadJob, MediaAsset
|
from app.models import DownloadJob, MediaAsset
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.worker")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
_stop = threading.Event()
|
_stop = threading.Event()
|
||||||
|
|
||||||
|
|
||||||
@@ -73,13 +71,14 @@ def _wait_for_schema(timeout: float = 300.0) -> bool:
|
|||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
||||||
return True
|
return True
|
||||||
except Exception as exc: # noqa: BLE001 — table not migrated yet / DB not up yet
|
except Exception: # noqa: BLE001 — table not migrated yet / DB not up yet
|
||||||
if time.monotonic() > deadline:
|
if time.monotonic() > deadline:
|
||||||
log.error(
|
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
||||||
"download schema still absent after %ss — the API's migrations have not landed; "
|
# before it exits, and which driver call failed is the whole question.
|
||||||
"exiting so the container restarts: %s",
|
log.exception(
|
||||||
|
"download schema still absent after %ss — the API's migrations have not "
|
||||||
|
"landed; exiting so the container restarts",
|
||||||
timeout,
|
timeout,
|
||||||
str(exc)[:120],
|
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
log.info("waiting for the database schema (API migrations)…")
|
log.info("waiting for the database schema (API migrations)…")
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ so they don't depend on a specific user's token; subscriptions.list?mine=true al
|
|||||||
uses OAuth.
|
uses OAuth.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
@@ -16,8 +15,7 @@ from app import logs, quota, sysconfig
|
|||||||
from app.models import User
|
from app.models import User
|
||||||
from app.security import decrypt, decrypt_optional, encrypt
|
from app.security import decrypt, decrypt_optional, encrypt
|
||||||
|
|
||||||
log = logging.getLogger("siftlode.youtube")
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
|
GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
|
||||||
API_BASE = "https://www.googleapis.com/youtube/v3"
|
API_BASE = "https://www.googleapis.com/youtube/v3"
|
||||||
|
|
||||||
|
|||||||
@@ -9,4 +9,19 @@
|
|||||||
line-length = 100
|
line-length = 100
|
||||||
|
|
||||||
[lint]
|
[lint]
|
||||||
|
# Beyond the default set, the rules that hold a LOG LINE to what E22.5 makes of it:
|
||||||
|
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
||||||
|
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
||||||
|
# never `str(exc)` where `exc_info` would carry the traceback
|
||||||
|
# T20 — `print` instead of a log line. Two deliberate ones remain, each with its `# noqa`
|
||||||
|
# and a why: `downloads/runner.py` prints before logging is configured, and
|
||||||
|
# `logsink.py` reports the SINK's own failure, which cannot be logged into the sink.
|
||||||
|
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
||||||
|
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
||||||
|
# S112 — `except: continue`, the silent swallow inside a loop
|
||||||
|
# S110 (`except: pass`) and BLE001 (blind `except Exception`) are NOT here yet: 5 and 18 sites are
|
||||||
|
# outstanding, and each is a judgement about one handler — E22.5 S3 rules on them and turns the two
|
||||||
|
# rules on in the same commit. A rule enabled ahead of that work is a rule enabled with 23 blanket
|
||||||
|
# `noqa`s, which is worse than no rule.
|
||||||
|
extend-select = ["LOG", "G", "T20", "TRY400", "TRY401", "S112"]
|
||||||
ignore = ["E402"]
|
ignore = ["E402"]
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""E22.5 S2 — every module's logger is named after the module.
|
||||||
|
|
||||||
|
The names used to be literals, and 25 of them covered 34 modules: `plex/client.py`, `plex/stream.py`,
|
||||||
|
`plex/subtitles.py`, `plex/sync.py` and two route modules all called themselves `siftlode.plex`, and
|
||||||
|
`downloads/pipeline.py` called itself `siftlode.worker`. A level pinned per logger could not separate
|
||||||
|
them, and neither can E22.6's console filter — the `logger` field is in every stored row precisely so
|
||||||
|
it can be filtered on.
|
||||||
|
|
||||||
|
So the name is derived, not written: `logs.get_logger(__name__)`. These cases pin the derivation and
|
||||||
|
the rule, because the rule is only worth having if a hand-rolled name fails the gate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import logging
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app import logs
|
||||||
|
|
||||||
|
APP = pathlib.Path(logs.__file__).resolve().parent
|
||||||
|
_ESCAPE = "logger-name-ok"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("module", "expected"),
|
||||||
|
[
|
||||||
|
("app.plex.sync", "siftlode.plex.sync"),
|
||||||
|
("app.plex.stream", "siftlode.plex.stream"),
|
||||||
|
("app.routes.channels", "siftlode.routes.channels"),
|
||||||
|
("app.worker", "siftlode.worker"),
|
||||||
|
# Not a module of ours (a script, a doctest): it lands on the root of our tree rather than
|
||||||
|
# inventing `siftlode.__main__`, which nothing would think to filter on.
|
||||||
|
("__main__", "siftlode"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_the_name_follows_the_module(module, expected):
|
||||||
|
assert logs.get_logger(module).name == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_derived_logger_stays_under_the_one_tree():
|
||||||
|
"""`configure_logging` hangs the level and the handler off the root, and the tests attach their
|
||||||
|
sink to `siftlode` — a name outside that tree would be invisible to both."""
|
||||||
|
for module in ("app.main", "app.plex.sync", "app.routes.plex.people", "app"):
|
||||||
|
assert logs.get_logger(module).name.startswith("siftlode")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_module_logger_is_reachable_by_its_derived_name():
|
||||||
|
"""Not a new logger object per call: the stdlib registry is keyed by name, and an operator
|
||||||
|
pinning `siftlode.plex.sync` has to reach the same instance the module logs through."""
|
||||||
|
from app.plex import sync
|
||||||
|
|
||||||
|
assert sync.log is logging.getLogger("siftlode.plex.sync")
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_module_hand_rolls_a_logger_name():
|
||||||
|
offenders = []
|
||||||
|
for path in sorted(APP.rglob("*.py")):
|
||||||
|
src = path.read_text(encoding="utf-8")
|
||||||
|
lines = src.splitlines()
|
||||||
|
for node in ast.walk(ast.parse(src)):
|
||||||
|
if not isinstance(node, ast.Call):
|
||||||
|
continue
|
||||||
|
func = node.func
|
||||||
|
if not (isinstance(func, ast.Attribute) and func.attr == "getLogger"):
|
||||||
|
continue
|
||||||
|
# A no-argument `logging.getLogger()` is the ROOT logger — level plumbing, not a
|
||||||
|
# module's own logger, and nothing to name.
|
||||||
|
if not node.args:
|
||||||
|
continue
|
||||||
|
start = max(0, node.lineno - 2) # the call's lines, plus the one above it
|
||||||
|
span = " ".join(lines[start : (node.end_lineno or node.lineno)])
|
||||||
|
if _ESCAPE in span:
|
||||||
|
continue
|
||||||
|
offenders.append(f"{path.relative_to(APP)}:{node.lineno}")
|
||||||
|
assert offenders == [], (
|
||||||
|
"a module names its own logger; use `logs.get_logger(__name__)`, or say why the name is "
|
||||||
|
f"not the module's with `# {_ESCAPE}: <why>` — {offenders}"
|
||||||
|
)
|
||||||
@@ -225,7 +225,9 @@ def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
|||||||
log.addHandler(sink)
|
log.addHandler(sink)
|
||||||
try:
|
try:
|
||||||
for i in range(logsink.BATCH_MAX + 5):
|
for i in range(logsink.BATCH_MAX + 5):
|
||||||
log.warning(f"line {i}") # distinct templates: no flood limiter involved
|
# noqa G004: the f-string is the POINT — a distinct rendered message per line is
|
||||||
|
# what keeps the flood limiter (which groups by template) out of this case.
|
||||||
|
log.warning(f"line {i}") # noqa: G004
|
||||||
finally:
|
finally:
|
||||||
log.removeHandler(sink)
|
log.removeHandler(sink)
|
||||||
sink._stopping.set()
|
sink._stopping.set()
|
||||||
|
|||||||
+2
-1
@@ -166,7 +166,8 @@ def main() -> None:
|
|||||||
saved = svid.id
|
saved = svid.id
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
print(
|
# noqa T201: this script's stdout IS its interface — the e2e runner parses this JSON.
|
||||||
|
print( # noqa: T201
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
"ok": True,
|
"ok": True,
|
||||||
|
|||||||
Reference in New Issue
Block a user